From 0709b0b3ab4ca3f2ef75d92104a5a3404ca7d626 Mon Sep 17 00:00:00 2001
From: Civitaspo <civitaspo@gmail.com>
Date: Wed, 14 Nov 2018 00:39:31 +0900
Subject: [PATCH 01/17] Add examples for ecs_task.{rb,sh}

---
 example/ecs_task.rb/echo.rb     |  5 +++++
 example/ecs_task.rb/example.dig | 19 +++++++++++++++++++
 example/ecs_task.sh/example.dig | 10 ++++++++++
 example/example.dig             | 33 ++++++++++++++-------------------
 example/run.sh                  | 13 ++++++++++++-
 5 files changed, 60 insertions(+), 20 deletions(-)
 create mode 100644 example/ecs_task.rb/echo.rb
 create mode 100644 example/ecs_task.rb/example.dig
 create mode 100644 example/ecs_task.sh/example.dig

diff --git a/example/ecs_task.rb/echo.rb b/example/ecs_task.rb/echo.rb
new file mode 100644
index 0000000..b8f89fc
--- /dev/null
+++ b/example/ecs_task.rb/echo.rb
@@ -0,0 +1,5 @@
+require 'awesome_print'
+
+def echo(message)
+  ap message
+end
diff --git a/example/ecs_task.rb/example.dig b/example/ecs_task.rb/example.dig
new file mode 100644
index 0000000..9e96d03
--- /dev/null
+++ b/example/ecs_task.rb/example.dig
@@ -0,0 +1,19 @@
+
++ecs_task.rb:
+  +step0:
+    ecs_task.rb>: echo
+    require: echo
+    gem_install: [awesome_print]
+    image: civitaspo/ruby-awscli:latest
+    _export:
+      message:
+        message: 'hello ecs_task.rb'
+        created_by: civitaspo
+    tmp_storage:
+      type: s3
+      uri: ${output}
+    family: hello_ecs_task_rb
+    cluster: ${cluster}
+    network_mode: host
+    memory: 1 GB
+
diff --git a/example/ecs_task.sh/example.dig b/example/ecs_task.sh/example.dig
new file mode 100644
index 0000000..ec6835b
--- /dev/null
+++ b/example/ecs_task.sh/example.dig
@@ -0,0 +1,10 @@
+
++ecs_task.sh:
+  +step0:
+    ecs_task.sh>: env
+    image: civitaspo/digdag-awscli:latest
+    _export:
+      message:
+        message: 'hello ecs_task.rb'
+        created_by: civitaspo
+
diff --git a/example/example.dig b/example/example.dig
index 15d0fd1..6f7965d 100644
--- a/example/example.dig
+++ b/example/example.dig
@@ -7,23 +7,18 @@ _export:
       - pro.civitaspo:digdag-operator-ecs_task:0.0.6
   ecs_task:
     auth_method: profile
+    tmp_storage:
+      type: s3
+      uri: ${output}
+    family_prefix: hello-
+    cluster: ${cluster}
+    network_mode: host
+    memory: 1 GB
+    task_role_arn: ${task_role_arn}
 
-+step0:
-  sh>: echo '{"store_params":{"civi":"taspo"}}' | aws s3 cp - ${output}
-+step1:
-  ecs_task.run>:
-  def:
-    network_mode: Host
-    container_definitions:
-      - name: uploader
-        image: amazonlinux:2
-        command: [yum, install, '-y', awscli]
-        essential: true
-        memory: 500
-        cpu: 10
-    family: hello_world
-  cluster: ${cluster}
-  count: 1
-  result_s3_uri: ${output}
-+step2:
-  echo>: ${civi}
+
++ecs_task.sh:
+  call>: ecs_task.sh/example
+
++ecs_task.rb:
+  call>: ecs_task.rb/example
diff --git a/example/run.sh b/example/run.sh
index fa1a646..d2023d0 100755
--- a/example/run.sh
+++ b/example/run.sh
@@ -6,6 +6,7 @@ LOCAL_MAVEN_REPO=$ROOT/build/repo
 
 CLUSTER="$1"
 OUTPUT="$2"
+TASK_ROLE_ARN="$3"
 
 if [ -z "$CLUSTER" ]; then
     echo "[ERROR] Set cluster as the first argument."
@@ -15,6 +16,10 @@ if [ -z "$OUTPUT" ]; then
     echo "[ERROR] Set output s3 URI as the second argument."
     exit 1
 fi
+if [ -z "$TASK_ROLE_ARN" ]; then
+    echo "[ERROR] Set task role arn as the third argument."
+    exit 1
+fi
 
 (
   cd $EXAMPLE_ROOT
@@ -23,5 +28,11 @@ fi
   rm -rfv .digdag
 
   ## run
-  digdag run example.dig -c digdag.properties -p repos=${LOCAL_MAVEN_REPO} -p output=${OUTPUT} -p cluster=${CLUSTER}  --no-save
+  digdag run example.dig                       \
+             -c digdag.properties              \
+             -p repos=${LOCAL_MAVEN_REPO}      \
+             -p output=${OUTPUT}               \
+             -p cluster=${CLUSTER}             \
+             -p task_role_arn=${TASK_ROLE_ARN} \
+             --no-save
 )

From 38889415296ed2962dc5d6a039d1ed25eb78d508 Mon Sep 17 00:00:00 2001
From: Civitaspo <civitaspo@gmail.com>
Date: Wed, 14 Nov 2018 00:55:31 +0900
Subject: [PATCH 02/17] [Enhancement] Always nomalize family name

---
 README.md                                     |  6 +-----
 .../command/EcsTaskCommandRunner.scala        | 19 +++++++++++++++----
 2 files changed, 16 insertions(+), 9 deletions(-)

diff --git a/README.md b/README.md
index 3e697b7..39825ea 100644
--- a/README.md
+++ b/README.md
@@ -138,12 +138,8 @@ In addition, the below configurations exist.
 - **execution_role_arn**: The Amazon Resource Name (ARN) of the task execution role that the Amazon ECS container agent and the Docker daemon can assume. (string, optional)
 - **family_prefix**: The family name prefix for a task definition. This is used if **family** is not defined. (string, default: `""`)
 - **family_infix**: The family name infix for a task definition. This is used if **family** is not defined. (string, default: `"${task_name}"`)
-    - The default value is replaced as below: 
-        - `+` -> `_`
-        - `^` -> `_`
-        - `=` -> `_`
 - **family_suffix**: The family name sufix for a task definition. This is used if **family** is not defined. (string, default: `""`)
-- **family**: You must specify a `family` for a task definition, which allows you to track multiple versions of the same task definition. The `family` is used as a name for your task definition. Up to 255 letters (uppercase and lowercase), numbers, hyphens, and underscores are allowed. (string, default: `"${family_prefix}${family_infix}${family_suffix}"`)
+- **family**: You must specify a `family` for a task definition, which allows you to track multiple versions of the same task definition. The `family` is used as a name for your task definition. Up to 255 letters (uppercase and lowercase), numbers, hyphens, and underscores are allowed. If invalid charactors are found, these are replaced to `"_"`. (string, default: `"${family_prefix}${family_infix}${family_suffix}"`)
 - **memory**: The amount of memory (in MiB) used by the task. It can be expressed as an integer using MiB, for example `1024`, or as a string using GB, for example `1GB` or `1 GB`, in a task definition. String values are converted to an integer indicating the MiB when the task definition is registered. (string, optional)
     - See the docs for more info: [ECS-RegisterTaskDefinition-request-memory](https://docs.aws.amazon.com/AmazonECS/latest/APIReference/API_RegisterTaskDefinition.html#ECS-RegisterTaskDefinition-request-memory)
 - **network_mode**: The Docker networking mode to use for the containers in the task. The valid values are `none`, `bridge`, `awsvpc`, and `host`. The default Docker network mode is `bridge`. If using the Fargate launch type, the `awsvpc` network mode is required. If using the EC2 launch type, any network mode can be used. If the network mode is set to `none`, you can't specify port mappings in your container definitions, and the task's containers do not have external connectivity. The `host` and `awsvpc` network modes offer the highest networking performance for containers because they use the EC2 network stack instead of the virtualized network stack provided by the `bridge` mode. With the `host` and `awsvpc` network modes, exposed container ports are mapped directly to the corresponding host port (for the `host` network mode) or the attached elastic network interface port (for the `awsvpc` network mode), so you cannot take advantage of dynamic host port mappings. If the network mode is `awsvpc`, the task is allocated an Elastic Network Interface, and you must specify the **network_configuration** option when you create a service or run a task with the task definition. For more information, see [Task Networking](http://docs.aws.amazon.com/AmazonECS/latest/developerguide/task-networking.html) in the Amazon Elastic Container Service Developer Guide. If the network mode is `host`, you can't run multiple instantiations of the same task on a single container instance when port mappings are used. Docker for Windows uses different network modes than Docker for Linux. When you register a task definition with Windows containers, you must not specify a network mode. (string, optional)
diff --git a/src/main/scala/pro/civitaspo/digdag/plugin/ecs_task/command/EcsTaskCommandRunner.scala b/src/main/scala/pro/civitaspo/digdag/plugin/ecs_task/command/EcsTaskCommandRunner.scala
index 6e91d86..659efbc 100644
--- a/src/main/scala/pro/civitaspo/digdag/plugin/ecs_task/command/EcsTaskCommandRunner.scala
+++ b/src/main/scala/pro/civitaspo/digdag/plugin/ecs_task/command/EcsTaskCommandRunner.scala
@@ -8,6 +8,7 @@ import pro.civitaspo.digdag.plugin.ecs_task.VERSION
 import pro.civitaspo.digdag.plugin.ecs_task.aws.AwsConf
 
 import scala.collection.JavaConverters._
+import scala.util.matching.Regex
 
 case class EcsTaskCommandRunner(
   tmpStorage: TmpStorage,
@@ -30,7 +31,7 @@ case class EcsTaskCommandRunner(
   val taskName: String = params.get("task_name", classOf[String])
   val familyPrefix: String = params.get("family_prefix", classOf[String], "")
   val familySuffix: String = params.get("family_suffix", classOf[String], "")
-  val familyInfix: String = params.get("family_infix", classOf[String], taskName.replaceAll("\\+", "_").replaceAll("\\^", "_").replaceAll("\\=", "_"))
+  val familyInfix: String = params.get("family_infix", classOf[String], taskName)
   val family: String = params.get("family", classOf[String], s"$familyPrefix$familyInfix$familySuffix")
   val memory: Optional[String] = params.getOptional("memory", classOf[String])
   val networkMode: Optional[String] = params.getOptional("network_mode", classOf[String])
@@ -69,7 +70,7 @@ case class EcsTaskCommandRunner(
   // NOTE: If you set it by container level, use the `overrides` option.
   // val memoryReservation: Optional[Int] = params.getOptional("memory_reservation", classOf[Int])
   val mountPoints: Seq[Config] = params.parseListOrGetEmpty("mount_points", classOf[Config]).asScala
-  val containerName: String = params.get("container_name", classOf[String], family)
+  val containerName: Optional[String]= params.getOptional("container_name", classOf[String])
   val portMappings: Seq[Config] = params.parseListOrGetEmpty("port_mappings", classOf[Config]).asScala
   val privileged: Optional[Boolean] = params.getOptional("privileged", classOf[Boolean])
   val pseudoTerminal: Optional[Boolean] = params.getOptional("pseudo_terminal", classOf[Boolean])
@@ -98,6 +99,8 @@ case class EcsTaskCommandRunner(
   // For ecs_task.wait operator
   val timeout: DurationParam = params.get("timeout", classOf[DurationParam], DurationParam.parse("15m"))
 
+  lazy val normalizedFamily: String = normalizeFamily(family)
+
   def run(): TaskResult = {
     val subTasks: Config = cf.create()
     subTasks.setNested("+register", ecsTaskRegisterSubTask())
@@ -171,7 +174,7 @@ case class EcsTaskCommandRunner(
     c.set("container_definitions", (Seq(containerDefinitionConfig()) ++ sidecars).asJava)
     c.setOptional("cpu", cpu)
     c.setOptional("execution_role_arn", executionRoleArn)
-    c.set("family", family)
+    c.set("family", normalizedFamily)
     c.setOptional("memory", memory)
     c.setOptional("network_mode", networkMode)
     c.set("requires_compatibilities", requiresCompatibilities.asJava)
@@ -203,7 +206,7 @@ case class EcsTaskCommandRunner(
     c.setOptional("linux_parameters", linuxParameters)
     c.setOptional("log_configuration", logConfiguration)
     c.set("mount_points", mountPoints.asJava)
-    c.set("name", containerName)
+    c.set("name", containerName.or(normalizedFamily))
     c.set("port_mappings", portMappings.asJava)
     c.setOptional("privileged", privileged)
     c.setOptional("pseudo_terminal", pseudoTerminal)
@@ -218,4 +221,12 @@ case class EcsTaskCommandRunner(
     c
   }
 
+  protected def normalizeFamily(family: String): String = {
+    // ref. https://docs.aws.amazon.com/AmazonECS/latest/APIReference/API_RegisterTaskDefinition.html#ECS-RegisterTaskDefinition-request-family
+    val validLetterRegex: Regex = "[a-zA-Z0-9_-]".r
+    val after: String = family.map{ case l @ validLetterRegex() => l; case _ => "_" }.mkString
+    if (!family.contentEquals(after)) logger.warn(s"Normalized family: $family -> $after")
+    after
+  }
+
 }

From 03efc769dbfbef2ba7e8a0503788f8c24d5cf745 Mon Sep 17 00:00:00 2001
From: Civitaspo <civitaspo@gmail.com>
Date: Sat, 17 Nov 2018 00:16:37 +0900
Subject: [PATCH 03/17] Add examples for ecs_task.embulk

---
 example/ecs_task.embulk/csv/data.01.csv.gz | Bin 0 -> 50 bytes
 example/ecs_task.embulk/csv/data.02.csv.gz | Bin 0 -> 50 bytes
 example/ecs_task.embulk/example.dig        |  31 +++++++++++++++++++++
 example/ecs_task.embulk/template.yml       |  20 +++++++++++++
 example/ecs_task.rb/example.dig            |   7 -----
 example/example.dig                        |   5 +++-
 6 files changed, 55 insertions(+), 8 deletions(-)
 create mode 100644 example/ecs_task.embulk/csv/data.01.csv.gz
 create mode 100644 example/ecs_task.embulk/csv/data.02.csv.gz
 create mode 100644 example/ecs_task.embulk/example.dig
 create mode 100644 example/ecs_task.embulk/template.yml

diff --git a/example/ecs_task.embulk/csv/data.01.csv.gz b/example/ecs_task.embulk/csv/data.01.csv.gz
new file mode 100644
index 0000000000000000000000000000000000000000..8e8226086269ccc270f3c1a206125fdaca519d96
GIT binary patch
literal 50
zcmb2|=HU4E<5e^Rb4p@KqMm`FUUG36gYgwlAD<^impr|^9vNNp^z(bh#Bkn>d8H5o
G0|Nlyd=bL{

literal 0
HcmV?d00001

diff --git a/example/ecs_task.embulk/csv/data.02.csv.gz b/example/ecs_task.embulk/csv/data.02.csv.gz
new file mode 100644
index 0000000000000000000000000000000000000000..8fda11cf30a1292648825fb6230a99c70812f497
GIT binary patch
literal 50
zcmb2|=HQ6C_b!@&IVG_qQP03gFS)pk!T5@&kIxgMOP*d{kBqK)`uRO$VmNQcyi$mP
GfdK%&{1FZS

literal 0
HcmV?d00001

diff --git a/example/ecs_task.embulk/example.dig b/example/ecs_task.embulk/example.dig
new file mode 100644
index 0000000..33763d1
--- /dev/null
+++ b/example/ecs_task.embulk/example.dig
@@ -0,0 +1,31 @@
+
++ecs_task.embulk:
+  _export:
+    path_prefix: ./csv/
+  +dig:
+    ecs_task.embulk>:
+      in:
+        type: file
+        path_prefix: ${path_prefix}
+        decoders:
+          - {type: gzip}
+        parser:
+          charset: UTF-8
+          newline: CRLF
+          type: csv
+          delimiter: ','
+          quote: '"'
+          escape: '"'
+          null_string: 'NULL'
+          skip_header_lines: 0
+          columns:
+            - {name: id, type: long}
+            - {name: comment, type: string}
+      out:
+        type: stdout
+    image: civitaspo/embulk-awscli:latest
+
+  +file:
+    ecs_task.embulk>: template.yml
+    image: civitaspo/embulk-awscli:latest
+
diff --git a/example/ecs_task.embulk/template.yml b/example/ecs_task.embulk/template.yml
new file mode 100644
index 0000000..bf75719
--- /dev/null
+++ b/example/ecs_task.embulk/template.yml
@@ -0,0 +1,20 @@
+in:
+  type: file
+  path_prefix: ${path_prefix}
+  decoders:
+    - {type: gzip}
+  parser:
+    charset: UTF-8
+    newline: CRLF
+    type: csv
+    delimiter: ','
+    quote: '"'
+    escape: '"'
+    null_string: 'NULL'
+    skip_header_lines: 0
+    columns:
+      - {name: id, type: long}
+      - {name: comment, type: string}
+out:
+  type: stdout
+
diff --git a/example/ecs_task.rb/example.dig b/example/ecs_task.rb/example.dig
index 9e96d03..47644dd 100644
--- a/example/ecs_task.rb/example.dig
+++ b/example/ecs_task.rb/example.dig
@@ -9,11 +9,4 @@
       message:
         message: 'hello ecs_task.rb'
         created_by: civitaspo
-    tmp_storage:
-      type: s3
-      uri: ${output}
-    family: hello_ecs_task_rb
-    cluster: ${cluster}
-    network_mode: host
-    memory: 1 GB
 
diff --git a/example/example.dig b/example/example.dig
index 6f7965d..a57d41f 100644
--- a/example/example.dig
+++ b/example/example.dig
@@ -16,9 +16,12 @@ _export:
     memory: 1 GB
     task_role_arn: ${task_role_arn}
 
-
 +ecs_task.sh:
   call>: ecs_task.sh/example
 
 +ecs_task.rb:
   call>: ecs_task.rb/example
+
++ecs_task.embulk:
+  call>: ecs_task.embulk/example
+

From 29142d66a40342db264e03ec91dd09e9c902ebc9 Mon Sep 17 00:00:00 2001
From: Civitaspo <civitaspo@gmail.com>
Date: Sat, 17 Nov 2018 10:58:53 +0900
Subject: [PATCH 04/17] Add examples for ecs_task.py

---
 example/ecs_task.py/echo.py     |  6 ++++++
 example/ecs_task.py/example.dig | 11 +++++++++++
 example/example.dig             |  3 +++
 3 files changed, 20 insertions(+)
 create mode 100644 example/ecs_task.py/echo.py
 create mode 100644 example/ecs_task.py/example.dig

diff --git a/example/ecs_task.py/echo.py b/example/ecs_task.py/echo.py
new file mode 100644
index 0000000..3eb2ed6
--- /dev/null
+++ b/example/ecs_task.py/echo.py
@@ -0,0 +1,6 @@
+import yaml
+
+
+def echo(message):
+    print(yaml.dump(message))
+
diff --git a/example/ecs_task.py/example.dig b/example/ecs_task.py/example.dig
new file mode 100644
index 0000000..07ba4bd
--- /dev/null
+++ b/example/ecs_task.py/example.dig
@@ -0,0 +1,11 @@
+
++ecs_task.py:
+  +step0:
+    ecs_task.py>: echo.echo
+    pip_install: [PyYaml]
+    image: civitaspo/python-awscli:latest
+    _export:
+      message:
+        message: 'hello ecs_task.py'
+        created_by: civitaspo
+
diff --git a/example/example.dig b/example/example.dig
index a57d41f..2e23f55 100644
--- a/example/example.dig
+++ b/example/example.dig
@@ -22,6 +22,9 @@ _export:
 +ecs_task.rb:
   call>: ecs_task.rb/example
 
++ecs_task.py:
+  call>: ecs_task.py/example
+
 +ecs_task.embulk:
   call>: ecs_task.embulk/example
 

From 8a80aaa58e8c5cfaf695f62f4c9af1e4a99303c3 Mon Sep 17 00:00:00 2001
From: Civitaspo <civitaspo@gmail.com>
Date: Sat, 17 Nov 2018 11:27:59 +0900
Subject: [PATCH 05/17] Add examples for ecs_task.run

---
 example/ecs_task.run/example.dig | 49 ++++++++++++++++++++++++++++++++
 example/example.dig              |  2 ++
 2 files changed, 51 insertions(+)
 create mode 100644 example/ecs_task.run/example.dig

diff --git a/example/ecs_task.run/example.dig b/example/ecs_task.run/example.dig
new file mode 100644
index 0000000..de786fd
--- /dev/null
+++ b/example/ecs_task.run/example.dig
@@ -0,0 +1,49 @@
+
++ecs_task.run:
+  +step1:
+    ecs_task.run>:
+    def:
+      network_mode: host
+      container_definitions:
+        - name: step1
+          image: civitaspo/python-awscli:latest
+          command: [echo, step1]
+          essential: true
+          memory: 500
+          cpu: 1
+      family: step1
+    count: 1
+  +step2:
+    ecs_task.run>:
+    def:
+      network_mode: host
+      container_definitions:
+        - name: step2
+          image: civitaspo/python-awscli:latest
+          command: [echo, step2]
+          essential: true
+          memory: 500
+          cpu: 1
+      family: step2
+    count: 1
+  +step3:
+    ecs_task.run>:
+    def:
+      network_mode: host
+      container_definitions:
+        - name: step3
+          image: civitaspo/python-awscli:latest
+          command:
+            - sh
+            - -c
+            - echo '{"store_params":{"civi":"taspo"}}' | aws s3 cp - ${output}/${session_uuid}.json
+          essential: true
+          memory: 500
+          cpu: 1
+      task_role_arn: ${task_role_arn}
+      family: step3
+    count: 1
+    result_s3_uri: ${output}/${session_uuid}.json
+
+  +step4:
+    echo>: ${civi}
diff --git a/example/example.dig b/example/example.dig
index 2e23f55..1b5803d 100644
--- a/example/example.dig
+++ b/example/example.dig
@@ -28,3 +28,5 @@ _export:
 +ecs_task.embulk:
   call>: ecs_task.embulk/example
 
++ecs_task.run:
+  call>: ecs_task.run/example

From 454d0714bddd34fedbdd1cd266e7cbaaf1a56d1b Mon Sep 17 00:00:00 2001
From: Civitaspo <civitaspo@gmail.com>
Date: Sun, 18 Nov 2018 03:59:42 +0900
Subject: [PATCH 06/17] Update README: usage

---
 README.md | 143 +++++++++++++++++++++++++++++++++++++++++++++---------
 1 file changed, 120 insertions(+), 23 deletions(-)

diff --git a/README.md b/README.md
index 39825ea..485bb58 100644
--- a/README.md
+++ b/README.md
@@ -18,31 +18,128 @@ _export:
       - pro.civitaspo:digdag-operator-ecs_task:0.0.6
   ecs_task:
     auth_method: profile
-
-+step0:
-  sh>: echo '{"store_params":{"civi":"taspo"}}' | aws s3 cp - ${output}
-
-+step1:
-  ecs_task.run>:
-  def:
-    network_mode: Host
-    container_definitions:
-      - name: uploader
-        image: amazonlinux:2
-        command: [yum, install, '-y', awscli]
-        essential: true
-        memory: 500
-        cpu: 10
-    family: hello_world
-  cluster: ${cluster}
-  count: 1
-  result_s3_uri: ${output}
-
-+step2:
-  echo>: ${civi}
+    tmp_storage:
+      type: s3
+      uri: ${output}
+    family_prefix: hello-
+    cluster: ${cluster}
+    network_mode: host
+    memory: 1 GB
+    task_role_arn: ${task_role_arn}
+
++ecs_task.run:
+  +step1:
+    ecs_task.run>:
+    def:
+      network_mode: host
+      container_definitions:
+        - name: step1
+          image: civitaspo/python-awscli:latest
+          command: [echo, step1]
+          essential: true
+          memory: 500
+          cpu: 1
+      family: step1
+    count: 1
+  +step2:
+    ecs_task.run>:
+    def:
+      network_mode: host
+      container_definitions:
+        - name: step2
+          image: civitaspo/python-awscli:latest
+          command: [echo, step2]
+          essential: true
+          memory: 500
+          cpu: 1
+      family: step2
+    count: 1
+  +step3:
+    ecs_task.run>:
+    def:
+      network_mode: host
+      container_definitions:
+        - name: step3
+          image: civitaspo/python-awscli:latest
+          command:
+            - sh
+            - -c
+            - echo '{"store_params":{"civi":"taspo"}}' | aws s3 cp - ${output}/${session_uuid}.json
+          essential: true
+          memory: 500
+          cpu: 1
+      task_role_arn: ${task_role_arn}
+      family: step3
+    count: 1
+    result_s3_uri: ${output}/${session_uuid}.json
+
+  +step4:
+    echo>: ${civi}
+
++ecs_task.sh:
+  +step0:
+    ecs_task.sh>: env
+    image: civitaspo/digdag-awscli:latest
+    _export:
+      message:
+        message: 'hello ecs_task.rb'
+        created_by: civitaspo
+
++ecs_task.rb:
+  +step0:
+    ecs_task.rb>: echo
+    require: echo
+    gem_install: [awesome_print]
+    image: civitaspo/ruby-awscli:latest
+    _export:
+      message:
+        message: 'hello ecs_task.rb'
+        created_by: civitaspo
+
++ecs_task.py:
+  +step0:
+    ecs_task.py>: echo.echo
+    pip_install: [PyYaml]
+    image: civitaspo/python-awscli:latest
+    _export:
+      message:
+        message: 'hello ecs_task.py'
+        created_by: civitaspo
+
++ecs_task.embulk:
+  _export:
+    path_prefix: ./csv/
+  +dig:
+    ecs_task.embulk>:
+      in:
+        type: file
+        path_prefix: ${path_prefix}
+        decoders:
+          - {type: gzip}
+        parser:
+          charset: UTF-8
+          newline: CRLF
+          type: csv
+          delimiter: ','
+          quote: '"'
+          escape: '"'
+          null_string: 'NULL'
+          skip_header_lines: 0
+          columns:
+            - {name: id, type: long}
+            - {name: comment, type: string}
+      out:
+        type: stdout
+    image: civitaspo/embulk-awscli:latest
+
+  +file:
+    ecs_task.embulk>: template.yml
+    image: civitaspo/embulk-awscli:latest
 
 ```
 
+See [example](./example).
+
 # Configuration
 
 ## Remarks
@@ -241,7 +338,7 @@ aws configure
 ### 3) run an example
 
 ```sh
-./example/run.sh
+./example/run.sh ${ECS Cluster Name} ${S3 URI Prefix for tmp storage} ${ECS Task Role ARN}
 ```
 
 ## (TODO) Run Tests

From 5942c4b4195772e1d504f1be3de7f3012b7e5e79 Mon Sep 17 00:00:00 2001
From: Civitaspo <civitaspo@gmail.com>
Date: Sun, 18 Nov 2018 04:04:30 +0900
Subject: [PATCH 07/17] Modify ecs_task.embulk example: Use raw file instead of
 gzip for readability

---
 README.md                                  |   2 --
 example/ecs_task.embulk/csv/data.01.csv    |   3 +++
 example/ecs_task.embulk/csv/data.01.csv.gz | Bin 50 -> 0 bytes
 example/ecs_task.embulk/csv/data.02.csv    |   3 +++
 example/ecs_task.embulk/csv/data.02.csv.gz | Bin 50 -> 0 bytes
 example/ecs_task.embulk/example.dig        |   2 --
 example/ecs_task.embulk/template.yml       |   2 --
 7 files changed, 6 insertions(+), 6 deletions(-)
 create mode 100644 example/ecs_task.embulk/csv/data.01.csv
 delete mode 100644 example/ecs_task.embulk/csv/data.01.csv.gz
 create mode 100644 example/ecs_task.embulk/csv/data.02.csv
 delete mode 100644 example/ecs_task.embulk/csv/data.02.csv.gz

diff --git a/README.md b/README.md
index 485bb58..5d038d3 100644
--- a/README.md
+++ b/README.md
@@ -114,8 +114,6 @@ _export:
       in:
         type: file
         path_prefix: ${path_prefix}
-        decoders:
-          - {type: gzip}
         parser:
           charset: UTF-8
           newline: CRLF
diff --git a/example/ecs_task.embulk/csv/data.01.csv b/example/ecs_task.embulk/csv/data.01.csv
new file mode 100644
index 0000000..231608c
--- /dev/null
+++ b/example/ecs_task.embulk/csv/data.01.csv
@@ -0,0 +1,3 @@
+1,aaa
+2,bbb
+3,ccc
diff --git a/example/ecs_task.embulk/csv/data.01.csv.gz b/example/ecs_task.embulk/csv/data.01.csv.gz
deleted file mode 100644
index 8e8226086269ccc270f3c1a206125fdaca519d96..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 50
zcmb2|=HU4E<5e^Rb4p@KqMm`FUUG36gYgwlAD<^impr|^9vNNp^z(bh#Bkn>d8H5o
G0|Nlyd=bL{

diff --git a/example/ecs_task.embulk/csv/data.02.csv b/example/ecs_task.embulk/csv/data.02.csv
new file mode 100644
index 0000000..231608c
--- /dev/null
+++ b/example/ecs_task.embulk/csv/data.02.csv
@@ -0,0 +1,3 @@
+1,aaa
+2,bbb
+3,ccc
diff --git a/example/ecs_task.embulk/csv/data.02.csv.gz b/example/ecs_task.embulk/csv/data.02.csv.gz
deleted file mode 100644
index 8fda11cf30a1292648825fb6230a99c70812f497..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 50
zcmb2|=HQ6C_b!@&IVG_qQP03gFS)pk!T5@&kIxgMOP*d{kBqK)`uRO$VmNQcyi$mP
GfdK%&{1FZS

diff --git a/example/ecs_task.embulk/example.dig b/example/ecs_task.embulk/example.dig
index 33763d1..250e2e4 100644
--- a/example/ecs_task.embulk/example.dig
+++ b/example/ecs_task.embulk/example.dig
@@ -7,8 +7,6 @@
       in:
         type: file
         path_prefix: ${path_prefix}
-        decoders:
-          - {type: gzip}
         parser:
           charset: UTF-8
           newline: CRLF
diff --git a/example/ecs_task.embulk/template.yml b/example/ecs_task.embulk/template.yml
index bf75719..c3a98be 100644
--- a/example/ecs_task.embulk/template.yml
+++ b/example/ecs_task.embulk/template.yml
@@ -1,8 +1,6 @@
 in:
   type: file
   path_prefix: ${path_prefix}
-  decoders:
-    - {type: gzip}
   parser:
     charset: UTF-8
     newline: CRLF

From db69f9630b7d04753fd81d5e00e6e70063b5f126 Mon Sep 17 00:00:00 2001
From: Civitaspo <civitaspo@gmail.com>
Date: Sun, 18 Nov 2018 04:13:46 +0900
Subject: [PATCH 08/17] Update aws-java-sdk 1.11.433 -> 1.11.451

---
 build.gradle | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/build.gradle b/build.gradle
index e36184e..075cb51 100644
--- a/build.gradle
+++ b/build.gradle
@@ -29,7 +29,7 @@ dependencies {
 
     ['ecs', 's3', 'sts'].each { svc ->
         // https://mvnrepository.com/artifact/com.amazonaws/
-        compile group: 'com.amazonaws', name: "aws-java-sdk-${svc}", version: '1.11.433'
+        compile group: 'com.amazonaws', name: "aws-java-sdk-${svc}", version: '1.11.451'
     }
 }
 

From 4313c9a6014e8c2687b04279273f991cbb072e1e Mon Sep 17 00:00:00 2001
From: Civitaspo <civitaspo@gmail.com>
Date: Sun, 18 Nov 2018 04:28:04 +0900
Subject: [PATCH 09/17] [ecs_task.register] Add `secrets` option for
 `container_definitions`

---
 .../register/EcsTaskRegisterOperator.scala       | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)

diff --git a/src/main/scala/pro/civitaspo/digdag/plugin/ecs_task/register/EcsTaskRegisterOperator.scala b/src/main/scala/pro/civitaspo/digdag/plugin/ecs_task/register/EcsTaskRegisterOperator.scala
index 930fa74..7cf6bc3 100644
--- a/src/main/scala/pro/civitaspo/digdag/plugin/ecs_task/register/EcsTaskRegisterOperator.scala
+++ b/src/main/scala/pro/civitaspo/digdag/plugin/ecs_task/register/EcsTaskRegisterOperator.scala
@@ -15,6 +15,7 @@ import com.amazonaws.services.ecs.model.{
   RegisterTaskDefinitionRequest,
   RegisterTaskDefinitionResult,
   RepositoryCredentials,
+  Secret,
   SystemControl,
   TaskDefinitionPlacementConstraint,
   Tmpfs,
@@ -104,6 +105,7 @@ class EcsTaskRegisterOperator(operatorName: String, context: OperatorContext, sy
     val pseudoTerminal: Optional[Boolean] = c.getOptional("pseudo_terminal", classOf[Boolean])
     val readonlyRootFilesystem: Optional[Boolean] = c.getOptional("readonly_root_filesystem", classOf[Boolean])
     val repositoryCredentials: Optional[RepositoryCredentials] = configureRepositoryCredentials(c.parseNestedOrGetEmpty("repository_credentials"))
+    val secrets: Seq[Secret] = c.parseListOrGetEmpty("secrets", classOf[Config]).asScala.map(configureSecrets).map(_.get)
     val systemControls: Seq[SystemControl] = c.parseListOrGetEmpty("system_controls", classOf[Config]).asScala.map(configureSystemControl).map(_.get)
     val ulimits: Seq[Ulimit] = c.parseListOrGetEmpty("ulimits", classOf[Config]).asScala.map(configureUlimit).map(_.get)
     val user: Optional[String] = c.getOptional("user", classOf[String])
@@ -138,6 +140,7 @@ class EcsTaskRegisterOperator(operatorName: String, context: OperatorContext, sy
     if (pseudoTerminal.isPresent) cd.setPseudoTerminal(pseudoTerminal.get)
     if (readonlyRootFilesystem.isPresent) cd.setReadonlyRootFilesystem(readonlyRootFilesystem.get)
     if (repositoryCredentials.isPresent) cd.setRepositoryCredentials(repositoryCredentials.get)
+    if (secrets.nonEmpty) cd.setSecrets(secrets.asJava)
     if (systemControls.nonEmpty) cd.setSystemControls(systemControls.asJava)
     if (ulimits.nonEmpty) cd.setUlimits(ulimits.asJava)
     if (user.isPresent) cd.setUser(user.get)
@@ -282,6 +285,19 @@ class EcsTaskRegisterOperator(operatorName: String, context: OperatorContext, sy
     Optional.of(rc)
   }
 
+  protected def configureSecrets(c: Config): Optional[Secret] = {
+    if (c.isEmpty) return Optional.absent()
+
+    val name: String = c.get("name", classOf[String])
+    val valueFrom: String = c.get("value_from", classOf[String])
+
+    val s: Secret = new Secret()
+    s.setName(name)
+    s.setValueFrom(valueFrom)
+
+    Optional.of(s)
+  }
+
   protected def configureSystemControl(c: Config): Optional[SystemControl] = {
     if (c.isEmpty) return Optional.absent()
 

From fb0073296798edd89da985943285e90b3c646e03 Mon Sep 17 00:00:00 2001
From: Civitaspo <civitaspo@gmail.com>
Date: Sun, 18 Nov 2018 04:34:01 +0900
Subject: [PATCH 10/17] [ecs_task.register] Add `ipc_mode`, `pid_mode` option
 (but, aws-java-sdk does not support yet, so throw
 UnsupportedOperationException)

---
 .../plugin/ecs_task/register/EcsTaskRegisterOperator.scala    | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/src/main/scala/pro/civitaspo/digdag/plugin/ecs_task/register/EcsTaskRegisterOperator.scala b/src/main/scala/pro/civitaspo/digdag/plugin/ecs_task/register/EcsTaskRegisterOperator.scala
index 7cf6bc3..b69fa38 100644
--- a/src/main/scala/pro/civitaspo/digdag/plugin/ecs_task/register/EcsTaskRegisterOperator.scala
+++ b/src/main/scala/pro/civitaspo/digdag/plugin/ecs_task/register/EcsTaskRegisterOperator.scala
@@ -44,8 +44,10 @@ class EcsTaskRegisterOperator(operatorName: String, context: OperatorContext, sy
     val cpu: Optional[String] = c.getOptional("cpu", classOf[String])
     val executionRoleArn: Optional[String] = c.getOptional("execution_role_arn", classOf[String])
     val family: String = c.get("family", classOf[String])
+    val ipcMode: Optional[String] = c.getOptional("ipc_mode", classOf[String])
     val memory: Optional[String] = c.getOptional("memory", classOf[String])
     val networkMode: Optional[String] = c.getOptional("network_mode", classOf[String])
+    val pidMode: Optional[String] = c.getOptional("pid_mode", classOf[String])
 
     val placementConstraints: Seq[TaskDefinitionPlacementConstraint] =
       c.parseListOrGetEmpty("placement_constraints", classOf[Config]).asScala.map(configureTaskDefinitionPlacementConstraint).map(_.get)
@@ -57,8 +59,10 @@ class EcsTaskRegisterOperator(operatorName: String, context: OperatorContext, sy
     if (cpu.isPresent) req.setCpu(cpu.get)
     if (executionRoleArn.isPresent) req.setExecutionRoleArn(executionRoleArn.get)
     req.setFamily(family)
+    if (ipcMode.isPresent) throw new UnsupportedOperationException("Currently aws-java-sdk does not support ipc_mode.")
     if (memory.isPresent) req.setMemory(memory.get)
     if (networkMode.isPresent) req.setNetworkMode(networkMode.get)
+    if (pidMode.isPresent) throw new UnsupportedOperationException("Currently aws-java-sdk does not support pid_mode.")
     if (placementConstraints.nonEmpty) req.setPlacementConstraints(placementConstraints.asJava)
     if (requiresCompatibilities.nonEmpty) req.setRequiresCompatibilities(requiresCompatibilities.asJava)
     if (taskRoleArn.isPresent) req.setTaskRoleArn(taskRoleArn.get)

From 11c7e0112bf03ca676dc0467de8f5a1984da76fe Mon Sep 17 00:00:00 2001
From: Civitaspo <civitaspo@gmail.com>
Date: Sun, 18 Nov 2018 04:43:14 +0900
Subject: [PATCH 11/17] [ecs_task.register] Add `tags` option

---
 .../plugin/ecs_task/register/EcsTaskRegisterOperator.scala     | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/src/main/scala/pro/civitaspo/digdag/plugin/ecs_task/register/EcsTaskRegisterOperator.scala b/src/main/scala/pro/civitaspo/digdag/plugin/ecs_task/register/EcsTaskRegisterOperator.scala
index b69fa38..6213b30 100644
--- a/src/main/scala/pro/civitaspo/digdag/plugin/ecs_task/register/EcsTaskRegisterOperator.scala
+++ b/src/main/scala/pro/civitaspo/digdag/plugin/ecs_task/register/EcsTaskRegisterOperator.scala
@@ -17,6 +17,7 @@ import com.amazonaws.services.ecs.model.{
   RepositoryCredentials,
   Secret,
   SystemControl,
+  Tag,
   TaskDefinitionPlacementConstraint,
   Tmpfs,
   Ulimit,
@@ -52,6 +53,7 @@ class EcsTaskRegisterOperator(operatorName: String, context: OperatorContext, sy
     val placementConstraints: Seq[TaskDefinitionPlacementConstraint] =
       c.parseListOrGetEmpty("placement_constraints", classOf[Config]).asScala.map(configureTaskDefinitionPlacementConstraint).map(_.get)
     val requiresCompatibilities: Seq[String] = c.parseListOrGetEmpty("requires_compatibilities", classOf[String]).asScala // Valid Values: EC2 | FARGATE
+    val tags: Seq[Tag] = c.getMapOrEmpty("tags", classOf[String], classOf[String]).asScala.map(t => new Tag().withKey(t._1).withValue(t._2)).toSeq
     val taskRoleArn: Optional[String] = c.getOptional("task_role_arn", classOf[String])
     val volumes: Seq[Volume] = c.parseListOrGetEmpty("volumes", classOf[Config]).asScala.map(configureVolume).map(_.get)
 
@@ -65,6 +67,7 @@ class EcsTaskRegisterOperator(operatorName: String, context: OperatorContext, sy
     if (pidMode.isPresent) throw new UnsupportedOperationException("Currently aws-java-sdk does not support pid_mode.")
     if (placementConstraints.nonEmpty) req.setPlacementConstraints(placementConstraints.asJava)
     if (requiresCompatibilities.nonEmpty) req.setRequiresCompatibilities(requiresCompatibilities.asJava)
+    if (tags.nonEmpty) req.setTags(tags.asJava)
     if (taskRoleArn.isPresent) req.setTaskRoleArn(taskRoleArn.get)
     if (volumes.nonEmpty) req.setVolumes(volumes.asJava)
 

From a79a04ae9cc598d8ff9726156d09204f83f97700 Mon Sep 17 00:00:00 2001
From: Civitaspo <civitaspo@gmail.com>
Date: Sun, 18 Nov 2018 04:46:31 +0900
Subject: [PATCH 12/17] [ecs_task.run] Add `tags` option

---
 .../plugin/ecs_task/run/EcsTaskRunInternalOperator.scala       | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/src/main/scala/pro/civitaspo/digdag/plugin/ecs_task/run/EcsTaskRunInternalOperator.scala b/src/main/scala/pro/civitaspo/digdag/plugin/ecs_task/run/EcsTaskRunInternalOperator.scala
index 070dd34..2f0078d 100644
--- a/src/main/scala/pro/civitaspo/digdag/plugin/ecs_task/run/EcsTaskRunInternalOperator.scala
+++ b/src/main/scala/pro/civitaspo/digdag/plugin/ecs_task/run/EcsTaskRunInternalOperator.scala
@@ -9,6 +9,7 @@ import com.amazonaws.services.ecs.model.{
   PlacementStrategy,
   RunTaskRequest,
   RunTaskResult,
+  Tag,
   TaskOverride
 }
 import com.google.common.base.Optional
@@ -38,6 +39,7 @@ class EcsTaskRunInternalOperator(operatorName: String, context: OperatorContext,
     params.parseListOrGetEmpty("placement_strategy", classOf[Config]).asScala.map(configurePlacementStrategy).map(_.get)
   val platformVersion: Optional[String] = params.getOptional("platform_version", classOf[String])
   val startedBy: Optional[String] = params.getOptional("started_by", classOf[String])
+  val tags: Seq[Tag] = params.getMapOrEmpty("tags", classOf[String], classOf[String]).asScala.map(t => new Tag().withKey(t._1).withValue(t._2)).toSeq
   val taskDefinition: String = params.get("task_definition", classOf[String]) // generated by ecs_task.register> operator if not set.
 
   val runRequestRetryInterval: DurationParam = params.get("run_request_retry_interval", classOf[DurationParam], DurationParam.parse("5s"))
@@ -56,6 +58,7 @@ class EcsTaskRunInternalOperator(operatorName: String, context: OperatorContext,
     if (placementStrategy.nonEmpty) req.setPlacementStrategy(placementStrategy.asJava)
     if (platformVersion.isPresent) req.setPlatformVersion(platformVersion.get)
     if (startedBy.isPresent) req.setStartedBy(startedBy.get)
+    if (tags.nonEmpty) req.setTags(tags.asJava)
     req.setTaskDefinition(taskDefinition)
 
     req

From c2ec570d921dfdaf4f16d364b8878ef3520f1c43 Mon Sep 17 00:00:00 2001
From: Civitaspo <civitaspo@gmail.com>
Date: Sun, 18 Nov 2018 04:58:09 +0900
Subject: [PATCH 13/17] [ecs_task.{sh,rb,py,embulk}] Add `ipc_mode`,
 `pid_mode`, `secrets`, `tags` option

---
 .../ecs_task/command/EcsTaskCommandRunner.scala       | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/src/main/scala/pro/civitaspo/digdag/plugin/ecs_task/command/EcsTaskCommandRunner.scala b/src/main/scala/pro/civitaspo/digdag/plugin/ecs_task/command/EcsTaskCommandRunner.scala
index 659efbc..5874cb9 100644
--- a/src/main/scala/pro/civitaspo/digdag/plugin/ecs_task/command/EcsTaskCommandRunner.scala
+++ b/src/main/scala/pro/civitaspo/digdag/plugin/ecs_task/command/EcsTaskCommandRunner.scala
@@ -33,11 +33,15 @@ case class EcsTaskCommandRunner(
   val familySuffix: String = params.get("family_suffix", classOf[String], "")
   val familyInfix: String = params.get("family_infix", classOf[String], taskName)
   val family: String = params.get("family", classOf[String], s"$familyPrefix$familyInfix$familySuffix")
+  val ipcMode: Optional[String] = params.getOptional("ipc_mode", classOf[String])
   val memory: Optional[String] = params.getOptional("memory", classOf[String])
   val networkMode: Optional[String] = params.getOptional("network_mode", classOf[String])
+  val pidMode: Optional[String] = params.getOptional("pid_mode", classOf[String])
   // NOTE: Use `ecs_task.run>`'s one.
   // val placementConstraints: Seq[TaskDefinitionPlacementConstraint] = params.parseListOrGetEmpty("placement_constraints", classOf[Config]).asScala.map(configureTaskDefinitionPlacementConstraint).map(_.get)
   val requiresCompatibilities: Seq[String] = params.parseListOrGetEmpty("requires_compatibilities", classOf[String]).asScala // Valid Values: EC2 | FARGATE
+  // NOTE: Use the same var as `ecs_task.run>`'s one.
+  // val tags: Optional[Config] = params.getOptionalNested("tags")
   val taskRoleArn: Optional[String] = params.getOptional("task_role_arn", classOf[String])
   val volumes: Seq[Config] = params.parseListOrGetEmpty("volumes", classOf[Config]).asScala
 
@@ -76,6 +80,7 @@ case class EcsTaskCommandRunner(
   val pseudoTerminal: Optional[Boolean] = params.getOptional("pseudo_terminal", classOf[Boolean])
   val readonlyRootFilesystem: Optional[Boolean] = params.getOptional("readonly_root_filesystem", classOf[Boolean])
   val repositoryCredentials: Optional[Config] = params.getOptionalNested("repository_credentials")
+  val secrets: Seq[Config] = params.parseListOrGetEmpty("secrets", classOf[Config]).asScala
   val systemControls: Seq[Config] = params.parseListOrGetEmpty("system_controls", classOf[Config]).asScala
   val ulimits: Seq[Config] = params.parseListOrGetEmpty("ulimits", classOf[Config]).asScala
   val user: Optional[String] = params.getOptional("user", classOf[String])
@@ -93,6 +98,7 @@ case class EcsTaskCommandRunner(
   val placementStrategy: Seq[Config] = params.parseListOrGetEmpty("placement_strategy", classOf[Config]).asScala
   val platformVersion: Optional[String] = params.getOptional("platform_version", classOf[String])
   val startedBy: Optional[String] = params.getOptional("started_by", classOf[String])
+  val tags: Optional[Config] = params.getOptionalNested("tags")
   // NOTE: Generated by ecs_task.register operator
   // val taskDefinition: String = params.get("task_definition", classOf[String])
 
@@ -133,6 +139,7 @@ case class EcsTaskCommandRunner(
       subTask.set("placement_strategy", placementStrategy.asJava)
       subTask.setOptional("platform_version", platformVersion)
       subTask.setOptional("started_by", startedBy)
+      subTask.setOptional("tags", tags)
       subTask.set("task_definition", "${last_ecs_task_register.task_definition_arn}")
     }
   }
@@ -175,9 +182,12 @@ case class EcsTaskCommandRunner(
     c.setOptional("cpu", cpu)
     c.setOptional("execution_role_arn", executionRoleArn)
     c.set("family", normalizedFamily)
+    c.setOptional("ipc_mode", ipcMode)
     c.setOptional("memory", memory)
     c.setOptional("network_mode", networkMode)
+    c.setOptional("pid_mode", pidMode)
     c.set("requires_compatibilities", requiresCompatibilities.asJava)
+    c.setOptional("tags", tags)
     c.setOptional("task_role_arn", taskRoleArn)
     c.set("volumes", volumes.asJava)
 
@@ -212,6 +222,7 @@ case class EcsTaskCommandRunner(
     c.setOptional("pseudo_terminal", pseudoTerminal)
     c.setOptional("readonly_root_filesystem", readonlyRootFilesystem)
     c.setOptional("repository_credentials", repositoryCredentials)
+    c.set("secrets", secrets.asJava)
     c.set("system_controls", systemControls.asJava)
     c.set("ulimits", ulimits.asJava)
     c.setOptional("user", user)

From a6289fd059dd365e725b85c904ed2cf4439c9823 Mon Sep 17 00:00:00 2001
From: Civitaspo <civitaspo@gmail.com>
Date: Sun, 18 Nov 2018 05:38:29 +0900
Subject: [PATCH 14/17] Write docs for the new options

---
 README.md           | 4 ++++
 example/example.dig | 5 +++++
 2 files changed, 9 insertions(+)

diff --git a/README.md b/README.md
index 5d038d3..b15cb6b 100644
--- a/README.md
+++ b/README.md
@@ -266,6 +266,8 @@ In addition, the below configurations exist.
 - **pseudo_terminal**: When this parameter is `true`, a TTY is allocated. (boolean, optional)
 - **readonly_root_filesystem**: When this parameter is `true`, the container is given read-only access to its root file system. (boolean, optional)
 - **repository_credentials**: The private repository authentication credentials to use. The configuration map is the same as the snake-cased [API_RepositoryCredentials](https://docs.aws.amazon.com/AmazonECS/latest/APIReference/API_RepositoryCredentials.html). (map, optional)
+- **secrets**: The secrets to pass to the container. (array of map, optional)
+    - The configuration map is the same as the snake-cased [API_Secret](https://docs.aws.amazon.com/AmazonECS/latest/APIReference/API_Secret.html).
 - **system_controls**: A list of namespaced kernel parameters to set in the container. For more information, see [ECS-Type-ContainerDefinition-systemControls](https://docs.aws.amazon.com/AmazonECS/latest/APIReference/API_ContainerDefinition.html#ECS-Type-ContainerDefinition-systemControls). (array of map, optional)
     - The configuration map is the same as the snake-cased [API_SystemControl](https://docs.aws.amazon.com/AmazonECS/latest/APIReference/API_SystemControl.html).
 - **ulimits**: A list of ulimits to set in the container. (array of map, optional)
@@ -286,6 +288,8 @@ In addition, the below configurations exist.
     - The configuration map is the same as the snake-cased [API_PlacementStrategy](https://docs.aws.amazon.com/AmazonECS/latest/APIReference/API_PlacementStrategy.html).
 - **platform_version**: The platform version on which to run your task. If one is not specified, the latest version is used by default. (string, optional)
 - **started_by**: An optional tag specified when a task is started. (string, optional)
+- **tags**: The metadata that you apply to the task to help you categorize and organize them. Each tag consists of a key and an optional value, both of which you define. Tag keys can have a maximum character length of 128 characters, and tag values can have a maximum length of 256 characters. (string to string map, optional)
+    - For using this option, require [migrating Amazon ECS deployment to the new ARN and resource ID format](Migrating your Amazon ECS deployment to the new ARN and resource ID format).
 - **workspace_s3_uri_prefix**: S3 uri prefix for using as workspace. (string, required)
     - Currently, input params, output params, stdout, stderr, and internal scripts are put on S3, and then they are not removed. So it's insecure unless strict access control to S3.
     - This option is **deprecated**. Please use **tmp_storage** option instead.
diff --git a/example/example.dig b/example/example.dig
index 1b5803d..7b35310 100644
--- a/example/example.dig
+++ b/example/example.dig
@@ -14,6 +14,11 @@ _export:
     cluster: ${cluster}
     network_mode: host
     memory: 1 GB
+    # NOTE: For using this option, require the ECS Cluster migration. See ttps://aws.amazon.com/jp/blogs/compute/migrating-your-amazon-ecs-deployment-to-the-new-arn-and-resource-id-format-2/
+    # tags:
+    #   environment: development
+    #   created_by: digdag-operator-ecs_task
+    #   digdag.session_uuid: ${session_uuid}
     task_role_arn: ${task_role_arn}
 
 +ecs_task.sh:

From 97c61da65df3025eb3ceb2c0934a4a49b891d150 Mon Sep 17 00:00:00 2001
From: Civitaspo <civitaspo@gmail.com>
Date: Mon, 19 Nov 2018 08:40:15 +0900
Subject: [PATCH 15/17] Remove an unused file

---
 example/template.txt | 1 -
 1 file changed, 1 deletion(-)
 delete mode 100644 example/template.txt

diff --git a/example/template.txt b/example/template.txt
deleted file mode 100644
index d794f34..0000000
--- a/example/template.txt
+++ /dev/null
@@ -1 +0,0 @@
-Worked? ${message}

From b636096bf42b8b6eaeea17ac7b8496b0b5c02144 Mon Sep 17 00:00:00 2001
From: Civitaspo <civitaspo@gmail.com>
Date: Mon, 19 Nov 2018 08:40:23 +0900
Subject: [PATCH 16/17] Ship v0.0.7

---
 CHANGELOG.md                                              | 8 ++++++++
 README.md                                                 | 2 +-
 build.gradle                                              | 2 +-
 example/example.dig                                       | 2 +-
 .../pro/civitaspo/digdag/plugin/ecs_task/package.scala    | 2 +-
 5 files changed, 12 insertions(+), 4 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index b094b4f..71721d4 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,3 +1,11 @@
+0.0.7 (2018-11-19)
+==================
+
+* [Enhancement] Add examples for scripting operators and update `ecs_task.run>` example.
+* [Enhancement] Always normalize ECS Task family name.
+* [Enhancement] Update aws-java-sdk 1.11.433 -> 1.11.451
+* [Enhancement] Add new options (`secrets`, `tags`) that follow ECS new release. `ipc_mode` and `pid_mode` are not supported yet because aws-java-sdk does not supports them.
+
 0.0.6 (2018-11-13)
 ==================
 
diff --git a/README.md b/README.md
index b15cb6b..41ca36a 100644
--- a/README.md
+++ b/README.md
@@ -15,7 +15,7 @@ _export:
     repositories:
       - https://jitpack.io
     dependencies:
-      - pro.civitaspo:digdag-operator-ecs_task:0.0.6
+      - pro.civitaspo:digdag-operator-ecs_task:0.0.7
   ecs_task:
     auth_method: profile
     tmp_storage:
diff --git a/build.gradle b/build.gradle
index 075cb51..15dd253 100644
--- a/build.gradle
+++ b/build.gradle
@@ -6,7 +6,7 @@ plugins {
 }
 
 group = 'pro.civitaspo'
-version = '0.0.6'
+version = '0.0.7'
 
 def digdagVersion = '0.9.31'
 def scalaSemanticVersion = "2.12.6"
diff --git a/example/example.dig b/example/example.dig
index 7b35310..91b8e85 100644
--- a/example/example.dig
+++ b/example/example.dig
@@ -4,7 +4,7 @@ _export:
       - file://${repos}
       # - https://jitpack.io
     dependencies:
-      - pro.civitaspo:digdag-operator-ecs_task:0.0.6
+      - pro.civitaspo:digdag-operator-ecs_task:0.0.7
   ecs_task:
     auth_method: profile
     tmp_storage:
diff --git a/src/main/scala/pro/civitaspo/digdag/plugin/ecs_task/package.scala b/src/main/scala/pro/civitaspo/digdag/plugin/ecs_task/package.scala
index 12b12eb..f52b9bd 100644
--- a/src/main/scala/pro/civitaspo/digdag/plugin/ecs_task/package.scala
+++ b/src/main/scala/pro/civitaspo/digdag/plugin/ecs_task/package.scala
@@ -2,6 +2,6 @@ package pro.civitaspo.digdag.plugin
 
 package object ecs_task {
 
-  val VERSION: String = "0.0.6"
+  val VERSION: String = "0.0.7"
 
 }

From 74fd1857906a5575445250a9908f2cd26830d70f Mon Sep 17 00:00:00 2001
From: Civitaspo <civitaspo@gmail.com>
Date: Mon, 19 Nov 2018 08:42:17 +0900
Subject: [PATCH 17/17] ./gradlew spotlessapply

---
 .../ecs_task/command/AbstractEcsTaskCommandOperator.scala     | 2 +-
 .../digdag/plugin/ecs_task/command/EcsTaskCommandRunner.scala | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/main/scala/pro/civitaspo/digdag/plugin/ecs_task/command/AbstractEcsTaskCommandOperator.scala b/src/main/scala/pro/civitaspo/digdag/plugin/ecs_task/command/AbstractEcsTaskCommandOperator.scala
index 1235882..cd31636 100644
--- a/src/main/scala/pro/civitaspo/digdag/plugin/ecs_task/command/AbstractEcsTaskCommandOperator.scala
+++ b/src/main/scala/pro/civitaspo/digdag/plugin/ecs_task/command/AbstractEcsTaskCommandOperator.scala
@@ -56,7 +56,7 @@ abstract class AbstractEcsTaskCommandOperator(operatorName: String, context: Ope
   protected def collectEnvironments(): Map[String, String] = {
     val vars: PrivilegedVariables = context.getPrivilegedVariables
     vars.getKeys.asScala.foldLeft(Map.empty[String, String]) { (env, key) =>
-    if (isValidEnvKey(key)) {
+      if (isValidEnvKey(key)) {
         env ++ Map(key -> vars.get(key))
       }
       else {
diff --git a/src/main/scala/pro/civitaspo/digdag/plugin/ecs_task/command/EcsTaskCommandRunner.scala b/src/main/scala/pro/civitaspo/digdag/plugin/ecs_task/command/EcsTaskCommandRunner.scala
index 5874cb9..bbc478a 100644
--- a/src/main/scala/pro/civitaspo/digdag/plugin/ecs_task/command/EcsTaskCommandRunner.scala
+++ b/src/main/scala/pro/civitaspo/digdag/plugin/ecs_task/command/EcsTaskCommandRunner.scala
@@ -74,7 +74,7 @@ case class EcsTaskCommandRunner(
   // NOTE: If you set it by container level, use the `overrides` option.
   // val memoryReservation: Optional[Int] = params.getOptional("memory_reservation", classOf[Int])
   val mountPoints: Seq[Config] = params.parseListOrGetEmpty("mount_points", classOf[Config]).asScala
-  val containerName: Optional[String]= params.getOptional("container_name", classOf[String])
+  val containerName: Optional[String] = params.getOptional("container_name", classOf[String])
   val portMappings: Seq[Config] = params.parseListOrGetEmpty("port_mappings", classOf[Config]).asScala
   val privileged: Optional[Boolean] = params.getOptional("privileged", classOf[Boolean])
   val pseudoTerminal: Optional[Boolean] = params.getOptional("pseudo_terminal", classOf[Boolean])
@@ -235,7 +235,7 @@ case class EcsTaskCommandRunner(
   protected def normalizeFamily(family: String): String = {
     // ref. https://docs.aws.amazon.com/AmazonECS/latest/APIReference/API_RegisterTaskDefinition.html#ECS-RegisterTaskDefinition-request-family
     val validLetterRegex: Regex = "[a-zA-Z0-9_-]".r
-    val after: String = family.map{ case l @ validLetterRegex() => l; case _ => "_" }.mkString
+    val after: String = family.map { case l @ validLetterRegex() => l; case _ => "_" }.mkString
     if (!family.contentEquals(after)) logger.warn(s"Normalized family: $family -> $after")
     after
   }