Merge pull request #4 from civitaspo/develop

v0.0.1

.circleci/config.yml

@@ -0,0 +1,41 @@
+version: 2
+
+references:
+  environment: &environment
+    docker:
+      - image: circleci/openjdk:8-jdk
+    working_directory: ~/repo
+    environment:
+      # Customize the JVM maximum heap limit
+      JVM_OPTS: -Xmx3200m
+      TERM: dumb
+
+jobs:
+  spotless:
+    <<: *environment
+    steps:
+      - checkout
+      - run: ./gradlew spotlessCheck
+
+  build:
+    <<: *environment
+    steps:
+      - checkout
+      - run: ./gradlew publish
+
+  ## Require an AWS account, and take some money to launch EMR, so it doesn't work now.
+  run_example:
+    <<: *environment
+    steps:
+      - checkout
+      - run: ./example/run.sh
+
+workflows:
+  version: 2
+
+  merge-before:
+    jobs:
+      - build
+      - spotless
+
+

.scalafmt.conf

@@ -0,0 +1,9 @@
+# https://scalameta.org/scalafmt/#Configuration
+
+style = IntelliJ
+maxColumn = 160
+align = none
+newlines.penalizeSingleSelectMultiArgList = false
+newlines.alwaysBeforeElseAfterCurlyIf = true
+newlines.alwaysBeforeTopLevelStatements = true
+

CHANGELOG.md

@@ -0,0 +1,4 @@
+0.0.1 (2018-10-23)
+==================
+
+* First Release

README.md

@@ -1,70 +1,169 @@
-# digdag-plugin-example
-[![Jitpack](https://jitpack.io/v/myui/digdag-plugin-example.svg)](https://jitpack.io/#myui/digdag-plugin-example) [![Digdag](https://img.shields.io/badge/digdag-v0.9.12-brightgreen.svg)](https://github.com/treasure-data/digdag/releases/tag/v0.9.12)
+# digdag-operator-ecs_task
+[![Jitpack](https://jitpack.io/v/pro.civitaspo/digdag-operator-ecs_task.svg)](https://jitpack.io/#pro.civitaspo/digdag-operator-ecs_task) [![CircleCI](https://circleci.com/gh/civitaspo/digdag-operator-ecs_task.svg?style=shield)](https://circleci.com/gh/civitaspo/digdag-operator-ecs_task) [![Digdag](https://img.shields.io/badge/digdag-v0.9.31-brightgreen.svg)](https://github.com/treasure-data/digdag/releases/tag/v0.9.31)
 
-# 1) build
+digdag plugin for AWS ECS Task.
 
-```sh
-./gradlew publish
-```
+# Overview
 
-Artifacts are build on local repos: `./build/repo`.
+- Plugin type: operator
 
-# 2) run an example
+# Usage
 
-```sh
-digdag selfupdate
+```yaml
+_export:
+  plugin:
+    repositories:
+      - https://jitpack.io
+    dependencies:
+      - pro.civitaspo:digdag-operator-ecs_task:0.0.1
+  ecs_task:
+    auth_method: profile
+
++step0:
+  sh>: echo '{"store_params":{"civi":"taspo"}}' | aws s3 cp - ${output}
+
++step1:
+  ecs_task.run>:
+  def:
+    network_mode: Host
+    container_definitions:
+      - name: uploader
+        image: amazonlinux:2
+        command: [yum, install, '-y', awscli]
+        essential: true
+        memory: 500
+        cpu: 10
+    family: hello_world
+  cluster: ${cluster}
+  count: 1
+  result_s3_uri: ${output}
+
++step2:
+  echo>: ${civi}
 
-digdag run --project sample plugin.dig -p repos=`pwd`/build/repo
 ```
 
-You'll find the result of the task in `./sample/example.out`.
+# Configuration
 
----
+## Remarks
 
-# Writing your own plugin
+- type `DurationParam` is strings matched `\s*(?:(?<days>\d+)\s*d)?\s*(?:(?<hours>\d+)\s*h)?\s*(?:(?<minutes>\d+)\s*m)?\s*(?:(?<seconds>\d+)\s*s)?\s*`.
+  - The strings is used as `java.time.Duration`.
 
-1. You need to implement [a Plugin class](https://github.com/myui/digdag-plugin-example/blob/master/src/main/java/io/digdag/plugin/example/ExamplePlugin.java) that implements `io.digdag.spi.Plugin`.
+## Common Configuration
 
-2. Then, list it on [io.digdag.spi.Plugin](https://github.com/myui/digdag-plugin-example/blob/master/src/main/resources/META-INF/services/io.digdag.spi.Plugin). The listed plugins are loaded by Digdag.
+### System Options
 
-You can optionally create Eclipse/Idea project files as follows:
-```sh
-gradle eclipse
-gradle idea
-```
+Define the below options on properties (which is indicated by `-c`, `--config`).
+
+- **ecs_task.allow_auth_method_env**: Indicates whether users can use **auth_method** `"env"` (boolean, default: `false`)
+- **ecs_task.allow_auth_method_instance**: Indicates whether users can use **auth_method** `"instance"` (boolean, default: `false`)
+- **ecs_task.allow_auth_method_profile**: Indicates whether users can use **auth_method** `"profile"` (boolean, default: `false`)
+- **ecs_task.allow_auth_method_properties**: Indicates whether users can use **auth_method** `"properties"` (boolean, default: `false`)
+- **ecs_task.assume_role_timeout_duration**: Maximum duration which server administer allows when users assume **role_arn**. (`DurationParam`, default: `1h`)
+
+### Secrets
+
+- **ecs_task.access_key_id**: The AWS Access Key ID (optional)
+- **ecs_task.secret_access_key**: The AWS Secret Access Key (optional)
+- **ecs_task.session_token**: The AWS session token. This is used only **auth_method** is `"session"` (optional)
+- **ecs_task.role_arn**: The AWS Role to assume. (optional)
+- **ecs_task.role_session_name**: The AWS Role Session Name when assuming the role. (default: `digdag-ecs_task-${session_uuid}`)
+- **ecs_task.http_proxy.host**: proxy host (required if **use_http_proxy** is `true`)
+- **ecs_task.http_proxy.port** proxy port (optional)
+- **ecs_task.http_proxy.scheme** `"https"` or `"http"` (default: `"https"`)
+- **ecs_task.http_proxy.user** proxy user (optional)
+- **ecs_task.http_proxy.password**: http proxy password (optional)
+
+### Options
+
+- **auth_method**: name of mechanism to authenticate requests (`"basic"`, `"env"`, `"instance"`, `"profile"`, `"properties"`, `"anonymous"`, or `"session"`. default: `"basic"`)
+  - `"basic"`: uses access_key_id and secret_access_key to authenticate.
+  - `"env"`: uses AWS_ACCESS_KEY_ID (or AWS_ACCESS_KEY) and AWS_SECRET_KEY (or AWS_SECRET_ACCESS_KEY) environment variables.
+  - `"instance"`: uses EC2 instance profile.
+  - `"profile"`: uses credentials written in a file. Format of the file is as following, where `[...]` is a name of profile.
+    - **profile_file**: path to a profiles file. (string, default: given by `AWS_CREDENTIAL_PROFILES_FILE` environment varialbe, or ~/.aws/credentials).
+    - **profile_name**: name of a profile. (string, default: `"default"`)
+  - `"properties"`: uses aws.accessKeyId and aws.secretKey Java system properties.
+  - `"anonymous"`: uses anonymous access. This auth method can access only public files.
+  - `"session"`: uses temporary-generated access_key_id, secret_access_key and session_token.
+- **use_http_proxy**: Indicate whether using when accessing AWS via http proxy. (boolean, default: `false`)
+- **region**: The AWS region. (string, optional)
+- **endpoint**: The AWS Service endpoint. (string, optional)
+
+## Configuration for `ecs_task.register>` operator
+
+- **ecs_task.register>**: The configuration is the same as the snake-cased [RegisterTaskDefinition API](https://docs.aws.amazon.com/AmazonECS/latest/APIReference/API_RegisterTaskDefinition.html) (map, required)
+
+## Configuration for `ecs_task.run>` operator
+
+The configuration is the same as the snake-cased [RunTask API](https://docs.aws.amazon.com/AmazonECS/latest/APIReference/API_RunTask.html).
+
+In addition, the below configurations exist.
 
-*Note:* _It's better to change the dependencies from `provided` to `compile` in [build.gradle](https://github.com/myui/digdag-plugin-example/blob/master/build.gradle) for creating idea/eclipse project config._
+- **def**: The definition for the task. The configuration is the same as `ecs_task.register>`'s one. (map, optional)
+  - **NOTE**: **task_definition** is required on the [RunTask API Doc](https://docs.aws.amazon.com/AmazonECS/latest/APIReference/API_RunTask.html), but it is not required if the **def** is defined.
+- **result_s3_uri**: The S3 uri for the task result. (string, optional)
+  - **NOTE**: This configuration is used by `ecs_task.result>` operator, so the result content must follow the rule.
+- **timeout**: Timeout duration for the task. (`DurationParam`, default: `15m`)
 
-# Plugin Loading
+## Configuration for `ecs_task.wait>` operator
 
-Digdag loads pluigins from Maven repositories by configuring [plugin options](https://github.com/myui/digdag-plugin-example/blob/master/sample/plugin.dig).
+- **cluster**: The short name or full ARN of the cluster that hosts the tasks. (string, required)
+- **tasks**: A list of up to 100 task IDs or full ARN entries. (array of string, required)
+- **timeout**: Timeout duration for the tasks. (`DurationParam`, default: `15m`)
+- **condition**: The condition of tasks to wait. Available values are `"all"` or `"any"`. (string, default: `"all"`)
+- **status**: The status of tasks to wait. Available values are `"PENDING"`, `"RUNNING"`, or `"STOPPED"` (string, default: `"STOPPED"`)
+- **ignore_failure**: Ignore even if any tasks exit with the code except 0. (boolean, default: `false`) 
 
-You can use a local Maven repository (local FS, Amazon S3) or any public Maven repository ([Maven Central](http://search.maven.org/), [Sonatype](https://www.sonatype.com/), [Bintary](https://bintray.com/), [Jitpack](https://jitpack.io/)) for the plugin artifact repository.
+## Configuration for `ecs_task.result>` operator
 
-# Publishing your plugin using Github and Jitpack
+- **ecs_task.result>**: S3 URI that the result is stored. (string, required)
+  - **NOTE**: The result content must follow the below rule.
+    - the format is json.
+    - the keys are `"subtask_config"`, `"export_params"`, `"store_params"`.
+    - the values are string to object map.
+      - the usage follows [Digdag Python API](https://docs.digdag.io/python_api.html), [Digdag Ruby API](https://docs.digdag.io/ruby_api.html). 
 
-[Jitpack](https://jitpack.io/) is useful for publishing your github repository as a maven repository.
+# Development
+
+## Run an Example
+
+### 1) build
 
 ```sh
-git tag v0.1.3
-git push origin v0.1.3
+./gradlew publish
 ```
 
-https://jitpack.io/#myui/digdag-plugin-example/v0.1.3
+Artifacts are build on local repos: `./build/repo`.
 
-Now, you can load the artifact from a github repository in [a dig file](https://github.com/myui/digdag-plugin-example/blob/master/sample/plugin.dig) as follows:
+### 2) get your aws profile
 
+```sh
+aws configure
 ```
-_export:
-  plugin:
-    repositories:
-      # - file://${repos}
-      - https://jitpack.io
-    dependencies:
-      # - io.digdag.plugin:digdag-plugin-example:0.1.3
-      - com.github.myui:digdag-plugin-example:v0.1.3
+
+### 3) run an example
+
+```sh
+./example/run.sh
+```
+
+## (TODO) Run Tests
+
+```sh
+./gradlew test
 ```
 
-# Further reading
+# ChangeLog
+
+[CHANGELOG.md](./CHANGELOG.md)
+
+# License
+
+[Apache License 2.0](./LICENSE.txt)
+
+# Author
+
+@civitaspo
 
-- [Operators](http://docs.digdag.io/operators.html) and [their implementations](https://github.com/treasure-data/digdag/tree/master/digdag-standards/src/main/java/io/digdag/standards/operator)

build.gradle

@@ -1,14 +1,16 @@
-apply plugin: 'java'
-apply plugin: 'maven'
-apply plugin: 'maven-publish'
-
-apply plugin: 'eclipse'
-apply plugin: 'idea'
+plugins {
+    id 'scala'
+    id 'maven-publish'
+    id 'com.github.johnrengelman.shadow' version '2.0.2'
+    id "com.diffplug.gradle.spotless" version "3.13.0"
+}
 
-group = 'io.digdag.plugin'
-version = '0.1.3'
+group = 'pro.civitaspo'
+version = '0.0.1'
 
-def digdagVersion = '0.9.12'
+def digdagVersion = '0.9.31'
+def scalaSemanticVersion = "2.12.6"
+def depScalaVersion = "2.12"
 
 repositories {
     mavenCentral()
@@ -18,29 +20,30 @@ repositories {
     }
 }
 
-configurations {
-    provided
-}
-
 dependencies {
-    provided 'io.digdag:digdag-spi:' + digdagVersion
-    // provided 'io.digdag:digdag-standards:' + digdagVersion
-    provided 'io.digdag:digdag-plugin-utils:' + digdagVersion  // this should be 'compile' once digdag 0.8.2 is released to a built-in repository
+    compile group: 'io.digdag', name: 'digdag-spi', version: digdagVersion
+    compile group: 'io.digdag', name: 'digdag-plugin-utils', version: digdagVersion
+
+    // https://mvnrepository.com/artifact/org.scala-lang/scala-library
+    compile group: 'org.scala-lang', name: 'scala-library', version: scalaSemanticVersion
+
+    ['ecs', 's3', 'sts'].each { svc ->
+        // https://mvnrepository.com/artifact/com.amazonaws/
+        compile group: 'com.amazonaws', name: "aws-java-sdk-${svc}", version: '1.11.433'
+    }
 }
 
-sourceSets {
-    main {
-        compileClasspath += configurations.provided
-        test.compileClasspath += configurations.provided
-        test.runtimeClasspath += configurations.provided
+shadowJar {
+    classifier = null
+    dependencies {
+        exclude(dependency('io.digdag:.*'))
     }
 }
 
 publishing {
     publications {
-        mavenJava(MavenPublication) {
-            // artifactId 'project1-sample'        
-            from components.java
+        shadow(MavenPublication) { publication ->
+            project.shadow.component(publication)
         }
     }
     repositories {
@@ -50,20 +53,16 @@ publishing {
     }
 }
 
+spotless {
+    scala {
+        scalafmt('1.5.1').configFile('.scalafmt.conf')
+    }
+}
+
 sourceCompatibility = 1.8
 targetCompatibility = 1.8
 
-compileJava.options.encoding = 'UTF-8'
-compileTestJava.options.encoding = 'UTF-8'
-
-tasks.withType(JavaCompile) {
-    options.compilerArgs << "-Xlint:unchecked" << "-Xlint:deprecation"
-}
-
-javadoc {
-    options {
-        locale = 'en_US'
-        encoding = 'UTF-8'
-    }
-}
+compileScala.options.encoding = 'UTF-8'
+compileTestScala.options.encoding = 'UTF-8'
+compileScala.options.compilerArgs << "-Xlint:unchecked" << "-Xlint:deprecation"
 

sample/.gitignore → example/.gitignore

@@ -1,5 +1,5 @@
 /.digdag-wrapper
 .digdag
 *.pyc
-
 example.out
+

example/digdag.properties

@@ -0,0 +1,4 @@
+ecs_task.allow_auth_method_env=true
+ecs_task.allow_auth_method_instance=true
+ecs_task.allow_auth_method_profile=true
+ecs_task.allow_auth_method_properties=true