From d3da8ccd3c99510786072fab5b9be9b7ddbfa813 Mon Sep 17 00:00:00 2001 From: Civitaspo Date: Sat, 20 Oct 2018 11:07:34 +0900 Subject: [PATCH 1/3] Update README: Write the explanation about **data_only** of **table_mode** --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index fa160c8..063b54d 100644 --- a/README.md +++ b/README.md @@ -145,7 +145,7 @@ Define the below options on properties (which is indicated by `-c`, `--config`). - **table_mode**: Specify the expected behavior of CTAS results. Available values are `"default"`, `"empty"`, `"data_only"`. See the below explanation of the behaviour. (string, default: `"default"`) - `"default"`: Do not do any care. This option require the least IAM privileges for digdag, but the behaviour depends on Athena. - `"empty_table"`: Create a new empty table with the same schema as the select query results. - - `"data_only"`: + - `"data_only"`: Create a new table with data by CTAS, but drop this after CTAS execution. The table created by CTAS is an external table, so the data is left even if the table is dropped. - **save_mode**: Specify the expected behavior of CTAS. Available values are `"none"`, `"error_if_exists"`, `"ignore"`, `"overwrite"`. See the below explanation of the behaviour. (string, default: `"overwrite"`) - `"none"`: Do not do any care. This option require the least IAM privileges for digdag, but the behaviour depends on Athena. - `"error_if_exists"`: Raise error if the distination table or location exists. From 40ea771f792b52bc3aed2a9eb364ebd742b4de15 Mon Sep 17 00:00:00 2001 From: Civitaspo Date: Fri, 7 Dec 2018 14:07:28 +0900 Subject: [PATCH 2/3] [athena.ctas] Skip removing objects if removable targets do not exist --- .../digdag/plugin/athena/operator/AthenaCtasOperator.scala | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main/scala/pro/civitaspo/digdag/plugin/athena/operator/AthenaCtasOperator.scala b/src/main/scala/pro/civitaspo/digdag/plugin/athena/operator/AthenaCtasOperator.scala index f6679db..4dda3fc 100644 --- a/src/main/scala/pro/civitaspo/digdag/plugin/athena/operator/AthenaCtasOperator.scala +++ b/src/main/scala/pro/civitaspo/digdag/plugin/athena/operator/AthenaCtasOperator.scala @@ -111,6 +111,7 @@ class AthenaCtasOperator(operatorName: String, context: OperatorContext, systemC protected def rmObjects(location: String): Unit = { val uri: AmazonS3URI = AmazonS3URI(location) val keys: Seq[String] = withS3(_.listObjectsV2(uri.getBucket, uri.getKey)).getObjectSummaries.asScala.map(_.getKey) + if (keys.isEmpty) return val r: DeleteObjectsResult = withS3(_.deleteObjects(new DeleteObjectsRequest(uri.getBucket).withKeys(keys: _*))) r.getDeletedObjects.asScala.foreach(o => logger.info(s"Deleted: s3://${uri.getBucket}/${o.getKey}")) } From 0e00ce675fa9205c22e5944e2a3ce9b31457a63c Mon Sep 17 00:00:00 2001 From: Civitaspo Date: Fri, 7 Dec 2018 14:11:19 +0900 Subject: [PATCH 3/3] Ship v0.1.1 --- CHANGELOG.md | 5 +++++ README.md | 2 +- build.gradle | 2 +- example/example.dig | 2 +- 4 files changed, 8 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f96d9d8..c1efc20 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,8 @@ +0.1.1 (2018-12-07) +================== + +* [Fix] `athena.ctas>` Skip removing objects if removable targets do not exist. + 0.1.0 (2018-10-20) ================== diff --git a/README.md b/README.md index 063b54d..a02d3e0 100644 --- a/README.md +++ b/README.md @@ -15,7 +15,7 @@ _export: repositories: - https://jitpack.io dependencies: - - pro.civitaspo:digdag-operator-athena:0.1.0 + - pro.civitaspo:digdag-operator-athena:0.1.1 athena: auth_method: profile diff --git a/build.gradle b/build.gradle index af824f5..2de97df 100644 --- a/build.gradle +++ b/build.gradle @@ -6,7 +6,7 @@ plugins { } group = 'pro.civitaspo' -version = '0.1.0' +version = '0.1.1' def digdagVersion = '0.9.27' def awsSdkVersion = "1.11.372" diff --git a/example/example.dig b/example/example.dig index bce25d4..bbda170 100644 --- a/example/example.dig +++ b/example/example.dig @@ -4,7 +4,7 @@ _export: - file://${repos} # - https://jitpack.io dependencies: - - pro.civitaspo:digdag-operator-athena:0.1.0 + - pro.civitaspo:digdag-operator-athena:0.1.1 athena: auth_method: profile value: 5