Skip to content

Commit

Permalink
Merge pull request #30 from civitaspo/fix/skip-if-no-removable-target
Browse files Browse the repository at this point in the history
[athena.ctas] Skip removing objects if removable targets do not exist
  • Loading branch information
civitaspo authored Dec 7, 2018
2 parents d3da8cc + 40ea771 commit 801f52c
Showing 1 changed file with 1 addition and 0 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -111,6 +111,7 @@ class AthenaCtasOperator(operatorName: String, context: OperatorContext, systemC
protected def rmObjects(location: String): Unit = {
val uri: AmazonS3URI = AmazonS3URI(location)
val keys: Seq[String] = withS3(_.listObjectsV2(uri.getBucket, uri.getKey)).getObjectSummaries.asScala.map(_.getKey)
if (keys.isEmpty) return
val r: DeleteObjectsResult = withS3(_.deleteObjects(new DeleteObjectsRequest(uri.getBucket).withKeys(keys: _*)))
r.getDeletedObjects.asScala.foreach(o => logger.info(s"Deleted: s3://${uri.getBucket}/${o.getKey}"))
}
Expand Down

0 comments on commit 801f52c

Please sign in to comment.