Skip to content

Commit

Permalink
[athena.ctas] Skip removing objects if removable targets do not exist
Browse files Browse the repository at this point in the history
  • Loading branch information
civitaspo committed Dec 7, 2018
1 parent d3da8cc commit 40ea771
Showing 1 changed file with 1 addition and 0 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -111,6 +111,7 @@ class AthenaCtasOperator(operatorName: String, context: OperatorContext, systemC
protected def rmObjects(location: String): Unit = {
val uri: AmazonS3URI = AmazonS3URI(location)
val keys: Seq[String] = withS3(_.listObjectsV2(uri.getBucket, uri.getKey)).getObjectSummaries.asScala.map(_.getKey)
if (keys.isEmpty) return
val r: DeleteObjectsResult = withS3(_.deleteObjects(new DeleteObjectsRequest(uri.getBucket).withKeys(keys: _*)))
r.getDeletedObjects.asScala.foreach(o => logger.info(s"Deleted: s3://${uri.getBucket}/${o.getKey}"))
}
Expand Down

0 comments on commit 40ea771

Please sign in to comment.