diff --git a/pom.xml b/pom.xml index ea6ddda5..99eb624b 100644 --- a/pom.xml +++ b/pom.xml @@ -64,6 +64,10 @@ io.jenkins.plugins commons-lang3-api + + io.jenkins.plugins + caffeine-api + org.jenkins-ci.plugins structs diff --git a/src/main/java/jenkins/plugins/slack/HttpClient.java b/src/main/java/jenkins/plugins/slack/HttpClient.java index 2dece14a..74787ce2 100644 --- a/src/main/java/jenkins/plugins/slack/HttpClient.java +++ b/src/main/java/jenkins/plugins/slack/HttpClient.java @@ -1,7 +1,7 @@ package jenkins.plugins.slack; import hudson.ProxyConfiguration; -import jenkins.plugins.slack.NoProxyHostCheckerRoutePlanner; +import hudson.util.Secret; import org.apache.http.HttpHost; import org.apache.http.auth.AuthScope; import org.apache.http.auth.Credentials; @@ -20,7 +20,7 @@ @Restricted(NoExternalUse.class) public class HttpClient { - public static CloseableHttpClient getCloseableHttpClient(ProxyConfiguration proxy) { + public static HttpClientBuilder getCloseableHttpClientBuilder(ProxyConfiguration proxy) { int timeoutInSeconds = 60; RequestConfig config = RequestConfig.custom() @@ -29,9 +29,9 @@ public static CloseableHttpClient getCloseableHttpClient(ProxyConfiguration prox .setSocketTimeout(timeoutInSeconds * 1000).build(); final HttpClientBuilder clientBuilder = HttpClients - .custom() - .useSystemProperties() - .setDefaultRequestConfig(config); + .custom() + .useSystemProperties() + .setDefaultRequestConfig(config); final CredentialsProvider credentialsProvider = new BasicCredentialsProvider(); clientBuilder.setDefaultCredentialsProvider(credentialsProvider); @@ -41,14 +41,20 @@ public static CloseableHttpClient getCloseableHttpClient(ProxyConfiguration prox clientBuilder.setRoutePlanner(routePlanner); String username = proxy.getUserName(); - String password = proxy.getPassword(); + Secret secretPassword = proxy.getSecretPassword(); + String password = Secret.toString(secretPassword); // Consider it to be passed if username specified. Sufficient? - if (username != null && !"".equals(username.trim())) { + if (username != null && !username.trim().isEmpty()) { credentialsProvider.setCredentials(new AuthScope(proxyHost), createCredentials(username, password)); } } - return clientBuilder.build(); + return clientBuilder; + + } + + public static CloseableHttpClient getCloseableHttpClient(ProxyConfiguration proxy) { + return getCloseableHttpClientBuilder(proxy).build(); } private static Credentials createCredentials(String userName, String password) { diff --git a/src/main/java/jenkins/plugins/slack/SlackNotifier.java b/src/main/java/jenkins/plugins/slack/SlackNotifier.java index 84776ebe..c0265046 100755 --- a/src/main/java/jenkins/plugins/slack/SlackNotifier.java +++ b/src/main/java/jenkins/plugins/slack/SlackNotifier.java @@ -27,6 +27,7 @@ import java.util.logging.Logger; import java.util.stream.Stream; import jenkins.model.Jenkins; +import jenkins.plugins.slack.cache.SlackChannelIdCache; import jenkins.plugins.slack.config.GlobalCredentialMigrator; import jenkins.plugins.slack.logging.BuildAwareLogger; import jenkins.plugins.slack.logging.BuildKey; @@ -870,6 +871,16 @@ public String getDisplayName() { return PLUGIN_DISPLAY_NAME; } + @POST + public FormValidation doClearCache() { + Jenkins.get().checkPermission(Jenkins.ADMINISTER); + + logger.info("Clearing channel ID cache"); + SlackChannelIdCache.clearCache(); + + return FormValidation.ok("Cache cleared"); + } + @POST public FormValidation doTestConnectionGlobal( @QueryParameter("baseUrl") final String baseUrl, diff --git a/src/main/java/jenkins/plugins/slack/StandardSlackService.java b/src/main/java/jenkins/plugins/slack/StandardSlackService.java index f5229ad8..232197b8 100755 --- a/src/main/java/jenkins/plugins/slack/StandardSlackService.java +++ b/src/main/java/jenkins/plugins/slack/StandardSlackService.java @@ -1,6 +1,7 @@ package jenkins.plugins.slack; import com.google.common.annotations.VisibleForTesting; +import hudson.AbortException; import hudson.FilePath; import hudson.ProxyConfiguration; import hudson.Util; @@ -20,6 +21,7 @@ import java.util.regex.Pattern; import java.util.stream.Collectors; import jenkins.model.Jenkins; +import jenkins.plugins.slack.cache.SlackChannelIdCache; import jenkins.plugins.slack.pipeline.SlackFileRequest; import jenkins.plugins.slack.pipeline.SlackUploadFileRunner; import jenkins.plugins.slack.user.SlackUserIdResolver; @@ -256,8 +258,17 @@ public boolean upload(FilePath workspace, String artifactIncludes, TaskListener boolean result = true; if(workspace!=null) { for(String roomId : roomIds) { + String channelId; + try { + channelId = SlackChannelIdCache.getChannelId(populatedToken, roomId); + } catch (ExecutionException | InterruptedException e) { + throw new RuntimeException(e); + } catch (AbortException e) { + return false; + } + SlackFileRequest slackFileRequest = new SlackFileRequest( - workspace, populatedToken, roomId, null, artifactIncludes); + workspace, populatedToken, channelId, null, artifactIncludes, null); try { workspace.getChannel().callAsync(new SlackUploadFileRunner(log, Jenkins.get().proxy, slackFileRequest)).get(); } catch (IllegalStateException | InterruptedException e) { diff --git a/src/main/java/jenkins/plugins/slack/cache/SlackChannelIdCache.java b/src/main/java/jenkins/plugins/slack/cache/SlackChannelIdCache.java new file mode 100644 index 00000000..a1606a61 --- /dev/null +++ b/src/main/java/jenkins/plugins/slack/cache/SlackChannelIdCache.java @@ -0,0 +1,159 @@ +package jenkins.plugins.slack.cache; + +import com.github.benmanes.caffeine.cache.Caffeine; +import com.github.benmanes.caffeine.cache.LoadingCache; +import hudson.AbortException; +import java.io.IOException; +import java.time.Duration; +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CompletionException; +import java.util.concurrent.ExecutionException; +import java.util.logging.Logger; +import jenkins.model.Jenkins; +import jenkins.plugins.slack.HttpClient; +import org.apache.http.Header; +import org.apache.http.HttpEntity; +import org.apache.http.HttpResponse; +import org.apache.http.HttpStatus; +import org.apache.http.client.ResponseHandler; +import org.apache.http.client.ServiceUnavailableRetryStrategy; +import org.apache.http.client.methods.RequestBuilder; +import org.apache.http.impl.client.CloseableHttpClient; +import org.apache.http.impl.client.HttpClientBuilder; +import org.apache.http.protocol.HttpContext; +import org.apache.http.util.EntityUtils; +import org.json.JSONArray; +import org.json.JSONObject; + +public class SlackChannelIdCache { + + private static final String UPLOAD_FAILED_TEMPLATE = "Failed to retrieve channel names. Response: "; + private static final Logger logger = Logger.getLogger(SlackChannelIdCache.class.getName()); + + // cache that includes all channel names and IDs for each workspace used + private static final LoadingCache> CHANNEL_METADATA_CACHE = Caffeine.newBuilder() + .maximumSize(100) + .refreshAfterWrite(Duration.ofHours(24)) + .build(SlackChannelIdCache::populateCache); + private static final int MAX_RETRIES = 10; + + private static Map populateCache(String token) { + HttpClientBuilder closeableHttpClientBuilder = HttpClient.getCloseableHttpClientBuilder(Jenkins.get().getProxy()) + .setRetryHandler((exception, executionCount, context) -> executionCount <= MAX_RETRIES) + .setServiceUnavailableRetryStrategy(new ServiceUnavailableRetryStrategy() { + + long retryInterval; + + @Override + public boolean retryRequest(HttpResponse response, int executionCount, HttpContext context) { + boolean shouldRetry = executionCount <= MAX_RETRIES && + response.getStatusLine().getStatusCode() == HttpStatus.SC_TOO_MANY_REQUESTS; + if (shouldRetry) { + Header firstHeader = response.getFirstHeader("Retry-After"); + if (firstHeader != null) { + retryInterval = Long.parseLong(firstHeader.getValue()) * 1000L; + logger.info(String.format("Rate limited by Slack, retrying in %dms", retryInterval)); + } + } + return shouldRetry; + } + + @Override + public long getRetryInterval() { + return retryInterval; + } + }); + try (CloseableHttpClient client = closeableHttpClientBuilder.build()) { + return convertChannelNameToId(client, token, new HashMap<>(), null); + } catch (IOException e) { + throw new RuntimeException(e); + } + } + + public static String getChannelId(String botUserToken, String channelName) throws ExecutionException, InterruptedException, AbortException { + Map channelNameToIdMap = CHANNEL_METADATA_CACHE.get(botUserToken); + String channelId = channelNameToIdMap.get(channelName); + + // most likely is that a new channel has been created since the last cache refresh + // or a typo in the channel name, a bit risky in larger workspaces but shouldn't happen too often + if (channelId == null) { + try { + CompletableFuture> newResult = CHANNEL_METADATA_CACHE.refresh(botUserToken); + channelNameToIdMap = newResult.get(); + } catch (CompletionException e) { + throw new AbortException("Failed uploading file to slack, channel not found: " + channelName + ", error: " + e.getMessage()); + } + + channelId = channelNameToIdMap.get(channelName); + } + + return channelId; + } + + private static Map convertChannelNameToId(CloseableHttpClient client, String token, Map channels, String cursor) throws IOException { + RequestBuilder requestBuilder = RequestBuilder.get("https://slack.com/api/conversations.list") + .addHeader("Authorization", "Bearer " + token) + .addParameter("exclude_archived", "true") + .addParameter("types", "public_channel,private_channel"); + + if (cursor != null) { + requestBuilder.addParameter("cursor", cursor); + } + ResponseHandler standardResponseHandler = getStandardResponseHandler(); + JSONObject result = client.execute(requestBuilder.build(), standardResponseHandler); + + if (!result.getBoolean("ok")) { + logger.warning("Couldn't convert channel name to ID in Slack: " + result); + return channels; + } + + JSONArray channelsArray = result.getJSONArray("channels"); + for (int i = 0; i < channelsArray.length(); i++) { + JSONObject channel = channelsArray.getJSONObject(i); + + String channelName = channel.getString("name"); + String channelId = channel.getString("id"); + + channels.put(channelName, channelId); + } + + cursor = result.getJSONObject("response_metadata").getString("next_cursor"); + if (cursor != null && !cursor.isEmpty()) { + return convertChannelNameToId(client, token, channels, cursor); + } + + return channels; + } + + private static ResponseHandler getStandardResponseHandler() { + return response -> { + int status = response.getStatusLine().getStatusCode(); + if (status >= 200 && status < 300) { + HttpEntity entity = response.getEntity(); + return entity != null ? new JSONObject(EntityUtils.toString(entity)) : null; + } else { + String errorMessage = UPLOAD_FAILED_TEMPLATE + status + " " + EntityUtils.toString(response.getEntity()); + throw new HttpStatusCodeException(response.getStatusLine().getStatusCode(), errorMessage); + } + }; + } + + public static class HttpStatusCodeException extends RuntimeException { + private final int statusCode; + + public HttpStatusCodeException(int statusCode, String message) { + super(message); + this.statusCode = statusCode; + } + + public int getStatusCode() { + return statusCode; + } + } + + public static void clearCache() { + CHANNEL_METADATA_CACHE.invalidateAll(); + } +} diff --git a/src/main/java/jenkins/plugins/slack/pipeline/SlackFileRequest.java b/src/main/java/jenkins/plugins/slack/pipeline/SlackFileRequest.java index f092851f..dbe12268 100644 --- a/src/main/java/jenkins/plugins/slack/pipeline/SlackFileRequest.java +++ b/src/main/java/jenkins/plugins/slack/pipeline/SlackFileRequest.java @@ -8,25 +8,27 @@ public class SlackFileRequest { private final String fileToUploadPath; private final String token; - private final String channels; + private final String channelId; + private final String threadTs; private final String initialComment; private final FilePath filePath; - public SlackFileRequest(FilePath filePath, String token, String channels, String initialComment, String fileToUploadPath) { + public SlackFileRequest(FilePath filePath, String token, String channelId, String initialComment, String fileToUploadPath, String threadTs) { this.token = token; - this.channels = channels; + this.channelId = channelId; this.initialComment = initialComment; this.filePath = filePath; this.fileToUploadPath = fileToUploadPath; + this.threadTs = threadTs; } public String getToken() { return token; } - public String getChannels() { - return channels; + public String getChannelId() { + return channelId; } public String getInitialComment() { @@ -40,4 +42,8 @@ public FilePath getFilePath() { public String getFileToUploadPath() { return fileToUploadPath; } + + public String getThreadTs() { + return threadTs; + } } diff --git a/src/main/java/jenkins/plugins/slack/pipeline/SlackUploadFileRunner.java b/src/main/java/jenkins/plugins/slack/pipeline/SlackUploadFileRunner.java index 3fd363ca..8731edeb 100644 --- a/src/main/java/jenkins/plugins/slack/pipeline/SlackUploadFileRunner.java +++ b/src/main/java/jenkins/plugins/slack/pipeline/SlackUploadFileRunner.java @@ -39,19 +39,21 @@ public class SlackUploadFileRunner extends MasterToSlaveCallable files) { - String threadTs = null; - String theChannels = channels; - - //thread_ts is passed once with roomId: Ex: roomId:threadTs - String[] splitThread = channels.split(":", 2); - if (splitThread.length == 2) { - theChannels = splitThread[0]; - threadTs = splitThread[1]; - } - List fileIds = new ArrayList<>(); try (CloseableHttpClient client = HttpClient.getCloseableHttpClient(proxy)) { for (File file : files) { @@ -113,8 +105,7 @@ private boolean doIt(List files) { String fileId = getUploadUrlResult.getString("file_id"); fileIds.add(fileId); } - String channelId = convertChannelNameToId(theChannels, client); - if (!completeUploadExternal(channelId, threadTs, fileIds, client)) { + if (!completeUploadExternal(channelId, threadTs , fileIds, client)) { listener.getLogger().println("Failed to complete uploading file to Slack"); return false; } @@ -177,56 +168,6 @@ private ResponseHandler getStandardResponseHandler() { }; } - private String convertChannelNameToId(String channels, CloseableHttpClient client) throws IOException { - return convertChannelNameToId(channels, client, null); - } - - private String convertChannelNameToId(String channelName, CloseableHttpClient client, String cursor) throws IOException { - RequestBuilder requestBuilder = RequestBuilder.get("https://slack.com/api/conversations.list") - .addHeader("Authorization", "Bearer " + token) - .addParameter("exclude_archived", "true") - .addParameter("types", "public_channel,private_channel"); - - if (cursor != null) { - requestBuilder.addParameter("cursor", cursor); - } - ResponseHandler standardResponseHandler = getStandardResponseHandler(); - JSONObject result = client.execute(requestBuilder.build(), standardResponseHandler); - - if (result == null) { - // logging should have been done in the result handler where the response is available - return null; - } - if (!result.getBoolean("ok")) { - listener.getLogger().println("Couldn't convert channel name to ID in Slack: " + result); - return null; - } - - JSONArray channelsArray = result.getJSONArray("channels"); - for (int i = 0; i < channelsArray.length(); i++) { - JSONObject channel = channelsArray.getJSONObject(i); - if (channel.getString("name").equals(cleanChannelName(channelName))) { - return channel.getString("id"); - } - } - - cursor = result.getJSONObject("response_metadata").getString("next_cursor"); - if (cursor != null && !cursor.isEmpty()) { - return convertChannelNameToId(channelName, client, cursor); - } - - listener.getLogger().println("Couldn't find channel id for channel name " + channelName); - - return null; - } - - private static String cleanChannelName(String channelName) { - if (channelName.startsWith("#")) { - return channelName.substring(1); - } - return channelName; - } - private boolean uploadFile(String uploadUrl, MultipartEntityBuilder multipartEntityBuilder, CloseableHttpClient client) throws IOException { HttpUriRequest request = RequestBuilder .post(uploadUrl) diff --git a/src/main/java/jenkins/plugins/slack/pipeline/SlackUploadFileStep.java b/src/main/java/jenkins/plugins/slack/pipeline/SlackUploadFileStep.java index d2c1d163..7a267e7f 100644 --- a/src/main/java/jenkins/plugins/slack/pipeline/SlackUploadFileStep.java +++ b/src/main/java/jenkins/plugins/slack/pipeline/SlackUploadFileStep.java @@ -2,6 +2,7 @@ import com.google.common.collect.ImmutableSet; import edu.umd.cs.findbugs.annotations.NonNull; +import hudson.AbortException; import hudson.Extension; import hudson.FilePath; import hudson.Util; @@ -12,11 +13,13 @@ import hudson.util.ListBoxModel; import java.io.IOException; import java.util.Set; +import java.util.concurrent.CompletionException; import java.util.concurrent.ExecutionException; import jenkins.model.Jenkins; import jenkins.plugins.slack.CredentialsObtainer; import jenkins.plugins.slack.Messages; import jenkins.plugins.slack.SlackNotifier; +import jenkins.plugins.slack.cache.SlackChannelIdCache; import org.jenkinsci.plugins.workflow.steps.Step; import org.jenkinsci.plugins.workflow.steps.StepContext; import org.jenkinsci.plugins.workflow.steps.StepDescriptor; @@ -35,6 +38,7 @@ public class SlackUploadFileStep extends Step { private String channel; private String initialComment; private String filePath; + private boolean failOnError; @DataBoundConstructor public SlackUploadFileStep(String filePath) { @@ -72,6 +76,15 @@ public String getFilePath() { return filePath; } + public boolean isFailOnError() { + return failOnError; + } + + @DataBoundSetter + public void setFailOnError(boolean failOnError) { + this.failOnError = failOnError; + } + @Override public StepExecution start(StepContext context) { return new SlackUploadFileStepExecution(this, context); @@ -126,17 +139,63 @@ protected Void run() throws IOException, InterruptedException, ExecutionExceptio String populatedToken = CredentialsObtainer.getTokenToUse(tokenCredentialId, item, null); String channel = step.channel != null ? step.channel : slackDesc.getRoom(); + String channelId; + try { + channelId = SlackChannelIdCache.getChannelId(populatedToken, cleanChannelName(channel)); + if (channelId == null) { + String message = "Failed uploading file to slack, channel not found: " + channel; + if (step.failOnError) { + throw new AbortException(message); + } else { + listener.error(message); + return null; + } + } + } catch (CompletionException | SlackChannelIdCache.HttpStatusCodeException e) { + throw new AbortException("Failed uploading file to slack, channel not found: " + channel + ", error: " + e.getMessage()); + } + + String threadTs = getThreadTs(channel); + SlackFileRequest slackFileRequest = new SlackFileRequest( - filePath, populatedToken, channel, step.initialComment, step.filePath + filePath, populatedToken, channelId, step.initialComment, step.filePath, threadTs ); assert filePath != null; VirtualChannel virtualChannel = filePath.getChannel(); assert virtualChannel != null; - virtualChannel.callAsync(new SlackUploadFileRunner(listener, Jenkins.get().proxy, slackFileRequest)).get(); + Boolean result = virtualChannel.callAsync(new SlackUploadFileRunner(listener, Jenkins.get().proxy, slackFileRequest)).get(); + if (!result) { + String errorMessage = "Failed uploading file to slack"; + if (step.failOnError) { + throw new AbortException(errorMessage); + } else { + listener.error(errorMessage); + } + } return null; } } + + private static String getThreadTs(String channelName) { + String[] splitForThread = channelName.split(":", 2); + if (splitForThread.length == 2) { + return splitForThread[1]; + } + return null; + } + + private static String cleanChannelName(String channelName) { + String[] splitForThread = channelName.split(":", 2); + String channel = channelName; + if (splitForThread.length == 2) { + channel = splitForThread[0]; + } + if (channel.startsWith("#")) { + return channel.substring(1); + } + return channelName; + } } diff --git a/src/main/resources/jenkins/plugins/slack/SlackNotifier/global.jelly b/src/main/resources/jenkins/plugins/slack/SlackNotifier/global.jelly index f19c630f..ef5bb72a 100755 --- a/src/main/resources/jenkins/plugins/slack/SlackNotifier/global.jelly +++ b/src/main/resources/jenkins/plugins/slack/SlackNotifier/global.jelly @@ -1,5 +1,5 @@ - + @@ -31,9 +31,14 @@ descriptors="${descriptor.getSlackUserIdResolverDescriptors()}"/> - + + + + diff --git a/src/main/resources/jenkins/plugins/slack/pipeline/SlackUploadFileStep/config.jelly b/src/main/resources/jenkins/plugins/slack/pipeline/SlackUploadFileStep/config.jelly index c1de5c70..e50de06e 100644 --- a/src/main/resources/jenkins/plugins/slack/pipeline/SlackUploadFileStep/config.jelly +++ b/src/main/resources/jenkins/plugins/slack/pipeline/SlackUploadFileStep/config.jelly @@ -14,5 +14,9 @@ + + + +