diff --git a/gate-core/src/main/java/com/netflix/spinnaker/gate/services/TaskService.java b/gate-core/src/main/java/com/netflix/spinnaker/gate/services/TaskService.java index 8b09b5bd8f..56d21a0af2 100644 --- a/gate-core/src/main/java/com/netflix/spinnaker/gate/services/TaskService.java +++ b/gate-core/src/main/java/com/netflix/spinnaker/gate/services/TaskService.java @@ -108,7 +108,7 @@ public Map createAndWaitForCompletion(Map body, int maxPolls, int intervalMs) { String taskId = ((String) createResult.get("ref")).split("/")[2]; log.info("Create succeeded; polling task for completion: " + taskId); - LinkedHashMap map = new LinkedHashMap(1); + LinkedHashMap map = new LinkedHashMap<>(1); map.put("id", taskId); Map task = map; for (int i = 0; i < maxPolls; i++) { @@ -122,7 +122,16 @@ public Map createAndWaitForCompletion(Map body, int maxPolls, int intervalMs) { .contains((String) task.get("status"))) { return task; } + log.debug( + "Task {} not completed after checking {} times. Waiting {}ms before checking again", + taskId, + i + 1, + intervalMs); } + log.error( + "Task {} still not complete after exhausting {} max polls. Not checking anymore.", + taskId, + maxPolls); return task; } diff --git a/gate-web/src/main/groovy/com/netflix/spinnaker/gate/config/GateConfig.groovy b/gate-web/src/main/groovy/com/netflix/spinnaker/gate/config/GateConfig.groovy index e3a3bf41ba..a7b3756e37 100644 --- a/gate-web/src/main/groovy/com/netflix/spinnaker/gate/config/GateConfig.groovy +++ b/gate-web/src/main/groovy/com/netflix/spinnaker/gate/config/GateConfig.groovy @@ -28,6 +28,7 @@ import com.netflix.spinnaker.fiat.shared.FiatPermissionEvaluator import com.netflix.spinnaker.fiat.shared.FiatService import com.netflix.spinnaker.fiat.shared.FiatStatus import com.netflix.spinnaker.filters.AuthenticatedRequestFilter +import com.netflix.spinnaker.gate.config.controllers.PipelineControllerConfigProperties import com.netflix.spinnaker.gate.converters.JsonHttpMessageConverter import com.netflix.spinnaker.gate.converters.YamlHttpMessageConverter import com.netflix.spinnaker.gate.filters.RequestLoggingFilter @@ -51,6 +52,7 @@ import org.springframework.beans.factory.annotation.Autowired import org.springframework.beans.factory.annotation.Value import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty +import org.springframework.boot.context.properties.EnableConfigurationProperties import org.springframework.boot.web.servlet.FilterRegistrationBean import org.springframework.context.annotation.Bean import org.springframework.context.annotation.Configuration @@ -71,6 +73,7 @@ import static retrofit.Endpoints.newFixedEndpoint @CompileStatic @Configuration @Slf4j +@EnableConfigurationProperties([PipelineControllerConfigProperties.class]) @Import([PluginsAutoConfiguration, DeckPluginConfiguration, PluginWebConfiguration]) class GateConfig { diff --git a/gate-web/src/main/groovy/com/netflix/spinnaker/gate/config/controllers/PipelineControllerConfigProperties.java b/gate-web/src/main/groovy/com/netflix/spinnaker/gate/config/controllers/PipelineControllerConfigProperties.java new file mode 100644 index 0000000000..241cb9a360 --- /dev/null +++ b/gate-web/src/main/groovy/com/netflix/spinnaker/gate/config/controllers/PipelineControllerConfigProperties.java @@ -0,0 +1,18 @@ +package com.netflix.spinnaker.gate.config.controllers; + +import lombok.Data; +import org.springframework.boot.context.properties.ConfigurationProperties; + +@Data +@ConfigurationProperties(prefix = "controller.pipeline") +public class PipelineControllerConfigProperties { + + /** Holds the configurations to be used for bulk save controller mapping */ + private BulkSaveConfigProperties bulksave = new BulkSaveConfigProperties(); + + @Data + public static class BulkSaveConfigProperties { + private int maxPollsForTaskCompletion = 300; + private int taskCompletionCheckIntervalMs = 2000; + } +} diff --git a/gate-web/src/main/groovy/com/netflix/spinnaker/gate/controllers/PipelineController.groovy b/gate-web/src/main/groovy/com/netflix/spinnaker/gate/controllers/PipelineController.groovy index 9fced36317..ea2dd033a9 100644 --- a/gate-web/src/main/groovy/com/netflix/spinnaker/gate/controllers/PipelineController.groovy +++ b/gate-web/src/main/groovy/com/netflix/spinnaker/gate/controllers/PipelineController.groovy @@ -18,6 +18,7 @@ package com.netflix.spinnaker.gate.controllers import com.fasterxml.jackson.databind.ObjectMapper +import com.netflix.spinnaker.gate.config.controllers.PipelineControllerConfigProperties import com.netflix.spinnaker.gate.services.PipelineService import com.netflix.spinnaker.gate.services.TaskService import com.netflix.spinnaker.gate.services.internal.Front50Service @@ -41,6 +42,8 @@ import org.springframework.security.access.prepost.PreAuthorize import org.springframework.web.bind.annotation.* import retrofit.RetrofitError +import java.nio.charset.StandardCharsets + import static net.logstash.logback.argument.StructuredArguments.value @Slf4j @@ -60,6 +63,9 @@ class PipelineController { @Autowired ObjectMapper objectMapper + @Autowired + PipelineControllerConfigProperties pipelineControllerConfigProperties + @CompileDynamic @ApiOperation(value = "Delete a pipeline definition") @DeleteMapping("/{application}/{pipelineName:.+}") @@ -125,6 +131,46 @@ class PipelineController { } } + @CompileDynamic + @ApiOperation(value = "Save a list of pipelines") + @PostMapping('/bulksave') + Map bulksavePipeline( + @RequestParam(defaultValue = "bulk_save_placeholder_app") + @ApiParam(value = "Application in which to run the bulk save task", + defaultValue = "bulk_save_placeholder_app", + required = false) String application, + @RequestBody List pipelines) { + def operation = [ + description: "Bulk save pipelines", + application: application, + job : [ + [ + type : "savePipeline", + pipelines : Base64.encoder + .encodeToString(objectMapper.writeValueAsString(pipelines).getBytes(StandardCharsets.UTF_8)), + user : AuthenticatedRequest.spinnakerUser.orElse("anonymous"), + isBulkSavingPipelines : true + ] + ] + ] + + def result = taskService.createAndWaitForCompletion(operation, + pipelineControllerConfigProperties.getBulksave().getMaxPollsForTaskCompletion(), + pipelineControllerConfigProperties.getBulksave().getTaskCompletionCheckIntervalMs()) + String resultStatus = result.get("status") + + if (!"SUCCEEDED".equalsIgnoreCase(resultStatus)) { + String exception = result.variables.find { it.key == "exception" }?.value?.details?.errors?.getAt(0) + throw new PipelineException( + exception ?: "Pipeline bulk save operation did not succeed: ${result.get("id", "unknown task id")} " + + "(status: ${resultStatus})" + ) + } else { + def retVal = result.variables.find { it.key == "bulksave"}?.value + return retVal + } + } + @ApiOperation(value = "Rename a pipeline definition") @PostMapping('move') void renamePipeline(@RequestBody Map renameCommand) { diff --git a/gate-web/src/test/groovy/com/netflix/spinnaker/gate/FunctionalSpec.groovy b/gate-web/src/test/groovy/com/netflix/spinnaker/gate/FunctionalSpec.groovy index dbb29e04f8..159e3feb40 100644 --- a/gate-web/src/test/groovy/com/netflix/spinnaker/gate/FunctionalSpec.groovy +++ b/gate-web/src/test/groovy/com/netflix/spinnaker/gate/FunctionalSpec.groovy @@ -22,6 +22,7 @@ import com.netflix.spinnaker.config.ErrorConfiguration import com.netflix.spinnaker.fiat.shared.FiatClientConfigurationProperties import com.netflix.spinnaker.fiat.shared.FiatStatus import com.netflix.spinnaker.gate.config.ServiceConfiguration +import com.netflix.spinnaker.gate.config.controllers.PipelineControllerConfigProperties import com.netflix.spinnaker.gate.controllers.ApplicationController import com.netflix.spinnaker.gate.controllers.PipelineController import com.netflix.spinnaker.gate.services.* @@ -39,7 +40,7 @@ import org.springframework.core.annotation.Order import org.springframework.security.config.annotation.web.builders.HttpSecurity import org.springframework.security.config.annotation.web.configuration.WebSecurityConfigurerAdapter import retrofit.RetrofitError -import retrofit.RestAdapter; +import retrofit.RestAdapter import retrofit.client.OkClient import retrofit.converter.JacksonConverter import retrofit.mime.TypedInput @@ -287,6 +288,11 @@ class FunctionalSpec extends Specification { ) } + @Bean + PipelineControllerConfigProperties pipelineControllerConfigProperties() { + new PipelineControllerConfigProperties(); + } + @Override protected void configure(HttpSecurity http) throws Exception { http diff --git a/gate-web/src/test/groovy/com/netflix/spinnaker/gate/controllers/PipelineControllerSpec.groovy b/gate-web/src/test/groovy/com/netflix/spinnaker/gate/controllers/PipelineControllerSpec.groovy index 2abe746f52..e30720592d 100644 --- a/gate-web/src/test/groovy/com/netflix/spinnaker/gate/controllers/PipelineControllerSpec.groovy +++ b/gate-web/src/test/groovy/com/netflix/spinnaker/gate/controllers/PipelineControllerSpec.groovy @@ -17,8 +17,10 @@ package com.netflix.spinnaker.gate.controllers import com.fasterxml.jackson.databind.ObjectMapper +import com.netflix.spinnaker.gate.config.controllers.PipelineControllerConfigProperties import com.netflix.spinnaker.gate.services.TaskService import com.netflix.spinnaker.gate.services.internal.Front50Service +import groovy.json.JsonSlurper import org.springframework.http.MediaType import org.springframework.test.web.servlet.MockMvc import org.springframework.test.web.servlet.setup.MockMvcBuilders @@ -28,18 +30,25 @@ import retrofit.client.Response import retrofit.converter.JacksonConverter import spock.lang.Specification +import java.nio.charset.StandardCharsets + import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.post import static org.springframework.test.web.servlet.request.MockMvcRequestBuilders.put class PipelineControllerSpec extends Specification { + def taskSerivce = Mock(TaskService) + def front50Service = Mock(Front50Service) + def pipelineControllerConfig = new PipelineControllerConfigProperties() + def mockMvc = MockMvcBuilders + .standaloneSetup(new PipelineController(objectMapper: new ObjectMapper(), + taskService: taskSerivce, + front50Service: front50Service, + pipelineControllerConfigProperties: pipelineControllerConfig)) + .build() + def "should update a pipeline"() { given: - def taskSerivce = Mock(TaskService) - def front50Service = Mock(Front50Service) - MockMvc mockMvc = MockMvcBuilders.standaloneSetup(new PipelineController(objectMapper: new ObjectMapper(), taskService: taskSerivce, front50Service: front50Service)).build() - - and: def pipeline = [ id: "id", name: "test pipeline", @@ -84,9 +93,6 @@ class PipelineControllerSpec extends Specification { def "should propagate pipeline template errors"() { given: - MockMvc mockMvc = MockMvcBuilders.standaloneSetup(new PipelineController(objectMapper: new ObjectMapper())).build() - - and: def pipeline = [ type: 'templatedPipeline', config: [:] @@ -118,4 +124,193 @@ class PipelineControllerSpec extends Specification { def e = thrown(RetrofitError) e.body == mockedHttpException } + + def "should bulk save pipelines"() { + given: + def pipelines = [ + [ + id: "pipeline1_id", + name: "test pipeline", + application: "application" + ], + [ + id: "pipeline2_id", + name: "test pipeline", + application: "application2" + ] + ] + def createAndWaitResult = [ + id: 'task-id', + application: 'bulk_save_placeholder_app', + status: 'SUCCEEDED', + variables: [ + [ + key: "isBulkSavingPipelines", + value: true + ], + [ + key: "bulksave", + value: [ + successful_pipelines_count: 2, + failed_pipelines_count: 0, + failed_pipelines_list: [] + ] + ] + ] + ] + + when: + def response = mockMvc + .perform( + post("/pipelines/bulksave") + .contentType(MediaType.APPLICATION_JSON) + .content(new ObjectMapper().writeValueAsString(pipelines))) + .andReturn() + .response + + then: + response.status == 200 + 1 * taskSerivce.createAndWaitForCompletion( + [ + description: "Bulk save pipelines", + application: 'bulk_save_placeholder_app', + job: [ + [ + type : 'savePipeline', + pipelines : Base64.encoder + .encodeToString(new ObjectMapper().writeValueAsString(pipelines).getBytes(StandardCharsets.UTF_8)), + user : 'anonymous', + isBulkSavingPipelines : true + ] + ] + ], + 300, + 2000 + ) >> createAndWaitResult + new JsonSlurper().parseText(response.getContentAsString()) == [ + "successful_pipelines_count": 2, + "failed_pipelines_count": 0, + "failed_pipelines_list": [] + ] + + // test with custom app + when: + response = mockMvc + .perform( + post("/pipelines/bulksave") + .param("application", "my_test_app") + .contentType(MediaType.APPLICATION_JSON) + .content(new ObjectMapper().writeValueAsString(pipelines))) + .andReturn() + .response + + then: + response.status == 200 + 1 * taskSerivce.createAndWaitForCompletion( + [ + description: "Bulk save pipelines", + application: 'my_test_app', + job: [ + [ + type : 'savePipeline', + pipelines : Base64.encoder + .encodeToString(new ObjectMapper().writeValueAsString(pipelines).getBytes(StandardCharsets.UTF_8)), + user : 'anonymous', + isBulkSavingPipelines : true + ] + ] + ], + 300, + 2000 + ) >> createAndWaitResult + + // Test with custom task completion configs + when: + pipelineControllerConfig.getBulksave().maxPollsForTaskCompletion = 10 + pipelineControllerConfig.getBulksave().taskCompletionCheckIntervalMs = 200 + response = mockMvc + .perform( + post("/pipelines/bulksave") + .param("application", "my_test_app") + .contentType(MediaType.APPLICATION_JSON) + .content(new ObjectMapper().writeValueAsString(pipelines))) + .andReturn() + .response + + then: + response.status == 200 + 1 * taskSerivce.createAndWaitForCompletion( + [ + description: "Bulk save pipelines", + application: 'my_test_app', + job: [ + [ + type : 'savePipeline', + pipelines : Base64.encoder + .encodeToString(new ObjectMapper().writeValueAsString(pipelines).getBytes(StandardCharsets.UTF_8)), + user : 'anonymous', + isBulkSavingPipelines : true + ] + ] + ], + 10, + 200 + ) >> createAndWaitResult + } + + def "bulk save raises exception on failure"() { + given: + def pipelines = [ + [ + id: "pipeline1_id", + name: "test pipeline", + application: "application" + ] + ] + + when: + def response = mockMvc + .perform( + post("/pipelines/bulksave") + .contentType(MediaType.APPLICATION_JSON) + .content(new ObjectMapper().writeValueAsString(pipelines))) + .andReturn() + .response + + then: + 1 * taskSerivce.createAndWaitForCompletion( + [ + description: "Bulk save pipelines", + application: 'bulk_save_placeholder_app', + job: [ + [ + type : 'savePipeline', + pipelines : Base64.encoder + .encodeToString(new ObjectMapper().writeValueAsString(pipelines).getBytes(StandardCharsets.UTF_8)), + user : 'anonymous', + isBulkSavingPipelines : true + ] + ] + ], + 300, + 2000 + ) >> { + [ + id: 'task-id', + application: 'bulk_save_placeholder_app', + status: 'TERMINAL', + variables: + [ + [ + key: "exception", + value: [ details: [ errors: ["error happened"] + ] + ] + ] + ] + ] + } + response.status == 400 + response.contentAsString == "" + } }