diff --git a/_infra/helm/action-exporter/Chart.yaml b/_infra/helm/action-exporter/Chart.yaml index b923cbc1..8467df3a 100644 --- a/_infra/helm/action-exporter/Chart.yaml +++ b/_infra/helm/action-exporter/Chart.yaml @@ -18,4 +18,4 @@ version: 1.3.0 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. -appVersion: 12.4.0 +appVersion: 12.4.1 diff --git a/_infra/helm/action-exporter/templates/deployment.yaml b/_infra/helm/action-exporter/templates/deployment.yaml index 8f643395..327b1298 100644 --- a/_infra/helm/action-exporter/templates/deployment.yaml +++ b/_infra/helm/action-exporter/templates/deployment.yaml @@ -194,6 +194,10 @@ spec: value: "{{ .Values.gcp.project }}" - name: GCP_TOPIC value: "{{ .Values.gcp.topic }}" + - name: GCS_ENABLED + value: "{{ .Values.gcs.enabled }}" + - name: GCS_BUCKET + value: "{{ .Values.gcs.bucket }}" - name: GOOGLE_APPLICATION_CREDENTIALS value: /var/secrets/google/credentials.json resources: diff --git a/_infra/helm/action-exporter/values.yaml b/_infra/helm/action-exporter/values.yaml index 1032428d..9b580408 100644 --- a/_infra/helm/action-exporter/values.yaml +++ b/_infra/helm/action-exporter/values.yaml @@ -48,6 +48,9 @@ resources: managedRabbitMQ: enabled: false +gcs: + bucket: "ras-rm-print-file" + gcp: project: ras-rm-sandbox topic: print-file-jobs \ No newline at end of file diff --git a/src/main/java/uk/gov/ons/ctp/response/action/export/config/AppConfig.java b/src/main/java/uk/gov/ons/ctp/response/action/export/config/AppConfig.java index 3b80dc2b..53ebd548 100644 --- a/src/main/java/uk/gov/ons/ctp/response/action/export/config/AppConfig.java +++ b/src/main/java/uk/gov/ons/ctp/response/action/export/config/AppConfig.java @@ -12,4 +12,5 @@ public class AppConfig { private Rabbitmq rabbitmq; private Logging logging; + private GCS gcs; } diff --git a/src/main/java/uk/gov/ons/ctp/response/action/export/config/GCS.java b/src/main/java/uk/gov/ons/ctp/response/action/export/config/GCS.java new file mode 100644 index 00000000..ef5d9d4b --- /dev/null +++ b/src/main/java/uk/gov/ons/ctp/response/action/export/config/GCS.java @@ -0,0 +1,9 @@ +package uk.gov.ons.ctp.response.action.export.config; + +import lombok.Data; + +/** Config POJO for GCS params */ +@Data +public class GCS { + private String bucket; +} diff --git a/src/main/java/uk/gov/ons/ctp/response/action/export/message/UploadObjectGCS.java b/src/main/java/uk/gov/ons/ctp/response/action/export/message/UploadObjectGCS.java new file mode 100644 index 00000000..7dc94ed2 --- /dev/null +++ b/src/main/java/uk/gov/ons/ctp/response/action/export/message/UploadObjectGCS.java @@ -0,0 +1,37 @@ +package uk.gov.ons.ctp.response.action.export.message; + +import com.google.cloud.storage.*; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Component; + +@Component +public class UploadObjectGCS { + private static final Logger log = LoggerFactory.getLogger(UploadObjectGCS.class); + private final Storage storage; + + public UploadObjectGCS(Storage storage) { + this.storage = storage; + } + + public boolean uploadObject(String filename, String bucket, byte[] data) { + BlobId blobId = BlobId.of(bucket, filename); + BlobInfo blobInfo = BlobInfo.newBuilder(blobId).setContentType("application/json").build(); + Boolean isSuccess = false; + log.info("file_name: ", filename + " bucket: " + bucket + ", Uploading to GCS bucket"); + try { + storage.create(blobInfo, data); + isSuccess = true; + log.info("file_name: " + filename + " bucket: " + bucket + ", Upload Successful!"); + } catch (StorageException exception) { + log.error( + "file_name: " + + filename + + " bucket: " + + bucket + + ", Error uploading the generated file to GCS", + exception); + } + return isSuccess; + } +} diff --git a/src/main/java/uk/gov/ons/ctp/response/action/export/service/NotificationFileCreator.java b/src/main/java/uk/gov/ons/ctp/response/action/export/service/NotificationFileCreator.java index ece9106a..361d927b 100644 --- a/src/main/java/uk/gov/ons/ctp/response/action/export/service/NotificationFileCreator.java +++ b/src/main/java/uk/gov/ons/ctp/response/action/export/service/NotificationFileCreator.java @@ -55,13 +55,15 @@ public void uploadData( log.info("filename: " + filename + ", uploading file"); - ExportFile exportFile = new ExportFile(); - exportFile.setExportJobId(exportJob.getId()); - exportFile.setFilename(filename); - exportFileRepository.saveAndFlush(exportFile); - // temporarily hook in here as at this point we know the name of the file // and all the action request instructions - printFileService.send(filename, actionRequestInstructions); + boolean success = printFileService.send(filename, actionRequestInstructions); + + if (success) { + ExportFile exportFile = new ExportFile(); + exportFile.setExportJobId(exportJob.getId()); + exportFile.setFilename(filename); + exportFileRepository.saveAndFlush(exportFile); + } } } diff --git a/src/main/java/uk/gov/ons/ctp/response/action/export/service/PrintFileService.java b/src/main/java/uk/gov/ons/ctp/response/action/export/service/PrintFileService.java index fb93e41e..317dbfcc 100644 --- a/src/main/java/uk/gov/ons/ctp/response/action/export/service/PrintFileService.java +++ b/src/main/java/uk/gov/ons/ctp/response/action/export/service/PrintFileService.java @@ -10,9 +10,12 @@ import java.util.List; import java.util.concurrent.ExecutionException; import lombok.extern.log4j.Log4j; +import org.apache.commons.io.FilenameUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; +import uk.gov.ons.ctp.response.action.export.config.AppConfig; import uk.gov.ons.ctp.response.action.export.domain.ActionRequestInstruction; +import uk.gov.ons.ctp.response.action.export.message.UploadObjectGCS; import uk.gov.ons.ctp.response.action.export.printfile.Contact; import uk.gov.ons.ctp.response.action.export.printfile.PrintFileEntry; @@ -22,26 +25,45 @@ public class PrintFileService { @Autowired private Publisher publisher; - public void send(String filename, List actionRequestInstructions) { + @Autowired private AppConfig appConfig; + + @Autowired private UploadObjectGCS uploadObjectGCS; + + public boolean send( + String printFilename, List actionRequestInstructions) { + boolean success = false; + String dataFilename = FilenameUtils.removeExtension(printFilename).concat(".json"); + List printFile = convertToPrintFile(actionRequestInstructions); try { + log.debug("creating json representation of print file"); String json = createJsonRepresentation(printFile); - log.info("json"); - ByteString data = ByteString.copyFromUtf8(json); - PubsubMessage pubsubMessage = - PubsubMessage.newBuilder().setData(data).putAttributes("filename", filename).build(); - ApiFuture messageIdFuture = publisher.publish(pubsubMessage); - String messageId = messageIdFuture.get(); - log.debug("messageId: " + messageId); - log.debug("print file pubsub sent sucessfully"); - log.info(json); + String bucket = appConfig.getGcs().getBucket(); + log.info("about to uploaded to bucket " + bucket); + boolean uploaded = uploadObjectGCS.uploadObject(dataFilename, bucket, data.toByteArray()); + + if (uploaded) { + ByteString pubsubData = ByteString.copyFromUtf8(dataFilename); + + PubsubMessage pubsubMessage = + PubsubMessage.newBuilder() + .setData(pubsubData) + .putAttributes("printFilename", printFilename) + .build(); + + ApiFuture messageIdFuture = publisher.publish(pubsubMessage); + String messageId = messageIdFuture.get(); + log.info("print file pubsub successfully sent with messageId:" + messageId); + success = true; + } } catch (JsonProcessingException e) { log.error("unable to convert to json", e); } catch (InterruptedException | ExecutionException e) { log.error("pub/sub error", e); } + return success; } private String createJsonRepresentation(List printFile) diff --git a/src/test/java/uk/gov/ons/ctp/response/action/export/service/NotificationFileCreatorTest.java b/src/test/java/uk/gov/ons/ctp/response/action/export/service/NotificationFileCreatorTest.java index b8598420..c34dcec1 100644 --- a/src/test/java/uk/gov/ons/ctp/response/action/export/service/NotificationFileCreatorTest.java +++ b/src/test/java/uk/gov/ons/ctp/response/action/export/service/NotificationFileCreatorTest.java @@ -51,13 +51,14 @@ public void shouldCreateTheCorrectFilename() { Date now = new Date(); // Given + String expectedFilename = String.format("BSNOT_%s.csv", FILENAME_DATE_FORMAT.format(now)); given(clock.millis()).willReturn(now.getTime()); + given(printFileService.send(expectedFilename, actionRequestInstructions)).willReturn(true); // When notificationFileCreator.uploadData("BSNOT", actionRequestInstructions, exportJob); // Then - String expectedFilename = String.format("BSNOT_%s.csv", FILENAME_DATE_FORMAT.format(now)); ArgumentCaptor exportFileArgumentCaptor = ArgumentCaptor.forClass(ExportFile.class); verify(exportFileRepository).saveAndFlush(exportFileArgumentCaptor.capture()); assertThat(exportFileArgumentCaptor.getValue().getFilename()).isEqualTo(expectedFilename); diff --git a/src/test/java/uk/gov/ons/ctp/response/action/export/service/PrintFileServiceTest.java b/src/test/java/uk/gov/ons/ctp/response/action/export/service/PrintFileServiceTest.java index 008c4493..72d780c6 100644 --- a/src/test/java/uk/gov/ons/ctp/response/action/export/service/PrintFileServiceTest.java +++ b/src/test/java/uk/gov/ons/ctp/response/action/export/service/PrintFileServiceTest.java @@ -3,6 +3,7 @@ import static org.junit.Assert.*; import static org.mockito.BDDMockito.given; import static org.mockito.Matchers.any; +import static org.mockito.Matchers.anyString; import static org.mockito.Mockito.verify; import com.google.api.core.ApiFuture; @@ -13,7 +14,10 @@ import org.mockito.InjectMocks; import org.mockito.Mock; import org.mockito.runners.MockitoJUnitRunner; +import uk.gov.ons.ctp.response.action.export.config.AppConfig; +import uk.gov.ons.ctp.response.action.export.config.GCS; import uk.gov.ons.ctp.response.action.export.domain.ActionRequestInstruction; +import uk.gov.ons.ctp.response.action.export.message.UploadObjectGCS; import uk.gov.ons.ctp.response.action.export.printfile.PrintFileEntry; import uk.gov.ons.ctp.response.action.export.utility.ObjectBuilder; @@ -23,6 +27,9 @@ public class PrintFileServiceTest { @Mock private Publisher publisher; @Mock ApiFuture apiFuture; + @Mock UploadObjectGCS uploadObjectGCS; + @Mock AppConfig appConfig; + @Mock GCS gcs; @InjectMocks private PrintFileService printFileService; @@ -54,16 +61,17 @@ public void testSend() throws Exception { given(publisher.publish(any())).willReturn(apiFuture); given(apiFuture.get()).willReturn("test"); + given(uploadObjectGCS.uploadObject(anyString(), anyString(), any())).willReturn(true); + given(appConfig.getGcs()).willReturn(gcs); + given(gcs.getBucket()).willReturn("test-bucket"); + List actionRequestInstructions = ObjectBuilder.buildListOfActionRequests(); - try { - printFileService.send("test.csv", actionRequestInstructions); - } catch (RuntimeException e) { - fail(e.getMessage()); - } + printFileService.send("test.csv", actionRequestInstructions); verify(publisher).publish(any()); verify(apiFuture).get(); + verify(uploadObjectGCS).uploadObject(anyString(), anyString(), any()); } }