Reworking to push to print file data to a gcs bucket (#138)

* reworking to push to gcs bucket

* [ci skip] auto patch increment
skip-checks: true


Co-authored-by: ras-rm-bot <[email protected]>

_infra/helm/action-exporter/Chart.yaml

@@ -18,4 +18,4 @@ version: 1.3.0
 
 # This is the version number of the application being deployed. This version number should be
 # incremented each time you make changes to the application.
-appVersion: 12.4.0
+appVersion: 12.4.1

_infra/helm/action-exporter/templates/deployment.yaml

@@ -194,6 +194,10 @@ spec:
             value: "{{ .Values.gcp.project }}"
           - name: GCP_TOPIC
             value: "{{ .Values.gcp.topic }}"
+          - name: GCS_ENABLED
+            value: "{{ .Values.gcs.enabled }}"
+          - name: GCS_BUCKET
+            value: "{{ .Values.gcs.bucket }}"
           - name: GOOGLE_APPLICATION_CREDENTIALS
             value: /var/secrets/google/credentials.json
           resources:

_infra/helm/action-exporter/values.yaml

@@ -48,6 +48,9 @@ resources:
 managedRabbitMQ:
   enabled: false
 
+gcs:
+  bucket: "ras-rm-print-file"
+
 gcp:
   project: ras-rm-sandbox
   topic: print-file-jobs

src/main/java/uk/gov/ons/ctp/response/action/export/config/AppConfig.java

@@ -12,4 +12,5 @@
 public class AppConfig {
   private Rabbitmq rabbitmq;
   private Logging logging;
+  private GCS gcs;
 }

src/main/java/uk/gov/ons/ctp/response/action/export/config/GCS.java

@@ -0,0 +1,9 @@
+package uk.gov.ons.ctp.response.action.export.config;
+
+import lombok.Data;
+
+/** Config POJO for GCS params */
+@Data
+public class GCS {
+  private String bucket;
+}

src/main/java/uk/gov/ons/ctp/response/action/export/message/UploadObjectGCS.java

@@ -0,0 +1,37 @@
+package uk.gov.ons.ctp.response.action.export.message;
+
+import com.google.cloud.storage.*;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.stereotype.Component;
+
+@Component
+public class UploadObjectGCS {
+  private static final Logger log = LoggerFactory.getLogger(UploadObjectGCS.class);
+  private final Storage storage;
+
+  public UploadObjectGCS(Storage storage) {
+    this.storage = storage;
+  }
+
+  public boolean uploadObject(String filename, String bucket, byte[] data) {
+    BlobId blobId = BlobId.of(bucket, filename);
+    BlobInfo blobInfo = BlobInfo.newBuilder(blobId).setContentType("application/json").build();
+    Boolean isSuccess = false;
+    log.info("file_name: ", filename + " bucket: " + bucket + ", Uploading to GCS bucket");
+    try {
+      storage.create(blobInfo, data);
+      isSuccess = true;
+      log.info("file_name: " + filename + " bucket: " + bucket + ", Upload Successful!");
+    } catch (StorageException exception) {
+      log.error(
+          "file_name: "
+              + filename
+              + " bucket: "
+              + bucket
+              + ", Error uploading the generated file to GCS",
+          exception);
+    }
+    return isSuccess;
+  }
+}

src/main/java/uk/gov/ons/ctp/response/action/export/service/NotificationFileCreator.java

@@ -55,13 +55,15 @@ public void uploadData(
 
     log.info("filename: " + filename + ", uploading file");
 
-    ExportFile exportFile = new ExportFile();
-    exportFile.setExportJobId(exportJob.getId());
-    exportFile.setFilename(filename);
-    exportFileRepository.saveAndFlush(exportFile);
-
     // temporarily hook in here as at this point we know the name of the file
     // and all the action request instructions
-    printFileService.send(filename, actionRequestInstructions);
+    boolean success = printFileService.send(filename, actionRequestInstructions);
+
+    if (success) {
+      ExportFile exportFile = new ExportFile();
+      exportFile.setExportJobId(exportJob.getId());
+      exportFile.setFilename(filename);
+      exportFileRepository.saveAndFlush(exportFile);
+    }
   }
 }

src/main/java/uk/gov/ons/ctp/response/action/export/service/PrintFileService.java

@@ -10,9 +10,12 @@
 import java.util.List;
 import java.util.concurrent.ExecutionException;
 import lombok.extern.log4j.Log4j;
+import org.apache.commons.io.FilenameUtils;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Service;
+import uk.gov.ons.ctp.response.action.export.config.AppConfig;
 import uk.gov.ons.ctp.response.action.export.domain.ActionRequestInstruction;
+import uk.gov.ons.ctp.response.action.export.message.UploadObjectGCS;
 import uk.gov.ons.ctp.response.action.export.printfile.Contact;
 import uk.gov.ons.ctp.response.action.export.printfile.PrintFileEntry;
 
@@ -22,26 +25,45 @@ public class PrintFileService {
 
   @Autowired private Publisher publisher;
 
-  public void send(String filename, List<ActionRequestInstruction> actionRequestInstructions) {
+  @Autowired private AppConfig appConfig;
+
+  @Autowired private UploadObjectGCS uploadObjectGCS;
+
+  public boolean send(
+      String printFilename, List<ActionRequestInstruction> actionRequestInstructions) {
+    boolean success = false;
+    String dataFilename = FilenameUtils.removeExtension(printFilename).concat(".json");
+
     List<PrintFileEntry> printFile = convertToPrintFile(actionRequestInstructions);
     try {
+      log.debug("creating json representation of print file");
       String json = createJsonRepresentation(printFile);
-      log.info("json");
-
       ByteString data = ByteString.copyFromUtf8(json);
-      PubsubMessage pubsubMessage =
-          PubsubMessage.newBuilder().setData(data).putAttributes("filename", filename).build();
 
-      ApiFuture<String> messageIdFuture = publisher.publish(pubsubMessage);
-      String messageId = messageIdFuture.get();
-      log.debug("messageId: " + messageId);
-      log.debug("print file pubsub sent sucessfully");
-      log.info(json);
+      String bucket = appConfig.getGcs().getBucket();
+      log.info("about to uploaded to bucket " + bucket);
+      boolean uploaded = uploadObjectGCS.uploadObject(dataFilename, bucket, data.toByteArray());
+
+      if (uploaded) {
+        ByteString pubsubData = ByteString.copyFromUtf8(dataFilename);
+
+        PubsubMessage pubsubMessage =
+            PubsubMessage.newBuilder()
+                .setData(pubsubData)
+                .putAttributes("printFilename", printFilename)
+                .build();
+
+        ApiFuture<String> messageIdFuture = publisher.publish(pubsubMessage);
+        String messageId = messageIdFuture.get();
+        log.info("print file pubsub successfully sent with messageId:" + messageId);
+        success = true;
+      }
     } catch (JsonProcessingException e) {
       log.error("unable to convert to json", e);
     } catch (InterruptedException | ExecutionException e) {
       log.error("pub/sub error", e);
     }
+    return success;
   }
 
   private String createJsonRepresentation(List<PrintFileEntry> printFile)

src/test/java/uk/gov/ons/ctp/response/action/export/service/NotificationFileCreatorTest.java

@@ -51,13 +51,14 @@ public void shouldCreateTheCorrectFilename() {
     Date now = new Date();
 
     // Given
+    String expectedFilename = String.format("BSNOT_%s.csv", FILENAME_DATE_FORMAT.format(now));
     given(clock.millis()).willReturn(now.getTime());
+    given(printFileService.send(expectedFilename, actionRequestInstructions)).willReturn(true);
 
     // When
     notificationFileCreator.uploadData("BSNOT", actionRequestInstructions, exportJob);
 
     // Then
-    String expectedFilename = String.format("BSNOT_%s.csv", FILENAME_DATE_FORMAT.format(now));
     ArgumentCaptor<ExportFile> exportFileArgumentCaptor = ArgumentCaptor.forClass(ExportFile.class);
     verify(exportFileRepository).saveAndFlush(exportFileArgumentCaptor.capture());
     assertThat(exportFileArgumentCaptor.getValue().getFilename()).isEqualTo(expectedFilename);

src/test/java/uk/gov/ons/ctp/response/action/export/service/PrintFileServiceTest.java

@@ -3,6 +3,7 @@
 import static org.junit.Assert.*;
 import static org.mockito.BDDMockito.given;
 import static org.mockito.Matchers.any;
+import static org.mockito.Matchers.anyString;
 import static org.mockito.Mockito.verify;
 
 import com.google.api.core.ApiFuture;
@@ -13,7 +14,10 @@
 import org.mockito.InjectMocks;
 import org.mockito.Mock;
 import org.mockito.runners.MockitoJUnitRunner;
+import uk.gov.ons.ctp.response.action.export.config.AppConfig;
+import uk.gov.ons.ctp.response.action.export.config.GCS;
 import uk.gov.ons.ctp.response.action.export.domain.ActionRequestInstruction;
+import uk.gov.ons.ctp.response.action.export.message.UploadObjectGCS;
 import uk.gov.ons.ctp.response.action.export.printfile.PrintFileEntry;
 import uk.gov.ons.ctp.response.action.export.utility.ObjectBuilder;
 
@@ -23,6 +27,9 @@ public class PrintFileServiceTest {
 
   @Mock private Publisher publisher;
   @Mock ApiFuture<String> apiFuture;
+  @Mock UploadObjectGCS uploadObjectGCS;
+  @Mock AppConfig appConfig;
+  @Mock GCS gcs;
 
   @InjectMocks private PrintFileService printFileService;
 
@@ -54,16 +61,17 @@ public void testSend() throws Exception {
 
     given(publisher.publish(any())).willReturn(apiFuture);
     given(apiFuture.get()).willReturn("test");
+    given(uploadObjectGCS.uploadObject(anyString(), anyString(), any())).willReturn(true);
+    given(appConfig.getGcs()).willReturn(gcs);
+    given(gcs.getBucket()).willReturn("test-bucket");
+
     List<ActionRequestInstruction> actionRequestInstructions =
         ObjectBuilder.buildListOfActionRequests();
 
-    try {
-      printFileService.send("test.csv", actionRequestInstructions);
-    } catch (RuntimeException e) {
-      fail(e.getMessage());
-    }
+    printFileService.send("test.csv", actionRequestInstructions);
 
     verify(publisher).publish(any());
     verify(apiFuture).get();
+    verify(uploadObjectGCS).uploadObject(anyString(), anyString(), any());
   }
 }