PHEE-415 Implemented the batch REST Api and Optimised imports and upd…

…ated the batch API (openMF#105)

* Implemented the batch REST Api and Optimised imports and updated the batch API

* Header parsing bug fixed in camel

* Reverted the codestyle changes

* Reverted the codestyle changes

* Reverted the codestyle changes

* Removed code Formatting

* Bug fixed in converting the batchRequest to txn object

* Added header validation

* Wired raw request with csv request

.circleci/config.yml

@@ -12,7 +12,7 @@ jobs:
     executor: docker-executor
     environment:
       JVM_OPTS: -Xmx512m
-      TERM: dumb  
+      TERM: dumb
       GITHUB_TOKEN: ${GITHUB_TOKEN}  # Add the GitHub token as an environment variable
 
     steps:
@@ -40,7 +40,7 @@ jobs:
             docker push "openmf/ph-ee-bulk-processor:$IMAGE_TAG"
 
           # when: always  # The job will be executed even if there's no match for the tag filter
-  
+
   build_and_push_latest_image:
     executor: docker-executor
     environment:
@@ -68,7 +68,7 @@ jobs:
       # Push the Docker image to DockerHub
       - run:
           name: Push Docker image to DockerHub
-          command: docker push openmf/ph-ee-bulk-processor:latest        
+          command: docker push openmf/ph-ee-bulk-processor:latest
 
 workflows:
   version: 2
@@ -79,8 +79,7 @@ workflows:
             tags:
               only: /^v\d+\.\d+\.\d+$/  # Match tags in the format v1.2.3
           context:
-              - DOCKER
+            - DOCKER
       - build_and_push_latest_image:
           context:
-              - DOCKER
-
+            - DOCKER

README.md

@@ -11,7 +11,7 @@ server:
     key-store-password: "<replace-with-password>"
   port: 8443
 ```
-#### NOTE: For disabling TLS, change the port to "8080" and add null values for all the "ssl" related fields. 
+#### NOTE: For disabling TLS, change the port to "8080" and add null values for all the "ssl" related fields.
 
 ## Checkstyle
 Use below command to execute the checkstyle test.

build.gradle

@@ -108,7 +108,7 @@ dependencies {
     // miscellaneous dependency
     implementation 'com.google.code.gson:gson:2.8.9'
     implementation 'org.json:json:20210307'
-    implementation 'org.mifos:ph-ee-connector-common:1.6.0-rc.1'
+    implementation 'org.mifos:ph-ee-connector-common:1.6.1-rc.1'
     implementation 'org.apache.camel.springboot:camel-spring-boot-starter:3.4.0'
     implementation 'org.apache.camel:camel-undertow:3.4.0'
     implementation 'org.springframework.boot:spring-boot-starter:2.5.2'

src/main/java/org/mifos/processor/bulk/API/definition/BatchTransactions.java

@@ -8,22 +8,28 @@
 import org.springframework.web.multipart.MultipartFile;
 
 import javax.mail.Multipart;
+import javax.servlet.http.HttpServletRequest;
 import javax.servlet.http.HttpServletResponse;
 
 import java.io.IOException;
 
 import static org.mifos.processor.bulk.camel.config.CamelProperties.*;
+import static org.mifos.processor.bulk.camel.config.CamelProperties.HEADER_CLIENT_CORRELATION_ID;
+import static org.mifos.processor.bulk.camel.config.CamelProperties.HEADER_PLATFORM_TENANT_ID;
 import static org.mifos.processor.bulk.zeebe.ZeebeVariables.*;
 
 public interface BatchTransactions {
 
-    @PostMapping(value = "/batchtransactions", produces="application/json")
-    String batchTransactions(HttpServletResponse httpServletResponse, @RequestHeader(value = "X-CorrelationID") String requestId,
-                             @RequestParam("data") MultipartFile file,
-                             @RequestHeader(value = FILE_NAME) String fileName,
-                             @RequestHeader(value = PURPOSE) String purpose,
-                             @RequestHeader(value = "Type") String type,
-                             @RequestHeader(value = HEADER_PLATFORM_TENANT_ID) String tenant,
-                             @RequestHeader(value = HEADER_REGISTERING_INSTITUTE_ID, required = false) String registeringInstitutionId,
-                             @RequestHeader(value = HEADER_PROGRAM_ID, required = false) String programId) throws IOException;
+    @PostMapping(value = "/batchtransactions", produces = "application/json")
+    String batchTransactions(
+            HttpServletRequest httpServletRequest,
+            HttpServletResponse httpServletResponse,
+            @RequestHeader(value = HEADER_CLIENT_CORRELATION_ID) String requestId,
+            @RequestHeader(value = FILE_NAME, required = false) String fileName,
+            @RequestHeader(value = PURPOSE) String purpose,
+            @RequestHeader(value = HEADER_TYPE) String type,
+            @RequestHeader(value = HEADER_PLATFORM_TENANT_ID) String tenant,
+            @RequestHeader(value = HEADER_REGISTERING_INSTITUTE_ID, required = false) String registeringInstitutionId,
+            @RequestHeader(value = HEADER_PROGRAM_ID, required = false) String programId) throws IOException;
+
 }

src/main/java/org/mifos/processor/bulk/API/definition/BulkTransfer.java

@@ -11,11 +11,11 @@
 import static org.mifos.processor.bulk.zeebe.ZeebeVariables.PURPOSE;
 
 public interface BulkTransfer {
+
+    @Deprecated
     @PostMapping(value = "/bulk/transfer/{requestId}/{fileName}", produces = "application/json")
     String bulkTransfer(@RequestHeader(value = "X-CorrelationID", required = false) String requestId,
-                        @RequestParam("data") MultipartFile file,
-                        @RequestHeader(value = FILE_NAME, required = false) String fileName,
-                        @RequestHeader(value = PURPOSE, required = false) String purpose,
-                        @RequestHeader(value = "Type", required = false) String type,
-                        @RequestHeader(value = "Platform-TenantId") String tenant) throws IOException;
+            @RequestParam("data") MultipartFile file, @RequestHeader(value = FILE_NAME, required = false) String fileName,
+            @RequestHeader(value = PURPOSE, required = false) String purpose, @RequestHeader(value = "Type", required = false) String type,
+            @RequestHeader(value = "Platform-TenantId") String tenant) throws IOException;
 }

src/main/java/org/mifos/processor/bulk/API/implementation/BatchTransactionsController.java

@@ -1,27 +1,46 @@
 package org.mifos.processor.bulk.api.implementation;
 
+import com.fasterxml.jackson.core.type.TypeReference;
 import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.dataformat.csv.CsvMapper;
 import lombok.SneakyThrows;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.camel.Exchange;
 import org.apache.camel.ProducerTemplate;
+import org.apache.commons.io.IOUtils;
 import org.json.JSONObject;
+import org.mifos.connector.common.interceptor.JWSUtil;
 import org.mifos.processor.bulk.api.definition.BatchTransactions;
 import org.mifos.processor.bulk.file.FileStorageService;
+import org.mifos.processor.bulk.format.RestRequestConvertor;
+import org.mifos.processor.bulk.schema.BatchRequestDTO;
+import org.mifos.processor.bulk.schema.CamelApiResponse;
+import org.mifos.processor.bulk.schema.Transaction;
+import org.mifos.processor.bulk.utility.CsvWriter;
 import org.mifos.processor.bulk.utility.Headers;
 import org.mifos.processor.bulk.utility.SpringWrapperUtil;
 import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Value;
 import org.springframework.web.bind.annotation.ExceptionHandler;
 import org.springframework.web.bind.annotation.RequestHeader;
 import org.springframework.web.bind.annotation.RestController;
 import org.springframework.web.multipart.MultipartException;
-import org.springframework.web.multipart.MultipartFile;
+import javax.servlet.http.HttpServletRequest;
 import javax.servlet.http.HttpServletResponse;
 import java.io.IOException;
-
 import static org.mifos.processor.bulk.camel.config.CamelProperties.*;
 import static org.mifos.processor.bulk.zeebe.ZeebeVariables.PURPOSE;
+import java.nio.charset.Charset;
+import java.util.List;
+import java.util.Optional;
+import java.util.UUID;
+import static org.mifos.processor.bulk.camel.config.CamelProperties.HEADER_PROGRAM_ID;
+import static org.mifos.processor.bulk.camel.config.CamelProperties.HEADER_REGISTERING_INSTITUTE_ID;
 import static org.mifos.processor.bulk.zeebe.ZeebeVariables.FILE_NAME;
+import static org.mifos.processor.bulk.zeebe.ZeebeVariables.HEADER_CLIENT_CORRELATION_ID;
+import static org.mifos.processor.bulk.zeebe.ZeebeVariables.HEADER_PLATFORM_TENANT_ID;
+import static org.mifos.processor.bulk.zeebe.ZeebeVariables.HEADER_TYPE;
+import static org.mifos.processor.bulk.zeebe.ZeebeVariables.PURPOSE;
 
 @Slf4j
 @RestController
@@ -36,41 +55,118 @@ public class BatchTransactionsController implements BatchTransactions {
     @Autowired
     FileStorageService fileStorageService;
 
+    @Autowired
+    RestRequestConvertor restRequestConvertor;
+
+    @Value("#{'${tenants}'.split(',')}")
+    protected List<String> tenants;
+    @Autowired
+    private CsvMapper csvMapper;
+
     @SneakyThrows
     @Override
-    public String batchTransactions(HttpServletResponse httpServletResponse,
-                                    String requestId, MultipartFile file, String fileName,
-                                    String purpose, String type, String tenant,
-                                    String registeringInstitutionId, String programId) throws IOException {
-        log.debug("Inside api logic");
-        String localFileName = fileStorageService.save(file);
-        Headers headers = new Headers.HeaderBuilder()
+    public String batchTransactions(
+            HttpServletRequest httpServletRequest,
+            HttpServletResponse httpServletResponse,
+            String requestId,
+            String fileName,
+            String purpose,
+            String type,
+            String tenant,
+            String registeringInstitutionId,
+            String programId) {
+
+        log.info("Inside api logic");
+        Headers.HeaderBuilder headerBuilder = new Headers.HeaderBuilder()
                 .addHeader(HEADER_CLIENT_CORRELATION_ID, requestId)
-                .addHeader(PURPOSE,purpose)
-                .addHeader(FILE_NAME,localFileName)
-                .addHeader("Type",type)
-                .addHeader(HEADER_PLATFORM_TENANT_ID,tenant)
+                .addHeader(PURPOSE, purpose)
+                .addHeader(HEADER_TYPE, type)
+                .addHeader(HEADER_PLATFORM_TENANT_ID, tenant)
                 .addHeader(HEADER_REGISTERING_INSTITUTE_ID, registeringInstitutionId)
-                .addHeader(HEADER_PROGRAM_ID, programId)
-                .build();
-		log.debug("Headers passed: {}", headers);
-        Exchange exchange = SpringWrapperUtil.getDefaultWrappedExchange(producerTemplate.getCamelContext(),
-              headers);
-        log.debug("Header in exchange: {}", exchange.getIn().getHeaders());
+                .addHeader(HEADER_PROGRAM_ID, programId);
+
+        Optional<String> validationResponse = isValidRequest(httpServletRequest, fileName, type);
+        if (validationResponse.isPresent()) {
+            httpServletResponse.setStatus(httpServletResponse.SC_BAD_REQUEST);
+            return validationResponse.get();
+        }
+
+        if (JWSUtil.isMultipartRequest(httpServletRequest)) {
+            log.info("This is file based request");
+            String localFileName = fileStorageService.save(JWSUtil.parseFormData(httpServletRequest), fileName);
+            Headers headers = headerBuilder.addHeader(FILE_NAME, localFileName).build();
+            log.info("Headers passed: {}", headers.getHeaders());
+
+            CamelApiResponse response = sendRequestToCamel(headers);
+            httpServletResponse.setStatus(response.getStatus());
+            return response.getBody();
+        } else {
+            log.info("This is json based request");
+            String jsonString = IOUtils.toString(httpServletRequest.getInputStream(), Charset.defaultCharset());
+            List<BatchRequestDTO> batchRequestDTOList = objectMapper.readValue(jsonString, new TypeReference<>() {});
+            List<Transaction> transactionList = restRequestConvertor.convertListFrom(batchRequestDTOList);
+
+            String localFileName = UUID.randomUUID() + ".csv";
+            CsvWriter.writeToCsv(transactionList, Transaction.class, csvMapper, true, localFileName);
+            Headers headers = headerBuilder
+                    .addHeader(HEADER_TYPE, "csv")
+                    .addHeader(FILE_NAME, localFileName).build();
+
+            CamelApiResponse response = sendRequestToCamel(headers);
+            httpServletResponse.setStatus(response.getStatus());
+            return response.getBody();
+        }
+    }
+
+    @ExceptionHandler({ MultipartException.class })
+    public String handleMultipartException(HttpServletResponse httpServletResponse) {
+        httpServletResponse.setStatus(httpServletResponse.SC_BAD_REQUEST);
+        return getErrorResponse("File not uploaded", "There was no fie uploaded with the request. " +
+                "Please upload a file and try again.", 400);
+    }
+
+    private CamelApiResponse sendRequestToCamel(Headers headers) {
+        Exchange exchange = SpringWrapperUtil.getDefaultWrappedExchange(producerTemplate.getCamelContext(), headers);
         exchange = producerTemplate.send("direct:post-batch-transactions", exchange);
         int statusCode = exchange.getIn().getHeader(Exchange.HTTP_RESPONSE_CODE, Integer.class);
-		httpServletResponse.setStatus(statusCode);
-       	return exchange.getIn().getBody(String.class);
+        String body = exchange.getIn().getBody(String.class);
+        return new CamelApiResponse(body, statusCode);
     }
 
-    @ExceptionHandler({MultipartException.class})
-    public String handleMultipartException(HttpServletResponse httpServletResponse) {
+    private String getErrorResponse(String information, String description, int code) {
         JSONObject json = new JSONObject();
-        json.put("Error Information: ", "File not uploaded");
-        json.put("Error Description : ", "There was no fie uploaded with the request. " +
-                "Please upload a file and try again.");
-        httpServletResponse.setStatus(httpServletResponse.SC_BAD_REQUEST);
+        json.put("errorInformation", "File not uploaded");
+        json.put("errorDescription", "There was no fie uploaded with the request. " + "Please upload a file and try again.");
+        json.put("errorCode", code);
         return json.toString();
     }
 
+    // validates the request header, and return errorJson string if the request is invalid else an empty optional
+    private Optional<String> isValidRequest(HttpServletRequest httpServletRequest,
+                                             String fileName,
+                                             String type) {
+
+		Optional<String> response = Optional.empty();
+        if ((JWSUtil.isMultipartRequest(httpServletRequest) && !type.equalsIgnoreCase("csv")) ||
+                (!JWSUtil.isMultipartRequest(httpServletRequest) && !type.equalsIgnoreCase("raw"))) {
+            String errorJson = getErrorResponse("Type mismatch",
+                    "The value of the header \"" + HEADER_TYPE +
+                    "\" doesn't match with the request content-type", 400);
+            response = Optional.of(errorJson);
+
+        }
+        if (JWSUtil.isMultipartRequest(httpServletRequest) && fileName.isEmpty()) {
+            String errorJson = getErrorResponse("Header can't be empty",
+                    "If the request is of type csv, the header \"" +
+                            FILE_NAME + "\"can't be empty", 400);
+            response = Optional.of(errorJson);
+        }
+        if (!type.equalsIgnoreCase("raw") && !type.equalsIgnoreCase("csv")) {
+            String errorJson = getErrorResponse("Invalid TYPE header value passed",
+                    "The value of the header \"" + HEADER_TYPE +
+                            "\" can be \"[raw,csv]\" but is " + type, 400);
+            response = Optional.of(errorJson);
+        }
+		return response;
+    }
 }

src/main/java/org/mifos/processor/bulk/API/implementation/BulkTransferController.java

@@ -10,15 +10,16 @@
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.web.bind.annotation.RestController;
 import org.springframework.web.multipart.MultipartFile;
-
 import java.io.IOException;
-
 import static org.mifos.processor.bulk.camel.config.CamelProperties.HEADER_PLATFORM_TENANT_ID;
 import static org.mifos.processor.bulk.zeebe.ZeebeVariables.FILE_NAME;
+import static org.mifos.processor.bulk.zeebe.ZeebeVariables.HEADER_CLIENT_CORRELATION_ID;
+import static org.mifos.processor.bulk.zeebe.ZeebeVariables.HEADER_TYPE;
 import static org.mifos.processor.bulk.zeebe.ZeebeVariables.PURPOSE;
 
 @RestController
 public class BulkTransferController implements BulkTransfer {
+
     @Autowired
     private ProducerTemplate producerTemplate;
 
@@ -29,16 +30,11 @@ public class BulkTransferController implements BulkTransfer {
     FileStorageService fileStorageService;
 
     @Override
-    public String bulkTransfer(String requestId, MultipartFile file, String fileName, String purpose, String type, String tenant) throws IOException {
-        Headers headers = new Headers.HeaderBuilder()
-                .addHeader("X-CorrelationID", requestId)
-                .addHeader(PURPOSE,purpose)
-                .addHeader(FILE_NAME,fileName)
-                .addHeader("Type",type)
-                .addHeader(HEADER_PLATFORM_TENANT_ID,tenant)
-                .build();
-        Exchange exchange = SpringWrapperUtil.getDefaultWrappedExchange(producerTemplate.getCamelContext(),
-                headers);
+    public String bulkTransfer(String requestId, MultipartFile file, String fileName, String purpose, String type, String tenant)
+            throws IOException {
+        Headers headers = new Headers.HeaderBuilder().addHeader(HEADER_CLIENT_CORRELATION_ID, requestId).addHeader(PURPOSE, purpose)
+                .addHeader(FILE_NAME, fileName).addHeader(HEADER_TYPE, type).addHeader(HEADER_PLATFORM_TENANT_ID, tenant).build();
+        Exchange exchange = SpringWrapperUtil.getDefaultWrappedExchange(producerTemplate.getCamelContext(), headers);
         fileStorageService.save(file);
         producerTemplate.send("direct:post-bulk-transfer", exchange);
         return exchange.getIn().getBody(String.class);

src/main/java/org/mifos/processor/bulk/camel/routes/ExternalApiCallRoute.java

@@ -5,7 +5,6 @@
 
 import org.apache.camel.LoggingLevel;
 import org.apache.camel.builder.RouteBuilder;
-import org.springframework.stereotype.Component;
 
 //@Component
 public class ExternalApiCallRoute extends RouteBuilder {

src/main/java/org/mifos/processor/bulk/camel/routes/FileProcessingRoute.java

@@ -20,6 +20,7 @@
 import java.util.List;
 import org.mifos.processor.bulk.schema.Transaction;
 import org.mifos.processor.bulk.schema.TransactionResult;
+import org.mifos.processor.bulk.utility.CsvWriter;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Component;
 
@@ -77,7 +78,7 @@ public void configure() {
                     // getting header
                     Boolean overrideHeader = exchange.getProperty(OVERRIDE_HEADER, Boolean.class);
 
-                    csvWriter(transactionList, TransactionResult.class, csvMapper, overrideHeader, filepath);
+                    CsvWriter.writeToCsv(transactionList, TransactionResult.class, csvMapper, overrideHeader, filepath);
                 }).log("Update complete");
 
         /**
@@ -113,20 +114,4 @@ public void configure() {
             }
         });
     }
-
-    private <T> void csvWriter(List<T> data, Class<T> tClass, CsvMapper csvMapper, boolean overrideHeader, String filepath)
-            throws IOException {
-        CsvSchema csvSchema = csvMapper.schemaFor(tClass);
-        if (overrideHeader) {
-            csvSchema = csvSchema.withHeader();
-        } else {
-            csvSchema = csvSchema.withoutHeader();
-        }
-
-        File file = new File(filepath);
-        SequenceWriter writer = csvMapper.writerWithSchemaFor(tClass).with(csvSchema).writeValues(file);
-        for (T object : data) {
-            writer.write(object);
-        }
-    }
 }