From 0a1905bc17d357c34d544cd2ab70dd3a07539a5e Mon Sep 17 00:00:00 2001 From: Anna Smirnova <132938234+smirnovaae@users.noreply.github.com> Date: Wed, 28 Feb 2024 16:40:38 -0800 Subject: [PATCH] AB2D-5950 Update Request file for attribution data (#83) --- attribution-data-file-share/build.gradle | 1 + ...ava => AttributionDataShareConstants.java} | 12 +++-- .../AttributionDataShareHandler.java | 7 ++- .../AttributionDataShareHelper.java | 54 ++++++++++++++----- .../AttributionDataShareHandlerTest.java | 4 +- .../AttributionDataShareHelperTest.java | 40 ++++++++------ .../S3MockAPIExtension.java | 2 +- 7 files changed, 82 insertions(+), 38 deletions(-) rename attribution-data-file-share/src/main/java/gov/cms/ab2d/attributionDataShare/{AttributionDataShareHandlerConstants.java => AttributionDataShareConstants.java} (57%) diff --git a/attribution-data-file-share/build.gradle b/attribution-data-file-share/build.gradle index 46f3ddd..3c86422 100644 --- a/attribution-data-file-share/build.gradle +++ b/attribution-data-file-share/build.gradle @@ -18,6 +18,7 @@ dependencies { implementation project(path: ':database-management') implementation project(path: ':lambda-lib') testImplementation 'org.mockito:mockito-core:4.8.0' + testImplementation 'com.mockrunner:mockrunner-jdbc:2.0.7' testImplementation 'org.testcontainers:junit-jupiter:1.17.6' testImplementation "org.testcontainers:postgresql:1.17.6" testImplementation project(':lambda-test-utils') diff --git a/attribution-data-file-share/src/main/java/gov/cms/ab2d/attributionDataShare/AttributionDataShareHandlerConstants.java b/attribution-data-file-share/src/main/java/gov/cms/ab2d/attributionDataShare/AttributionDataShareConstants.java similarity index 57% rename from attribution-data-file-share/src/main/java/gov/cms/ab2d/attributionDataShare/AttributionDataShareHandlerConstants.java rename to attribution-data-file-share/src/main/java/gov/cms/ab2d/attributionDataShare/AttributionDataShareConstants.java index 349a362..cffcb26 100644 --- a/attribution-data-file-share/src/main/java/gov/cms/ab2d/attributionDataShare/AttributionDataShareHandlerConstants.java +++ b/attribution-data-file-share/src/main/java/gov/cms/ab2d/attributionDataShare/AttributionDataShareConstants.java @@ -2,9 +2,9 @@ import software.amazon.awssdk.regions.Region; -public class AttributionDataShareHandlerConstants { +public class AttributionDataShareConstants { - private AttributionDataShareHandlerConstants() { + private AttributionDataShareConstants() { } public static final String BFD_S3_BUCKET_NAME = "ab2d-opt-out-temp-349849222861-us-east-1"; @@ -15,5 +15,11 @@ private AttributionDataShareHandlerConstants() { public static final String FILE_PARTIAL_NAME = "ab2d-beneids_"; public static final String FILE_FORMAT = ".txt"; public static final String PATTERN = "yyyy-MM-dd'T'HH:mm:ss.SSSZ"; - public static final String COPY_STATEMENT = "COPY public.current_mbi TO STDOUT"; + + public static final String FIRST_LINE = "HDR_BENEDATAREQ_"; + public static final String LAST_LINE = "TLR_BENEDATAREQ_"; + public static final String SELECT_STATEMENT = "SELECT * FROM public.current_mbi"; + public static final int CURRENT_MBI_LENGTH = 11; + public static final String EFFECTIVE_DATE_PATTERN = "yyyyMMdd"; + public static final int EFFECTIVE_DATE_LENGTH = 8; } diff --git a/attribution-data-file-share/src/main/java/gov/cms/ab2d/attributionDataShare/AttributionDataShareHandler.java b/attribution-data-file-share/src/main/java/gov/cms/ab2d/attributionDataShare/AttributionDataShareHandler.java index f4e79f3..f8e9919 100644 --- a/attribution-data-file-share/src/main/java/gov/cms/ab2d/attributionDataShare/AttributionDataShareHandler.java +++ b/attribution-data-file-share/src/main/java/gov/cms/ab2d/attributionDataShare/AttributionDataShareHandler.java @@ -3,6 +3,7 @@ import com.amazonaws.services.lambda.runtime.Context; import com.amazonaws.services.lambda.runtime.LambdaLogger; import com.amazonaws.services.lambda.runtime.RequestStreamHandler; +import gov.cms.ab2d.databasemanagement.DatabaseUtil; import gov.cms.ab2d.lambdalibs.lib.FileUtil; import software.amazon.awssdk.services.s3.S3Client; @@ -15,21 +16,23 @@ import java.text.SimpleDateFormat; import java.util.Date; -import static gov.cms.ab2d.attributionDataShare.AttributionDataShareHandlerConstants.*; +import static gov.cms.ab2d.attributionDataShare.AttributionDataShareConstants.*; public class AttributionDataShareHandler implements RequestStreamHandler { // Writes out a file to the FILE_PATH. // I.E: "ab2d-beneids_2023-08-16T12:08:56.235-0700.txt" + public void handleRequest(InputStream inputStream, OutputStream outputStream, Context context) throws IOException { LambdaLogger logger = context.getLogger(); logger.log("AttributionDataShare Lambda is started"); + String currentDate = new SimpleDateFormat(PATTERN).format(new Date()); String fileName = FILE_PARTIAL_NAME + currentDate + FILE_FORMAT; String fileFullPath = FILE_PATH + fileName; AttributionDataShareHelper helper = helperInit(fileName, fileFullPath, logger); try { - helper.copyDataToFile(); + helper.copyDataToFile(DatabaseUtil.getConnection()); helper.writeFileToFinalDestination(getS3Client(ENDPOINT)); } catch (NullPointerException | URISyntaxException ex) { throwAttributionDataShareException(logger, ex); diff --git a/attribution-data-file-share/src/main/java/gov/cms/ab2d/attributionDataShare/AttributionDataShareHelper.java b/attribution-data-file-share/src/main/java/gov/cms/ab2d/attributionDataShare/AttributionDataShareHelper.java index 6056624..de0c31a 100644 --- a/attribution-data-file-share/src/main/java/gov/cms/ab2d/attributionDataShare/AttributionDataShareHelper.java +++ b/attribution-data-file-share/src/main/java/gov/cms/ab2d/attributionDataShare/AttributionDataShareHelper.java @@ -2,9 +2,6 @@ import com.amazonaws.services.lambda.runtime.LambdaLogger; import com.amazonaws.services.s3.model.AmazonS3Exception; -import gov.cms.ab2d.databasemanagement.DatabaseUtil; -import org.postgresql.copy.CopyManager; -import org.postgresql.core.BaseConnection; import software.amazon.awssdk.core.sync.RequestBody; import software.amazon.awssdk.services.s3.S3Client; import software.amazon.awssdk.services.s3.model.PutObjectRequest; @@ -13,29 +10,61 @@ import java.io.File; import java.io.FileWriter; import java.io.IOException; -import java.sql.SQLException; +import java.sql.*; +import java.text.SimpleDateFormat; +import java.util.Date; - -import static gov.cms.ab2d.attributionDataShare.AttributionDataShareHandlerConstants.*; +import static gov.cms.ab2d.attributionDataShare.AttributionDataShareConstants.*; public class AttributionDataShareHelper { LambdaLogger logger; String fileName; String fileFullPath; - public AttributionDataShareHelper(String fileName, String fileFullPath, LambdaLogger logger){ + + public AttributionDataShareHelper(String fileName, String fileFullPath, LambdaLogger logger) { this.fileName = fileName; this.fileFullPath = fileFullPath; this.logger = logger; } - void copyDataToFile() { - try (BufferedWriter writer = new BufferedWriter(new FileWriter(fileFullPath, true))) { - getCopyManager().copyOut(COPY_STATEMENT, writer); + + void copyDataToFile(Connection connection) { + String date = new SimpleDateFormat(EFFECTIVE_DATE_PATTERN).format(new Date()); + try (var stmt = connection.createStatement(); + var writer = new BufferedWriter(new FileWriter(fileFullPath, true))) { + var rs = getExecuteQuery(stmt); + writer.write(FIRST_LINE + date); + writer.newLine(); + long records = 0; + while (rs.next()) { + var line = getResponseLine(rs.getString(1), rs.getTimestamp(2), rs.getBoolean(3)); + writer.write(line); + writer.newLine(); + records++; + } + writer.write(LAST_LINE + date + String.format("%010d", records)); } catch (SQLException | IOException ex) { String errorMessage = "An error occurred while exporting data to a file. "; logger.log(errorMessage + ex.getMessage()); throw new AttributionDataShareException(errorMessage, ex); } } + + String getResponseLine(String currentMbi, Timestamp effectiveDate, boolean optOutFlag) { + var result = new StringBuilder(); + result.append(currentMbi); + // Adding spaces to the end of a string to achieve the required position index + if (currentMbi.length() < CURRENT_MBI_LENGTH) + result.append(" ".repeat(Math.max(0, CURRENT_MBI_LENGTH - currentMbi.length()))); + + if (effectiveDate == null) + result.append(" ".repeat(EFFECTIVE_DATE_LENGTH)); + else + result.append(new SimpleDateFormat(EFFECTIVE_DATE_PATTERN).format(effectiveDate)); + + result.append((optOutFlag) ? 'Y' : 'N'); + return result.toString(); + } + void writeFileToFinalDestination(S3Client s3Client) { try { var objectRequest = PutObjectRequest.builder() @@ -51,7 +80,8 @@ void writeFileToFinalDestination(S3Client s3Client) { } } - CopyManager getCopyManager() throws SQLException { - return new CopyManager((BaseConnection) DatabaseUtil.getConnection()); + static ResultSet getExecuteQuery(Statement statement) throws SQLException { + return statement.executeQuery(SELECT_STATEMENT); } + } diff --git a/attribution-data-file-share/src/test/java/gov/cms/ab2d/attributionDataShare/AttributionDataShareHandlerTest.java b/attribution-data-file-share/src/test/java/gov/cms/ab2d/attributionDataShare/AttributionDataShareHandlerTest.java index 418ba79..3dae584 100644 --- a/attribution-data-file-share/src/test/java/gov/cms/ab2d/attributionDataShare/AttributionDataShareHandlerTest.java +++ b/attribution-data-file-share/src/test/java/gov/cms/ab2d/attributionDataShare/AttributionDataShareHandlerTest.java @@ -3,16 +3,14 @@ import com.amazonaws.services.lambda.runtime.LambdaLogger; import gov.cms.ab2d.testutils.AB2DPostgresqlContainer; import gov.cms.ab2d.testutils.TestContext; -import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; -import org.junit.jupiter.api.extension.ExtendWith; import org.testcontainers.containers.PostgreSQLContainer; import org.testcontainers.junit.jupiter.Container; import org.testcontainers.junit.jupiter.Testcontainers; import java.net.URISyntaxException; -import static gov.cms.ab2d.attributionDataShare.AttributionDataShareHandlerConstants.TEST_ENDPOINT; +import static gov.cms.ab2d.attributionDataShare.AttributionDataShareConstants.TEST_ENDPOINT; import static org.junit.jupiter.api.Assertions.*; import static org.mockito.Mockito.*; diff --git a/attribution-data-file-share/src/test/java/gov/cms/ab2d/attributionDataShare/AttributionDataShareHelperTest.java b/attribution-data-file-share/src/test/java/gov/cms/ab2d/attributionDataShare/AttributionDataShareHelperTest.java index b9fcddb..fa476e0 100644 --- a/attribution-data-file-share/src/test/java/gov/cms/ab2d/attributionDataShare/AttributionDataShareHelperTest.java +++ b/attribution-data-file-share/src/test/java/gov/cms/ab2d/attributionDataShare/AttributionDataShareHelperTest.java @@ -1,13 +1,12 @@ package gov.cms.ab2d.attributionDataShare; import com.amazonaws.services.lambda.runtime.LambdaLogger; +import com.mockrunner.mock.jdbc.MockResultSet; import gov.cms.ab2d.lambdalibs.lib.FileUtil; import gov.cms.ab2d.testutils.AB2DPostgresqlContainer; -import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.api.extension.ExtendWith; -import org.postgresql.copy.CopyManager; import org.testcontainers.containers.PostgreSQLContainer; import org.testcontainers.junit.jupiter.Container; import org.testcontainers.junit.jupiter.Testcontainers; @@ -17,11 +16,13 @@ import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Paths; -import java.sql.SQLException; +import java.sql.*; import java.text.SimpleDateFormat; +import java.util.Collections; import java.util.Date; -import static gov.cms.ab2d.attributionDataShare.AttributionDataShareHandlerConstants.*; +import static gov.cms.ab2d.attributionDataShare.AttributionDataShareConstants.*; +import static gov.cms.ab2d.attributionDataShare.AttributionDataShareHelper.getExecuteQuery; import static org.junit.jupiter.api.Assertions.*; import static org.mockito.Mockito.*; @@ -35,6 +36,8 @@ public class AttributionDataShareHelperTest { String FILE_NAME = FILE_PARTIAL_NAME + new SimpleDateFormat(PATTERN).format(new Date()) + FILE_FORMAT; String FILE_FULL_PATH = FILE_PATH + FILE_NAME; + + String MBI = "1AA2BB3CC45"; AttributionDataShareHelper helper; @BeforeEach @@ -43,20 +46,30 @@ public void beforeEach() { } @Test - void copyDataToFileTest() throws SQLException, IOException { - CopyManager copyManager = mock(CopyManager.class); - when(helper.getCopyManager()).thenReturn(copyManager); - assertDoesNotThrow(() -> helper.copyDataToFile()); + void copyDataToFileTest() throws IOException, SQLException { + var connection = mock(Connection.class); + var stmt = mock(Statement.class); + var rs = new MockResultSet(""); + rs.addColumn("mbi", Collections.singletonList(MBI)); + rs.addColumn("effective_date", Collections.singletonList(null)); + rs.addColumn("opt_out_flag", Collections.singletonList(true)); + when(connection.createStatement()).thenReturn(stmt); + + when(getExecuteQuery(stmt)).thenReturn(rs); + assertDoesNotThrow(() -> helper.copyDataToFile(connection)); assertTrue(Files.exists(Paths.get(FILE_FULL_PATH))); FileUtil.deleteDirectoryRecursion(Paths.get(FILE_FULL_PATH)); } @Test - void copyDataToFileExceptionTest() { - assertThrows(AttributionDataShareException.class, () -> helper.copyDataToFile()); + void getResponseLineTest(){ + assertEquals(MBI +" Y", helper.getResponseLine(MBI, null, true)); + assertEquals(MBI +"20240226N", helper.getResponseLine(MBI, Timestamp.valueOf("2024-02-26 00:00:00"), false)); + assertEquals("A Y", helper.getResponseLine("A", null, true)); } + @Test void writeFileToFinalDestinationTest() throws IOException { createTestFile(); @@ -65,16 +78,9 @@ void writeFileToFinalDestinationTest() throws IOException { S3MockAPIExtension.deleteFile(FILE_NAME); } - @Test - void getCopyManagerTest() throws SQLException { - Assertions.assertNotNull(helper.getCopyManager()); - } - private void createTestFile() throws IOException { PrintWriter writer = new PrintWriter(FILE_FULL_PATH, StandardCharsets.UTF_8); writer.println("Test"); writer.close(); } - - } diff --git a/attribution-data-file-share/src/test/java/gov/cms/ab2d/attributionDataShare/S3MockAPIExtension.java b/attribution-data-file-share/src/test/java/gov/cms/ab2d/attributionDataShare/S3MockAPIExtension.java index 7c92eb3..9272823 100644 --- a/attribution-data-file-share/src/test/java/gov/cms/ab2d/attributionDataShare/S3MockAPIExtension.java +++ b/attribution-data-file-share/src/test/java/gov/cms/ab2d/attributionDataShare/S3MockAPIExtension.java @@ -8,7 +8,7 @@ import java.net.URI; -import static gov.cms.ab2d.attributionDataShare.AttributionDataShareHandlerConstants.*; +import static gov.cms.ab2d.attributionDataShare.AttributionDataShareConstants.*; import static org.junit.jupiter.api.extension.ExtensionContext.Namespace.GLOBAL; public class S3MockAPIExtension implements BeforeAllCallback, ExtensionContext.Store.CloseableResource {