Skip to content

Commit

Permalink
wip
Browse files Browse the repository at this point in the history
  • Loading branch information
@smirnovaae
smirnovaae committed Mar 8, 2024
1 parent d7bc30d commit 6f848e2
Show file tree
Hide file tree
Showing 10 changed files with 108 additions and 79 deletions.
2 changes: 1 addition & 1 deletion ...-share/src/main/java/gov/cms/ab2d/attributionDataShare/AttributionDataShareConstants.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ public class AttributionDataShareConstants {
private AttributionDataShareConstants() {
}

public static final String BFD_S3_BUCKET_NAME = "ab2d-opt-out-temp-349849222861-us-east-1";
// public static final String BFD_S3_BUCKET_NAME = "ab2d-opt-out-temp-349849222861-us-east-1";
public static final String ENDPOINT = "https://s3.amazonaws.com";
public static final String TEST_ENDPOINT = "http://127.0.0.1:8001";
public static final Region S3_REGION = Region.US_EAST_1;
Expand Down
13 changes: 11 additions & 2 deletions ...ile-share/src/main/java/gov/cms/ab2d/attributionDataShare/AttributionDataShareHelper.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -68,8 +68,8 @@ String getResponseLine(String currentMbi, Timestamp effectiveDate, boolean optOu
void writeFileToFinalDestination(S3Client s3Client) {
try {
var objectRequest = PutObjectRequest.builder()
.bucket(BFD_S3_BUCKET_NAME)
.key(fileName)
.bucket(getBucketName())
.key(getUploadPath() + fileName)
.build();

s3Client.putObject(objectRequest, RequestBody.fromFile(new File(fileFullPath)));
Expand All @@ -80,6 +80,15 @@ void writeFileToFinalDestination(S3Client s3Client) {
}
}

public String getBucketName(){
return System.getenv("S3_UPLOAD_BUCKET");
}

public String getUploadPath(){
return System.getenv("S3_UPLOAD_PATH") + "/";
}


static ResultSet getExecuteQuery(Statement statement) throws SQLException {
return statement.executeQuery(SELECT_STATEMENT);
}
Expand Down
14 changes: 14 additions & 0 deletions ...-file-share/src/test/java/gov/cms/ab2d/attributionDataShare/AttributionDataShareTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,8 @@

import static gov.cms.ab2d.attributionDataShare.AttributionDataShareConstants.*;
import static gov.cms.ab2d.attributionDataShare.AttributionDataShareHelper.getExecuteQuery;
import static gov.cms.ab2d.attributionDataShare.S3MockAPIExtension.BUCKET_NAME;
import static gov.cms.ab2d.attributionDataShare.S3MockAPIExtension.UPLOAD_PATH;
import static org.junit.jupiter.api.Assertions.*;
import static org.mockito.Mockito.*;

Expand Down Expand Up @@ -78,6 +80,18 @@ void writeFileToFinalDestinationTest() throws IOException {
S3MockAPIExtension.deleteFile(FILE_NAME);
}

@Test
void getBucketNameTest(){
assertEquals(BUCKET_NAME, helper.getBucketName());
}

@Test
void getUploadPathTest(){
assertEquals(UPLOAD_PATH, helper.getUploadPath());
}



private void createTestFile() throws IOException {
PrintWriter writer = new PrintWriter(FILE_FULL_PATH, StandardCharsets.UTF_8);
writer.println("Test");
Expand Down
18 changes: 12 additions & 6 deletions ...n-data-file-share/src/test/java/gov/cms/ab2d/attributionDataShare/S3MockAPIExtension.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,10 @@
import static org.junit.jupiter.api.extension.ExtensionContext.Namespace.GLOBAL;

public class S3MockAPIExtension implements BeforeAllCallback, ExtensionContext.Store.CloseableResource {

private static final S3Mock API = S3Mock.create(8001, "/tmp/s3");
public static String BUCKET_NAME = "bfd-test-eft";
public static String UPLOAD_PATH = "bfdeft01/test/out";
public static S3Client S3_CLIENT;
private static boolean STARTED = false;

Expand All @@ -29,6 +32,9 @@ public void beforeAll(ExtensionContext context) throws Exception {
.endpointOverride(new URI(TEST_ENDPOINT))
.build();

System.setProperty("S3_UPLOAD_BUCKET", BUCKET_NAME);
System.setProperty("S3_UPLOAD_PATH", UPLOAD_PATH);

createBucket();
}
}
Expand All @@ -42,7 +48,7 @@ public void close() {

private static void createBucket() {
var bucketRequest = CreateBucketRequest.builder()
.bucket(BFD_S3_BUCKET_NAME)
.bucket(BUCKET_NAME)
.build();

S3_CLIENT.createBucket(bucketRequest);
Expand All @@ -51,8 +57,8 @@ private static void createBucket() {
public static boolean isObjectExists(String fileName) {
try {
HeadObjectRequest headObjectRequest = HeadObjectRequest.builder()
.bucket(BFD_S3_BUCKET_NAME)
.key(fileName)
.bucket(BUCKET_NAME)
.key(UPLOAD_PATH + "/" + fileName)
.build();

S3_CLIENT.headObject(headObjectRequest);
Expand All @@ -69,16 +75,16 @@ public static boolean isObjectExists(String fileName) {
public static void deleteFile(String fileName) {
if (isObjectExists(fileName)) {
DeleteObjectRequest deleteObjectRequest = DeleteObjectRequest.builder()
.bucket(BFD_S3_BUCKET_NAME)
.key(fileName)
.bucket(BUCKET_NAME)
.key(UPLOAD_PATH + "/" + fileName)
.build();
S3_CLIENT.deleteObject(deleteObjectRequest);
}
}

private static void deleteBucket() {
DeleteBucketRequest deleteBucketRequest = DeleteBucketRequest.builder()
.bucket(BFD_S3_BUCKET_NAME)
.bucket(BUCKET_NAME)
.build();

S3_CLIENT.deleteBucket(deleteBucketRequest);
Expand Down
1 change: 1 addition & 0 deletions optout/build.gradle
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ dependencies {
implementation 'org.postgresql:postgresql:42.5.1'
implementation 'software.amazon.awssdk:s3:2.21.7'
implementation 'software.amazon.awssdk:secretsmanager:2.23.12'
implementation 'com.googlecode.json-simple:json-simple:1.1.1'
implementation project(':database-management')
implementation(project(":lambda-lib"))
testImplementation 'com.amazonaws:aws-java-sdk-sqs:1.11.792'
Expand Down
47 changes: 26 additions & 21 deletions optout/src/main/java/gov/cms/ab2d/optout/OptOutHandler.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,41 +3,46 @@
import com.amazonaws.services.lambda.runtime.Context;
import com.amazonaws.services.lambda.runtime.LambdaLogger;
import com.amazonaws.services.lambda.runtime.RequestHandler;
import com.amazonaws.services.lambda.runtime.events.S3Event;
import com.amazonaws.services.lambda.runtime.events.SQSEvent;
import com.amazonaws.services.s3.event.S3EventNotification;
import org.json.simple.JSONObject;
import org.json.simple.parser.JSONParser;

import java.net.URISyntaxException;

import static gov.cms.ab2d.optout.OptOutConstants.ENDPOINT;

public class OptOutHandler implements RequestHandler<S3Event, Void> {
public class OptOutHandler implements RequestHandler<SQSEvent, Void> {

@Override
public Void handleRequest(S3Event s3event, Context context) {
var logger = context.getLogger();
public Void handleRequest(SQSEvent sqsEvent, Context context) {
for (SQSEvent.SQSMessage msg : sqsEvent.getRecords()) {
processSQSMessage(msg, context);
}
context.getLogger().log("OptOut Lambda completed");
return null;
}

logger.log(s3event.toString());
logger.log("OptOut Lambda started");
public void processSQSMessage(SQSEvent.SQSMessage msg, Context context) {
var logger = context.getLogger();
try {
logger.log("OptOut Lambda started. Processing message from SQS " + msg.getBody());

var records = s3event.getRecords();//.get(0);
var srcBucket = getBucketName(s3event);
var srcKey = getFileName(s3event);
JSONParser parser = new JSONParser();
JSONObject json = (JSONObject) parser.parse(msg.getBody());
var s3EventMessage = json.get("Message");
var notification = S3EventNotification.parseJson(s3EventMessage.toString()).getRecords().get(0);

logger.log("bucket: " + srcBucket);
logger.log("key: " + srcKey);
try {
var optOutProcessing = processorInit(srcKey, srcBucket, logger);
var optOutProcessing = processorInit(getFileName(notification), getBucketName(notification), logger);
optOutProcessing.process();
} catch (Exception ex) {
logger.log("An error occurred");
throw new OptOutException("An error occurred", ex);
}
logger.log("OptOut Lambda completed");
return null;
}

public OptOutProcessor processorInit(String fileName, String bucketName, LambdaLogger logger) throws URISyntaxException {
return new OptOutProcessor(fileName, bucketName, ENDPOINT, logger);
public OptOutProcessor processorInit(String fileName, String bfdBucket, LambdaLogger logger) throws URISyntaxException {
return new OptOutProcessor(fileName, bfdBucket, ENDPOINT, logger);
//ToDo: uncomment when permanent credentials will be available
// var creds = SecretManager.getS3Credentials(ACCESS_KEY_ID, SECRET_ACCESS_KEY, ACCESS_TOKEN, logger);
// if (creds.isPresent())
Expand All @@ -48,11 +53,11 @@ public OptOutProcessor processorInit(String fileName, String bucketName, LambdaL
// }
}

public String getBucketName(S3Event s3event){
return s3event.getRecords().get(0).getS3().getBucket().getName();
public String getBucketName(S3EventNotification.S3EventNotificationRecord record) {
return record.getS3().getBucket().getName();
}

public String getFileName(S3Event s3event){
return s3event.getRecords().get(0).getS3().getObject().getUrlDecodedKey();
public String getFileName(S3EventNotification.S3EventNotificationRecord record) {
return record.getS3().getObject().getUrlDecodedKey();
}
}
2 changes: 1 addition & 1 deletion optout/src/main/java/gov/cms/ab2d/optout/OptOutProcessor.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -106,7 +106,7 @@ public String getRecordStatus() {
return (isRejected) ? RecordStatus.REJECTED.toString() : RecordStatus.ACCEPTED.toString();
}

public String getEffectiveDate(String date){
public String getEffectiveDate(String date) {
return (isRejected) ? " ".repeat(EFFECTIVE_DATE_LENGTH) : date;
}

Expand Down
60 changes: 30 additions & 30 deletions optout/src/test/java/gov/cms/ab2d/optout/OptOutHandlerTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,10 +2,13 @@

import com.amazonaws.services.lambda.runtime.Context;
import com.amazonaws.services.lambda.runtime.LambdaLogger;
import com.amazonaws.services.lambda.runtime.events.S3Event;
import com.amazonaws.services.lambda.runtime.events.SQSEvent;
import com.amazonaws.services.s3.event.S3EventNotification;
import com.amazonaws.services.s3.model.AmazonS3Exception;
import gov.cms.ab2d.testutils.AB2DPostgresqlContainer;
import org.json.simple.JSONObject;
import org.json.simple.parser.JSONParser;
import org.json.simple.parser.ParseException;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Test;
import org.testcontainers.containers.PostgreSQLContainer;
Expand All @@ -18,8 +21,7 @@
import java.nio.file.Paths;
import java.util.Collections;

import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.*;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.Mockito.*;

Expand All @@ -31,46 +33,44 @@ public class OptOutHandlerTest {
private static final PostgreSQLContainer POSTGRE_SQL_CONTAINER = new AB2DPostgresqlContainer();
private final static OptOutHandler handler = spy(new OptOutHandler());
private final static OptOutProcessor OPT_OUT_PROCESSOR = mock(OptOutProcessor.class);
private final static S3Event s3event = mock(S3Event.class);

static String srcBucket;
static String srcKey;
private final static SQSEvent sqsEvent = mock(SQSEvent.class);
private final static SQSEvent.SQSMessage sqsMessage = mock(SQSEvent.SQSMessage.class);

@BeforeAll
static void beforeAll() throws URISyntaxException, IOException {
var payload = Files.readString(Paths.get("src/test/resources/1.json"));
S3EventNotification records = S3EventNotification.parseJson(payload);
var record = records.getRecords().get(0);
srcBucket = record.getS3().getBucket().getName();
srcKey = record.getS3().getObject().getUrlDecodedKey();
when(s3event.getRecords()).thenReturn(records.getRecords());
when(handler.processorInit(eq(srcBucket), eq(srcKey), any(LambdaLogger.class))).thenReturn(OPT_OUT_PROCESSOR);
when(sqsEvent.getRecords()).thenReturn(Collections.singletonList(sqsMessage));
when(sqsMessage.getBody()).thenReturn(getPayload());
when(handler.processorInit(anyString(), anyString(), any(LambdaLogger.class))).thenReturn(OPT_OUT_PROCESSOR);
}

@Test
void getBucketNameTest() throws IOException {
var payload = Files.readString(Paths.get("src/test/resources/1.json"));
var notification = S3EventNotification.parseJson(payload);
void getTest() throws IOException, ParseException {
JSONParser parser = new JSONParser();
JSONObject json = (JSONObject) parser.parse(getPayload());
var s3EventMessage = json.get("Message");

S3Event s3event = new S3Event(notification.getRecords());
var a = handler.getFileName(s3event);
var b = handler.getBucketName(s3event);
var s = "";
var notification = S3EventNotification.parseJson(s3EventMessage.toString()).getRecords().get(0);

assertEquals("bfdeft01/ab2d/in/optOutDummy.txt", handler.getFileName(notification));
assertEquals("bfd-test-eft", handler.getBucketName(notification));
}

@Test
void optOutHandlerInvoke() {
Context context = mock(Context.class);
LambdaLogger logger = mock(LambdaLogger.class);
when(context.getLogger()).thenReturn(logger);

assertDoesNotThrow(() -> handler.handleRequest(sqsEvent, context));
}
//
// @Test
// void optOutHandlerInvoke() {
// Context context = mock(Context.class);
// LambdaLogger logger = mock(LambdaLogger.class);
// when(context.getLogger()).thenReturn(logger);
//
// assertDoesNotThrow(() -> handler.handleRequest(s3event, context));
// }

@Test
void optOutHandlerException() {
doThrow(new OptOutException("errorMessage", new AmazonS3Exception("errorMessage"))).when(OPT_OUT_PROCESSOR).process();
assertThrows(OptOutException.class, OPT_OUT_PROCESSOR::process);
}

static private String getPayload() throws IOException {
return Files.readString(Paths.get("src/test/resources/sqsEvent.json"));
}
}
18 changes: 0 additions & 18 deletions optout/src/test/resources/s3event.json
View file

This file was deleted.

12 changes: 12 additions & 0 deletions optout/src/test/resources/sqsEvent.json
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
{
"Type" : "Notification",
"MessageId" : "5da6e965-ca05-5123-8347-b72ad7893428",
"TopicArn" : "arn:aws:sns:us-east-1:577373831711:bfd-test-eft-inbound-received-s3-dpc",
"Subject" : "Amazon S3 Notification",
"Message" : "{\"Records\":[{\"eventVersion\":\"2.1\",\"eventSource\":\"aws:s3\",\"awsRegion\":\"us-east-1\",\"eventTime\":\"2024-02-20T21:36:11.419Z\",\"eventName\":\"ObjectCreated:Put\",\"userIdentity\":{\"principalId\":\"AWS:AIDAYM3RJQIPTYMO35QPN\"},\"requestParameters\":{\"sourceIPAddress\":\"66.61.78.110\"},\"responseElements\":{\"x-amz-request-id\":\"JM2GMJJNSSEH9PKY\",\"x-amz-id-2\":\"gO4Fl4AxgEJ8LYteQ6LW4gHNtmB9+CcRDOnE20HW9ZumAQTsP+HG1DKDYLxQa67VCG6XIyIfSsZOW8cOfPuOQAD3+7HfBs/3\"},\"s3\":{\"s3SchemaVersion\":\"1.0\",\"configurationId\":\"bfd-test-eft-inbound-received-s3-dpc\",\"bucket\":{\"name\":\"bfd-test-eft\",\"ownerIdentity\":{\"principalId\":\"A5VBSMJFI0FCE\"},\"arn\":\"arn:aws:s3:::bfd-test-eft\"},\"object\":{\"key\":\"bfdeft01/ab2d/in/optOutDummy.txt\",\"size\":12,\"eTag\":\"fdad692379d13b4c029804195d0a4aa5\",\"sequencer\":\"0065D51B4B4098E68C\"}}}]}",
"Timestamp" : "2024-02-20T21:36:12.405Z",
"SignatureVersion" : "1",
"Signature" : "geHsbUYOSZ6HZLDo0cVnMsmbrsncnGNubwLHZfcSmAtIpm4ciCeR3tkoZeZhWySXFW8N1a9URlgo1WIy0Cf08yspRG0HCtdYJCVv0hJtxiHpS2M3qyDJY0o3hsX4Uk29PcczjUAu89/j7niPRID/zh0LQcUWkA9UPaHDrxAJR69WDV5EvPLOBl4B2gNA/zUGgIlQL7jXhYiGnvRhldNUD2ZibY+wSrlYEs/b+U/dJsid1bvzT30Dw6nwpEB6Xqpw8KH6DX32E2/XDvKOqELd5aAv1P/+uLMhohzD/DSmRd31pZUECF99/GLdT2b41/M1L1KQm31r7uF9Lvy4DrG8CQ==",
"SigningCertURL" : "https://sns.us-east-1.amazonaws.com/SimpleNotificationService-60eadc530605d63b8e62a523676ef735.pem",
"UnsubscribeURL" : "https://sns.us-east-1.amazonaws.com/?Action=Unsubscribe&SubscriptionArn=arn:aws:sns:us-east-1:577373831711:bfd-test-eft-inbound-received-s3-dpc:f14e07ab-36d0-48ec-bdcd-41b0be370fc8"
}

0 comments on commit 6f848e2

Please sign in to comment.