Tests: S3 settings functional tests (#2837)

pom.xml

@@ -332,6 +332,11 @@
             <artifactId>mysql</artifactId>
             <scope>test</scope>
         </dependency>
+        <dependency>
+            <groupId>org.testcontainers</groupId>
+            <artifactId>localstack</artifactId>
+            <scope>test</scope>
+        </dependency>
         <dependency>
             <groupId>org.testcontainers</groupId>
             <artifactId>postgresql</artifactId>

src/test/groovy/org/prebid/server/functional/service/S3Service.groovy

@@ -0,0 +1,103 @@
+package org.prebid.server.functional.service
+
+import org.prebid.server.functional.model.config.AccountConfig
+import org.prebid.server.functional.model.db.StoredImp
+import org.prebid.server.functional.model.db.StoredRequest
+import org.prebid.server.functional.model.db.StoredResponse
+import org.prebid.server.functional.util.ObjectMapperWrapper
+import org.testcontainers.containers.localstack.LocalStackContainer
+import software.amazon.awssdk.auth.credentials.AwsBasicCredentials
+import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider
+import software.amazon.awssdk.core.sync.RequestBody
+import software.amazon.awssdk.regions.Region
+import software.amazon.awssdk.services.s3.S3Client
+import software.amazon.awssdk.services.s3.model.CreateBucketRequest
+import software.amazon.awssdk.services.s3.model.DeleteBucketRequest
+import software.amazon.awssdk.services.s3.model.DeleteObjectRequest
+import software.amazon.awssdk.services.s3.model.ListObjectsV2Request
+import software.amazon.awssdk.services.s3.model.PutObjectRequest
+import software.amazon.awssdk.services.s3.model.PutObjectResponse
+
+final class S3Service implements ObjectMapperWrapper {
+
+    private final S3Client s3PbsService
+    private final LocalStackContainer localStackContainer
+
+    static final def DEFAULT_ACCOUNT_DIR = 'account'
+    static final def DEFAULT_IMPS_DIR = 'stored-impressions'
+    static final def DEFAULT_REQUEST_DIR = 'stored-requests'
+    static final def DEFAULT_RESPONSE_DIR = 'stored-responses'
+
+    S3Service(LocalStackContainer localStackContainer) {
+        this.localStackContainer = localStackContainer
+        s3PbsService = S3Client.builder()
+                .endpointOverride(localStackContainer.getEndpointOverride(LocalStackContainer.Service.S3))
+                .credentialsProvider(
+                        StaticCredentialsProvider.create(
+                                AwsBasicCredentials.create(
+                                        localStackContainer.getAccessKey(),
+                                        localStackContainer.getSecretKey())))
+                .region(Region.of(localStackContainer.getRegion()))
+                .build()
+    }
+
+    String getAccessKeyId() {
+        localStackContainer.accessKey
+    }
+
+    String getSecretKeyId() {
+        localStackContainer.secretKey
+    }
+
+    String getEndpoint() {
+        "http://${localStackContainer.getNetworkAliases().get(0)}:${localStackContainer.getExposedPorts().get(0)}"
+    }
+
+    String getRegion() {
+        localStackContainer.region
+    }
+
+    void createBucket(String bucketName) {
+        CreateBucketRequest createBucketRequest = CreateBucketRequest.builder()
+                .bucket(bucketName)
+                .build()
+        s3PbsService.createBucket(createBucketRequest)
+    }
+
+    void deleteBucket(String bucketName) {
+        DeleteBucketRequest deleteBucketRequest = DeleteBucketRequest.builder()
+                .bucket(bucketName)
+                .build()
+        s3PbsService.deleteBucket(deleteBucketRequest)
+    }
+
+    void purgeBucketFiles(String bucketName) {
+        s3PbsService.listObjectsV2(ListObjectsV2Request.builder().bucket(bucketName).build()).contents().each { files ->
+            s3PbsService.deleteObject(DeleteObjectRequest.builder().bucket(bucketName).key(files.key()).build())
+        }
+    }
+
+    PutObjectResponse uploadAccount(String bucketName, AccountConfig account, String fileName = account.id) {
+        uploadFile(bucketName, encode(account), "${DEFAULT_ACCOUNT_DIR}/${fileName}.json")
+    }
+
+    PutObjectResponse uploadStoredRequest(String bucketName, StoredRequest storedRequest, String fileName = storedRequest.requestId) {
+        uploadFile(bucketName, encode(storedRequest.requestData), "${DEFAULT_REQUEST_DIR}/${fileName}.json")
+    }
+
+    PutObjectResponse uploadStoredResponse(String bucketName, StoredResponse storedRequest, String fileName = storedRequest.responseId) {
+        uploadFile(bucketName, encode(storedRequest.storedAuctionResponse), "${DEFAULT_RESPONSE_DIR}/${fileName}.json")
+    }
+
+    PutObjectResponse uploadStoredImp(String bucketName, StoredImp storedImp, String fileName = storedImp.impId) {
+        uploadFile(bucketName, encode(storedImp.impData), "${DEFAULT_IMPS_DIR}/${fileName}.json")
+    }
+
+    PutObjectResponse uploadFile(String bucketName, String fileBody, String path) {
+        PutObjectRequest putObjectRequest = PutObjectRequest.builder()
+                .bucket(bucketName)
+                .key(path)
+                .build()
+        s3PbsService.putObject(putObjectRequest, RequestBody.fromString(fileBody))
+    }
+}

src/test/groovy/org/prebid/server/functional/testcontainers/Dependencies.groovy

@@ -4,8 +4,10 @@ import org.prebid.server.functional.testcontainers.container.NetworkServiceConta
 import org.prebid.server.functional.util.SystemProperties
 import org.testcontainers.containers.MySQLContainer
 import org.testcontainers.containers.Network
+import org.testcontainers.containers.localstack.LocalStackContainer
 import org.testcontainers.containers.PostgreSQLContainer
 import org.testcontainers.lifecycle.Startables
+import org.testcontainers.utility.DockerImageName
 
 import static org.prebid.server.functional.util.SystemProperties.MOCKSERVER_VERSION
 
@@ -34,16 +36,20 @@ class Dependencies {
     static final NetworkServiceContainer networkServiceContainer = new NetworkServiceContainer(MOCKSERVER_VERSION)
             .withNetwork(network)
 
+    static final LocalStackContainer localStackContainer = new LocalStackContainer(DockerImageName.parse("localstack/localstack:s3-latest"))
+            .withNetwork(Dependencies.network)
+            .withServices(LocalStackContainer.Service.S3)
+
     static void start() {
         if (IS_LAUNCH_CONTAINERS) {
-            Startables.deepStart([networkServiceContainer, mysqlContainer])
+            Startables.deepStart([networkServiceContainer, mysqlContainer, localStackContainer])
                       .join()
         }
     }
 
     static void stop() {
         if (IS_LAUNCH_CONTAINERS) {
-            [networkServiceContainer, mysqlContainer].parallelStream()
+            [networkServiceContainer, mysqlContainer, localStackContainer].parallelStream()
                                                      .forEach({ it.stop() })
         }
     }

src/test/groovy/org/prebid/server/functional/tests/storage/AccountS3Spec.groovy

@@ -0,0 +1,118 @@
+package org.prebid.server.functional.tests.storage
+
+import org.prebid.server.functional.model.AccountStatus
+import org.prebid.server.functional.model.config.AccountConfig
+import org.prebid.server.functional.model.request.auction.BidRequest
+import org.prebid.server.functional.service.PrebidServerException
+import org.prebid.server.functional.service.PrebidServerService
+import org.prebid.server.functional.service.S3Service
+import org.prebid.server.functional.testcontainers.PbsServiceFactory
+import org.prebid.server.functional.util.PBSUtils
+
+import static io.netty.handler.codec.http.HttpResponseStatus.UNAUTHORIZED
+
+class AccountS3Spec extends StorageBaseSpec {
+
+    protected PrebidServerService s3StorageAccountPbsService = PbsServiceFactory.getService(s3StorageConfig +
+                                                                                            mySqlDisabledConfig +
+                                                                                            ['settings.enforce-valid-account': 'true'])
+
+    def "PBS should process request when active account is present in S3 storage"() {
+        given: "Default BidRequest with account"
+        def accountId = PBSUtils.randomNumber as String
+        def bidRequest = BidRequest.defaultBidRequest.tap {
+            setAccountId(accountId)
+        }
+
+        and: "Active account config"
+        def account = new AccountConfig(id: accountId, status: AccountStatus.ACTIVE)
+
+        and: "Saved account in AWS S3 storage"
+        s3Service.uploadAccount(DEFAULT_BUCKET, account)
+
+        when: "PBS processes auction request"
+        def response = s3StorageAccountPbsService.sendAuctionRequest(bidRequest)
+
+        then: "Response should contain seatbid"
+        assert response.seatbid.size() == 1
+    }
+
+    def "PBS should throw exception when inactive account is present in S3 storage"() {
+        given: "Default BidRequest with account"
+        def accountId = PBSUtils.randomNumber as String
+        def bidRequest = BidRequest.defaultBidRequest.tap {
+            setAccountId(accountId)
+        }
+
+        and: "Inactive account config"
+        def account = new AccountConfig(id: accountId, status: AccountStatus.INACTIVE)
+
+        and: "Saved account in AWS S3 storage"
+        s3Service.uploadAccount(DEFAULT_BUCKET, account)
+
+        when: "PBS processes auction request"
+        s3StorageAccountPbsService.sendAuctionRequest(bidRequest)
+
+        then: "PBS should reject the entire auction"
+        def exception = thrown(PrebidServerException)
+        assert exception.statusCode == UNAUTHORIZED.code()
+        assert exception.responseBody == "Account $accountId is inactive"
+    }
+
+    def "PBS should throw exception when account id isn't match with bid request account id"() {
+        given: "Default BidRequest with account"
+        def accountId = PBSUtils.randomNumber as String
+        def bidRequest = BidRequest.defaultBidRequest.tap {
+            setAccountId(accountId)
+        }
+
+        and: "Account config with different accountId"
+        def account = new AccountConfig(id: PBSUtils.randomString, status: AccountStatus.ACTIVE)
+
+        and: "Saved account in AWS S3 storage"
+        s3Service.uploadAccount(DEFAULT_BUCKET, account, accountId)
+
+        when: "PBS processes auction request"
+        s3StorageAccountPbsService.sendAuctionRequest(bidRequest)
+
+        then: "PBS should reject the entire auction"
+        def exception = thrown(PrebidServerException)
+        assert exception.statusCode == UNAUTHORIZED.code()
+        assert exception.responseBody == "Unauthorized account id: ${accountId}"
+    }
+
+    def "PBS should throw exception when account is invalid in S3 storage json file"() {
+        given: "Default BidRequest"
+        def accountId = PBSUtils.randomNumber as String
+        def bidRequest = BidRequest.defaultBidRequest.tap {
+            setAccountId(accountId)
+        }
+
+        and: "Saved invalid account in AWS S3 storage"
+        s3Service.uploadFile(DEFAULT_BUCKET, INVALID_FILE_BODY, "${S3Service.DEFAULT_ACCOUNT_DIR}/${accountId}.json")
+
+        when: "PBS processes auction request"
+        s3StorageAccountPbsService.sendAuctionRequest(bidRequest)
+
+        then: "PBS should reject the entire auction"
+        def exception = thrown(PrebidServerException)
+        assert exception.statusCode == UNAUTHORIZED.code()
+        assert exception.responseBody == "Unauthorized account id: ${accountId}"
+    }
+
+    def "PBS should throw exception when account is not present in S3 storage and valid account enforced"() {
+        given: "Default BidRequest"
+        def accountId = PBSUtils.randomNumber as String
+        def bidRequest = BidRequest.defaultBidRequest.tap {
+            setAccountId(accountId)
+        }
+
+        when: "PBS processes auction request"
+        s3StorageAccountPbsService.sendAuctionRequest(bidRequest)
+
+        then: "PBS should reject the entire auction"
+        def exception = thrown(PrebidServerException)
+        assert exception.statusCode == UNAUTHORIZED.code()
+        assert exception.responseBody == "Unauthorized account id: ${accountId}"
+    }
+}

src/test/groovy/org/prebid/server/functional/tests/storage/AmpS3Spec.groovy

@@ -0,0 +1,115 @@
+package org.prebid.server.functional.tests.storage
+
+import org.prebid.server.functional.model.db.StoredRequest
+import org.prebid.server.functional.model.request.amp.AmpRequest
+import org.prebid.server.functional.model.request.auction.BidRequest
+import org.prebid.server.functional.model.request.auction.Site
+import org.prebid.server.functional.service.PrebidServerException
+import org.prebid.server.functional.service.S3Service
+import org.prebid.server.functional.util.PBSUtils
+import spock.lang.PendingFeature
+
+import static io.netty.handler.codec.http.HttpResponseStatus.BAD_REQUEST
+
+class AmpS3Spec extends StorageBaseSpec {
+
+    def "PBS should take parameters from the stored request on S3 service when it's not specified in the request"() {
+        given: "AMP request"
+        def ampRequest = new AmpRequest(tagId: PBSUtils.randomString).tap {
+            account = PBSUtils.randomNumber as String
+        }
+
+        and: "Default stored request"
+        def ampStoredRequest = BidRequest.defaultStoredRequest.tap {
+            site = Site.defaultSite
+            setAccountId(ampRequest.account)
+        }
+
+        and: "Stored request in S3 service"
+        def storedRequest = StoredRequest.getStoredRequest(ampRequest, ampStoredRequest)
+        s3Service.uploadStoredRequest(DEFAULT_BUCKET, storedRequest)
+
+        when: "PBS processes amp request"
+        s3StoragePbsService.sendAmpRequest(ampRequest)
+
+        then: "Bidder request should contain parameters from the stored request"
+        def bidderRequest = bidder.getBidderRequest(ampStoredRequest.id)
+
+        assert bidderRequest.site?.page == ampStoredRequest.site.page
+        assert bidderRequest.site?.publisher?.id == ampStoredRequest.site.publisher.id
+        assert !bidderRequest.imp[0]?.tagId
+        assert bidderRequest.imp[0]?.banner?.format[0]?.height == ampStoredRequest.imp[0].banner.format[0].height
+        assert bidderRequest.imp[0]?.banner?.format[0]?.weight == ampStoredRequest.imp[0].banner.format[0].weight
+        assert bidderRequest.regs?.gdpr == ampStoredRequest.regs.gdpr
+    }
+
+    @PendingFeature
+    def "PBS should throw exception when trying to take parameters from the stored request on S3 service with invalid id in file"() {
+        given: "AMP request"
+        def ampRequest = new AmpRequest(tagId: PBSUtils.randomString).tap {
+            account = PBSUtils.randomNumber as String
+        }
+
+        and: "Default stored request"
+        def ampStoredRequest = BidRequest.defaultStoredRequest.tap {
+            site = Site.defaultSite
+            setAccountId(ampRequest.account)
+        }
+
+        and: "Stored request in S3 service"
+        def storedRequest = StoredRequest.getStoredRequest(ampRequest, ampStoredRequest).tap {
+            it.requestId = PBSUtils.randomNumber
+        }
+        s3Service.uploadStoredRequest(DEFAULT_BUCKET, storedRequest, ampRequest.tagId)
+
+        when: "PBS processes amp request"
+        s3StoragePbsService.sendAmpRequest(ampRequest)
+
+        then: "PBS should throw request format error"
+        def exception = thrown(PrebidServerException)
+        assert exception.statusCode == BAD_REQUEST.code()
+        assert exception.responseBody == "Invalid request format: Stored request processing failed: " +
+                "No stored request found for id: ${ampRequest.tagId}"
+    }
+
+    def "PBS should throw exception when trying to take parameters from request where id isn't match with stored request id"() {
+        given: "AMP request"
+        def ampRequest = new AmpRequest(tagId: PBSUtils.randomString).tap {
+            account = PBSUtils.randomNumber as String
+        }
+
+        and: "Default stored request"
+        def ampStoredRequest = BidRequest.defaultStoredRequest.tap {
+            site = Site.defaultSite
+            setAccountId(ampRequest.account)
+        }
+
+        and: "Stored request in S3 service"
+        s3Service.uploadFile(DEFAULT_BUCKET, INVALID_FILE_BODY, "${S3Service.DEFAULT_REQUEST_DIR}/${ampRequest.tagId}.json")
+
+        when: "PBS processes amp request"
+        s3StoragePbsService.sendAmpRequest(ampRequest)
+
+        then: "PBS should throw request format error"
+        def exception = thrown(PrebidServerException)
+        assert exception.statusCode == BAD_REQUEST.code()
+        assert exception.responseBody == "Invalid request format: Stored request processing failed: " +
+                "Can't parse Json for stored request with id ${ampRequest.tagId}"
+    }
+
+    def "PBS should throw an exception when trying to take parameters from stored request on S3 service that do not exist"() {
+        given: "AMP request"
+        def ampRequest = new AmpRequest(tagId: PBSUtils.randomString).tap {
+            account = PBSUtils.randomNumber as String
+        }
+
+        when: "PBS processes amp request"
+        s3StoragePbsService.sendAmpRequest(ampRequest)
+
+        then: "PBS should throw request format error"
+        def exception = thrown(PrebidServerException)
+        assert exception.statusCode == BAD_REQUEST.code()
+        assert exception.responseBody == "Invalid request format: Stored request processing failed: " +
+                "No stored request found for id: ${ampRequest.tagId}"
+    }
+}