Core: S3 application settings (prebid#3418)

openx · Dec 23, 2024 · 9954741 · 9954741
1 parent 5f208d6
commit 9954741
Show file tree

Hide file tree

Showing 32 changed files with 1,821 additions and 46 deletions.
diff --git a/docs/application-settings.md b/docs/application-settings.md
@@ -259,6 +259,51 @@ Here's an example YAML file containing account-specific settings:
           default: true
 ```
 
+## Setting Account Configuration in S3
+
+This is identical to the account configuration in a file system, with the main difference that your file system is
+[AWS S3](https://aws.amazon.com/de/s3/) or any S3 compatible storage, such as [MinIO](https://min.io/).
+
+
+The general idea is that you'll place all the account-specific settings in a separate YAML file and point to that file.
+
+```yaml
+settings:
+  s3:
+      accessKeyId: <S3 access key>
+      secretAccessKey: <S3 access key>
+      endpoint: <endpoint> # http://s3.storage.com
+      bucket: <bucket name> # prebid-application-settings
+      region: <region name> # if not provided AWS_GLOBAL will be used. Example value: 'eu-central-1'
+      accounts-dir: accounts
+      stored-imps-dir: stored-impressions
+      stored-requests-dir: stored-requests
+      stored-responses-dir: stored-responses
+
+  # recommended to configure an in memory cache, but this is optional
+  in-memory-cache:
+      # example settings, tailor to your needs
+      cache-size: 100000
+      ttl-seconds: 1200 # 20 minutes
+      # recommended to configure
+      s3-update:
+          refresh-rate: 900000 # Refresh every 15 minutes
+          timeout: 5000
+```
+
+### File format
+
+We recommend using the `json` format for your account configuration. A minimal configuration may look like this.
+
+```json
+{
+  "id" : "979c7116-1f5a-43d4-9a87-5da3ccc4f52c",
+  "status" : "active"
+}
+```
+
+This pairs nicely if you have a default configuration defined in your prebid server config under `settings.default-account-config`.
+
 ## Setting Account Configuration in the Database
 
 In database approach account properties are stored in database table(s).

diff --git a/...er/hooks/modules/com/confiant/adquality/v1/ConfiantAdQualityBidResponsesScanHookTest.java b/...er/hooks/modules/com/confiant/adquality/v1/ConfiantAdQualityBidResponsesScanHookTest.java
@@ -71,11 +71,7 @@ public void setUp() {
 
     @Test
     public void codeShouldHaveValidConfigsWhenInitialized() {
-        // given
-
-        // when
-
-        // then
+        // when and then
         assertThat(target.code()).isEqualTo("confiant-ad-quality-bid-responses-scan-hook");
     }
 

diff --git a/...rg/prebid/server/hooks/modules/com/confiant/adquality/v1/ConfiantAdQualityModuleTest.java b/...rg/prebid/server/hooks/modules/com/confiant/adquality/v1/ConfiantAdQualityModuleTest.java
@@ -8,11 +8,7 @@ public class ConfiantAdQualityModuleTest {
 
     @Test
     public void shouldHaveValidInitialConfigs() {
-        // given
-
-        // when
-
-        // then
+        // when and then
         assertThat(ConfiantAdQualityModule.CODE).isEqualTo("confiant-ad-quality");
     }
 }
diff --git a/...s/fiftyone/devicedetection/v1/hooks/FiftyOneDeviceDetectionRawAuctionRequestHookTest.java b/...s/fiftyone/devicedetection/v1/hooks/FiftyOneDeviceDetectionRawAuctionRequestHookTest.java
@@ -402,7 +402,6 @@ public void callShouldReturnUpdateActionWhenFilterIsNull() {
     @Test
     public void callShouldReturnUpdateActionWhenNoWhitelistAndNoAuctionContext() {
         // given
-
         final AuctionInvocationContext context = AuctionInvocationContextImpl.of(
                 null,
                 null,
@@ -470,7 +469,6 @@ public void callShouldReturnNoUpdateActionWhenWhitelistFilledAndNoAuctionContext
     @Test
     public void callShouldReturnUpdateActionWhenNoWhitelistAndNoAccount() {
         // given
-
         final AuctionContext auctionContext = AuctionContext.builder().build();
         final AuctionInvocationContext context = AuctionInvocationContextImpl.of(
                 null,
@@ -493,7 +491,6 @@ public void callShouldReturnUpdateActionWhenNoWhitelistAndNoAccount() {
     @Test
     public void callShouldReturnNoUpdateActionWhenNoWhitelistAndNoAccountButDeviceIdIsSet() {
         // given
-
         final AuctionContext auctionContext = AuctionContext.builder().build();
         final AuctionInvocationContext context = AuctionInvocationContextImpl.of(
                 null,
@@ -568,7 +565,6 @@ public void callShouldReturnNoUpdateActionWhenWhitelistFilledAndNoAccount() {
     @Test
     public void callShouldReturnUpdateActionWhenNoWhitelistAndNoAccountID() {
         // given
-
         final AuctionContext auctionContext = AuctionContext.builder()
                 .account(Account.builder()
                         .build())
@@ -648,7 +644,6 @@ public void callShouldReturnNoUpdateActionWhenWhitelistFilledAndNoAccountID() {
     @Test
     public void callShouldReturnUpdateActionWhenNoWhitelistAndEmptyAccountID() {
         // given
-
         final AuctionContext auctionContext = AuctionContext.builder()
                 .account(Account.builder()
                         .id("")
@@ -731,7 +726,6 @@ public void callShouldReturnNoUpdateActionWhenWhitelistFilledAndEmptyAccountID()
     @Test
     public void callShouldReturnUpdateActionWhenNoWhitelistAndAllowedAccountID() {
         // given
-
         final AuctionContext auctionContext = AuctionContext.builder()
                 .account(Account.builder()
                         .id("42")
@@ -814,7 +808,6 @@ public void callShouldReturnUpdateActionWhenWhitelistFilledAndAllowedAccountID()
     @Test
     public void callShouldReturnUpdateActionWhenNoWhitelistAndNotAllowedAccountID() {
         // given
-
         final AuctionContext auctionContext = AuctionContext.builder()
                 .account(Account.builder()
                         .id("29")

diff --git a/extra/pom.xml b/extra/pom.xml
@@ -52,6 +52,7 @@
         <protobuf.version>3.21.7</protobuf.version>
         <protoc.version>3.17.3</protoc.version>
         <json-logic.version>1.0.7</json-logic.version>
+        <aws.awssdk.version>2.26.24</aws.awssdk.version>
 
         <!-- Project test dependency versions -->
         <wiremock.version>3.9.1</wiremock.version>
@@ -212,6 +213,11 @@
                 <artifactId>geoip2</artifactId>
                 <version>${maxmind-client.version}</version>
             </dependency>
+            <dependency>
+                <groupId>software.amazon.awssdk</groupId>
+                <artifactId>s3</artifactId>
+                <version>${aws.awssdk.version}</version>
+            </dependency>
             <dependency>
                 <groupId>com.google.protobuf</groupId>
                 <artifactId>protobuf-java</artifactId>

diff --git a/pom.xml b/pom.xml
@@ -170,6 +170,10 @@
             <groupId>org.postgresql</groupId>
             <artifactId>postgresql</artifactId>
         </dependency>
+        <dependency>
+            <groupId>software.amazon.awssdk</groupId>
+            <artifactId>s3</artifactId>
+        </dependency>
         <dependency>
             <groupId>com.github.ben-manes.caffeine</groupId>
             <artifactId>caffeine</artifactId>
@@ -328,6 +332,11 @@
             <artifactId>mysql</artifactId>
             <scope>test</scope>
         </dependency>
+        <dependency>
+            <groupId>org.testcontainers</groupId>
+            <artifactId>localstack</artifactId>
+            <scope>test</scope>
+        </dependency>
         <dependency>
             <groupId>org.testcontainers</groupId>
             <artifactId>postgresql</artifactId>

diff --git a/sample/configs/prebid-config-s3.yaml b/sample/configs/prebid-config-s3.yaml
@@ -0,0 +1,60 @@
+status-response: "ok"
+
+server:
+  enable-quickack: true
+  enable-reuseport: true
+
+adapters:
+  appnexus:
+    enabled: true
+  ix:
+    enabled: true
+  openx:
+    enabled: true
+  pubmatic:
+    enabled: true
+  rubicon:
+    enabled: true
+metrics:
+  prefix: prebid
+cache:
+  scheme: http
+  host: localhost
+  path: /cache
+  query: uuid=
+settings:
+  enforce-valid-account: false
+  generate-storedrequest-bidrequest-id: true
+  s3:
+    accessKeyId: prebid-server-test
+    secretAccessKey: nq9h6whXQURNL2NnWg3rcMlLMtGGDJeWrdl8hC9g
+    endpoint: http://localhost:9000
+    bucket: prebid-server-configs.example.com # prebid-application-settings
+    force-path-style: true # virtual bucketing
+    # region: <region name> # if not provided AWS_GLOBAL will be used. Example value: 'eu-central-1'
+    accounts-dir: accounts
+    stored-imps-dir: stored-impressions
+    stored-requests-dir: stored-requests
+    stored-responses-dir: stored-responses
+
+  in-memory-cache:
+    cache-size: 10000
+    ttl-seconds: 1200 # 20 minutes
+    s3-update:
+      refresh-rate: 900000 # Refresh every 15 minutes
+      timeout: 5000
+
+gdpr:
+  default-value: 1
+  vendorlist:
+    v2:
+      cache-dir: /var/tmp/vendor2
+    v3:
+      cache-dir: /var/tmp/vendor3
+
+admin-endpoints:
+  logging-changelevel:
+    enabled: true
+    path: /logging/changelevel
+    on-application-port: true
+    protected: false