[wip] Add DSP Bridge integration

CHANGELOG.md

@@ -7,6 +7,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
 ## [Unreleased]
+### Added
+- DSP Bridge integration
 
 ## [2.6.2] - 2023-08-07
 ### Added

app/Client/GuzzleAdSelectClient.php

@@ -180,6 +180,12 @@ public function findBanners(array $zones, ImpressionContext $context, string $im
                     if (isset($zones[$i]['options']['banner_mime'])) {
                         $sitesMap[$siteId]['filters']['require']['mime'] = (array)$zones[$i]['options']['banner_mime'];
                     }
+                    if (isset($zones[$i]['options']['exclude'])) {
+                        $sitesMap[$siteId]['filters']['exclude'] = array_merge_recursive(
+                            $sitesMap[$siteId]['filters']['exclude'],
+                            $zones[$i]['options']['exclude'],
+                        );
+                    }
                     if ($site->only_direct_deals) {
                         $sitesMap[$siteId]['filters']['require']['require:site:domain'] = $site->domain;
                     }
@@ -312,6 +318,7 @@ private function fetchInOrderOfAppearance(
                     $campaign = $banner->campaign;
                     $data = [
                         'id'            => $bannerId,
+                        'demand_id'     => $banner->demand_banner_id,
                         'publisher_id'  => $zone->site->user->uuid,
                         'zone_id'       => $zone->uuid,
                         'pay_from'      => $campaign->source_address,

app/Client/GuzzleDemandClient.php

@@ -28,6 +28,7 @@
 use Adshares\Adserver\Models\NetworkCampaign;
 use Adshares\Adserver\Repository\Common\ClassifierExternalRepository;
 use Adshares\Adserver\Services\Common\ClassifierExternalSignatureVerifier;
+use Adshares\Adserver\Services\Supply\DspBridge;
 use Adshares\Adserver\Services\Supply\SiteFilteringUpdater;
 use Adshares\Adserver\Utilities\AdsAuthenticator;
 use Adshares\Adserver\ViewModel\MediumName;
@@ -260,9 +261,11 @@ private function processData(
             'updated_at' => DateTime::createFromFormat(DateTimeInterface::ATOM, $data['updated_at']),
         ];
 
+        $isFromDspBridge = DspBridge::isDspAddress($sourceAddress);
         $classifiersRequired = $this->classifierRepository->fetchRequiredClassifiersNames();
         $banners = [];
-        foreach ((array)$data['banners'] as $banner) {
+        $bannersInput = $data['creatives'] ?? $data['banners'];// legacy fallback, field 'banners' is deprecated
+        foreach ((array)$bannersInput as $banner) {
             $banner['demand_banner_id'] = Uuid::fromString($banner['id']);
 
             if (array_key_exists($banner['id'], $bannerDemandIdsToSupplyIds)) {
@@ -271,7 +274,9 @@ private function processData(
                 unset($banner['id']);
             }
 
-            $mappedClassification = $this->validateAndMapClassification($banner);
+            $mappedClassification = $isFromDspBridge
+                ? self::flattenClassification($banner['classification'] ?? [])
+                : $this->validateAndMapClassification($banner);
             if ($this->missingRequiredClassifier($classifiersRequired, $mappedClassification)) {
                 continue;
             }
@@ -365,7 +370,8 @@ private function getBannerDemandIdsToSupplyIds(array $campaigns, string $sourceA
     {
         $bannerDemandIds = [];
         foreach ($campaigns as $campaign) {
-            foreach ((array)$campaign['banners'] as $banner) {
+            $banners = $campaign['creatives'] ?? $campaign['banners'];// legacy fallback, field 'banners' is deprecated
+            foreach ((array)$banners as $banner) {
                 $bannerDemandIds[] = $banner['id'];
             }
         }
@@ -399,17 +405,17 @@ private function validateAndMapClassification(array $banner): array
             unset($classification[$invalidClassifier]);
         }
 
+        return self::flattenClassification($classification);
+    }
+
+    private static function flattenClassification(mixed $classification): array
+    {
         $flatClassification = [];
         foreach ($classification as $classifier => $classificationItem) {
             $keywords = $classificationItem['keywords'] ?? [];
-            $keywords[SiteFilteringUpdater::KEYWORD_CLASSIFIED] =
-                SiteFilteringUpdater::KEYWORD_CLASSIFIED_VALUE;
-
-            $flatClassification[$classifier] = AbstractFilterMapper::generateNestedStructure(
-                $keywords
-            );
+            $keywords[SiteFilteringUpdater::KEYWORD_CLASSIFIED] = SiteFilteringUpdater::KEYWORD_CLASSIFIED_VALUE;
+            $flatClassification[$classifier] = AbstractFilterMapper::generateNestedStructure($keywords);
         }
-
         return $flatClassification;
     }
 

app/Console/Commands/AdsFetchHosts.php

@@ -32,6 +32,7 @@
 use Adshares\Adserver\Http\Response\InfoResponse;
 use Adshares\Adserver\Models\NetworkHost;
 use Adshares\Adserver\Utilities\DomainReader;
+use Adshares\Adserver\Services\Supply\DspBridgeRegistrar;
 use Adshares\Adserver\ViewModel\ServerEventType;
 use Adshares\Common\Exception\RuntimeException;
 use Adshares\Config\AppMode;
@@ -56,16 +57,20 @@ class AdsFetchHosts extends BaseCommand
     protected $signature = 'ads:fetch-hosts';
     protected $description = 'Fetches Demand AdServers';
 
-    public function __construct(Locker $locker, private readonly DemandClient $client)
-    {
+    public function __construct(
+        Locker $locker,
+        private readonly AdsClient $adsClient,
+        private readonly DemandClient $client,
+        private readonly DspBridgeRegistrar $dspBridgeRegistrar,
+    ) {
         parent::__construct($locker);
     }
 
-    public function handle(AdsClient $adsClient): void
+    public function handle(): int
     {
         if (!$this->lock()) {
             $this->info('Command ' . $this->signature . ' already running');
-            return;
+            return self::FAILURE;
         }
 
         $this->info('Start command ' . $this->signature);
@@ -79,12 +84,15 @@ public function handle(AdsClient $adsClient): void
         $progressBar->start();
         while ($timeBlock <= $timeNow - self::BLOCK_TIME) {
             $blockId = dechex($timeBlock);
-            $found += $this->handleBlock($adsClient, $blockId);
+            $found += $this->handleBlock($blockId);
             $timeBlock += self::BLOCK_TIME;
             $progressBar->advance();
         }
         $progressBar->finish();
         $this->newLine();
+        if ($this->dspBridgeRegistrar->registerAsNetworkHost()) {
+            $found++;
+        }
 
         $this->comment('Cleaning up old hosts...');
         $added = NetworkHost::all()->count() - $hostsCount;
@@ -103,6 +111,7 @@ public function handle(AdsClient $adsClient): void
         $this->info($removed > 0 ? sprintf('Removed %d hosts', $removed) : 'Nothing to remove');
 
         $this->info('Finished command ' . $this->signature);
+        return self::SUCCESS;
     }
 
     private function getTimeOfFirstBlock(int $timeNow): int
@@ -119,11 +128,11 @@ private function getTimeOfFirstBlock(int $timeNow): int
         return $timeBlock;
     }
 
-    private function handleBlock(AdsClient $adsClient, string $blockId): int
+    private function handleBlock(string $blockId): int
     {
         $foundHosts = 0;
         try {
-            $response = $adsClient->getBroadcast($blockId);
+            $response = $this->adsClient->getBroadcast($blockId);
             $broadcastArray = $response->getBroadcast();
 
             foreach ($broadcastArray as $broadcast) {

app/Console/Commands/SupplyProcessPayments.php

@@ -30,8 +30,10 @@
 use Adshares\Adserver\Models\NetworkHost;
 use Adshares\Adserver\Models\TurnoverEntry;
 use Adshares\Adserver\Services\Dto\PaymentProcessingResult;
+use Adshares\Adserver\Services\Dto\ProcessedPaymentsMetaData;
 use Adshares\Adserver\Services\LicenseFeeSender;
 use Adshares\Adserver\Services\PaymentDetailsProcessor;
+use Adshares\Adserver\Services\Supply\DspBridge;
 use Adshares\Adserver\Utilities\DateUtils;
 use Adshares\Adserver\ViewModel\ServerEventType;
 use Adshares\Common\Infrastructure\Service\LicenseReader;
@@ -45,6 +47,8 @@
 
 class SupplyProcessPayments extends BaseCommand
 {
+    private const COLUMN_ADS_PAYMENT_ID = 'ads_payment_id';
+    private const COLUMN_BRIDGE_PAYMENT_ID = 'bridge_payment_id';
     private const TRY_OUT_PERIOD_FOR_EVENT_PAYMENT = '-24 hours';
 
     private const SQL_QUERY_GET_PROCESSED_PAYMENTS_AMOUNT = <<<SQL
@@ -61,11 +65,14 @@ class SupplyProcessPayments extends BaseCommand
   (
     SELECT DISTINCT DATE(pay_time) AS d, HOUR(pay_time) AS h
     FROM network_case_payments
-    WHERE ads_payment_id IN (%s)
+    WHERE :paymentIdColumn IN (:whereInPlaceholder)
     UNION
     SELECT DISTINCT DATE(created_at) AS d, HOUR(created_at) AS h
     FROM network_cases
-    WHERE id IN (SELECT DISTINCT network_case_id FROM network_case_payments WHERE ads_payment_id IN (%s))) t;
+    WHERE id IN (
+      SELECT DISTINCT network_case_id FROM network_case_payments WHERE :paymentIdColumn IN (:whereInPlaceholder)
+    )
+  ) t;
 SQL;
 
     protected $signature = 'ops:supply:payments:process {--c|chunkSize=5000}';
@@ -76,7 +83,7 @@ public function __construct(
         private readonly AdsClient $adsClient,
         private readonly DemandClient $demandClient,
         private readonly LicenseReader $licenseReader,
-        private readonly PaymentDetailsProcessor $paymentDetailsProcessor
+        private readonly PaymentDetailsProcessor $paymentDetailsProcessor,
     ) {
         parent::__construct($locker);
     }
@@ -90,13 +97,56 @@ public function handle(): void
 
         $this->info('Start command ' . $this->getName());
 
-        $adsPayments = AdsPayment::fetchByStatus(AdsPayment::STATUS_EVENT_PAYMENT_CANDIDATE);
+        $processedAdsPaymentMetaData = $this->processAdsPayments();
+        $processedPaymentsForAds = $processedAdsPaymentMetaData->getProcessedPaymentsForAds();
+        $processedPaymentsTotal = $processedAdsPaymentMetaData->getProcessedPaymentsTotal();
+        $timestamps = $this->fetchTimestampsToUpdate(
+            $processedAdsPaymentMetaData->getPaymentIds(),
+            self::COLUMN_ADS_PAYMENT_ID,
+        );
 
-        $earliestTryOutDateTime = new DateTimeImmutable(self::TRY_OUT_PERIOD_FOR_EVENT_PAYMENT);
+        if (DspBridge::isActive()) {
+            $bridge = new DspBridge();
+            $bridge->fetchAndStorePayments();
+
+            $processedBridgePaymentMetaData = $bridge->processPayments(
+                $this->demandClient,
+                $this->paymentDetailsProcessor,
+                (int)$this->option('chunkSize'),
+            );
+            $processedPaymentsForAds += $processedBridgePaymentMetaData->getProcessedPaymentsForAds();
+            $processedPaymentsTotal += $processedBridgePaymentMetaData->getProcessedPaymentsTotal();
+            $timestamps = array_unique(
+                array_merge(
+                    $timestamps,
+                    $this->fetchTimestampsToUpdate(
+                        $processedBridgePaymentMetaData->getPaymentIds(),
+                        self::COLUMN_BRIDGE_PAYMENT_ID,
+                    ),
+                )
+            );
+        }
+
+        foreach ($timestamps as $timestamp) {
+            NetworkCaseLogsHourlyMeta::invalidate($timestamp);
+        }
+        ServerEvent::dispatch(ServerEventType::IncomingAdPaymentProcessed, [
+            'adsPaymentCount' => $processedPaymentsForAds,
+            'totalPaymentCount' => $processedPaymentsTotal,
+        ]);
+
+        $this->info('End command ' . $this->getName());
+    }
+
+    private function processAdsPayments(): ProcessedPaymentsMetaData
+    {
         $processedAdsPaymentIds = [];
         $processedPaymentsTotal = 0;
         $processedPaymentsForAds = 0;
 
+        $adsPayments = AdsPayment::fetchByStatus(AdsPayment::STATUS_EVENT_PAYMENT_CANDIDATE);
+        $earliestTryOutDateTime = new DateTimeImmutable(self::TRY_OUT_PERIOD_FOR_EVENT_PAYMENT);
+
         /** @var AdsPayment $adsPayment */
         foreach ($adsPayments as $adsPayment) {
             if ($adsPayment->created_at < $earliestTryOutDateTime) {
@@ -122,24 +172,18 @@ public function handle(): void
                 DB::rollBack();
                 $this->error(
                     sprintf(
-                        'Error during handling paid events for id=%d (%s)',
+                        'Error during handling paid events for ads payment id=%d (%s)',
                         $adsPayment->id,
                         $throwable->getMessage()
                     )
                 );
             }
         }
-
-        $timestamps = $this->fetchTimestampsToUpdate($processedAdsPaymentIds);
-        foreach ($timestamps as $timestamp) {
-            NetworkCaseLogsHourlyMeta::invalidate($timestamp);
-        }
-        ServerEvent::dispatch(ServerEventType::IncomingAdPaymentProcessed, [
-            'adsPaymentCount' => $processedPaymentsForAds,
-            'totalPaymentCount' => $processedPaymentsTotal,
-        ]);
-
-        $this->info('End command ' . $this->getName());
+        return new ProcessedPaymentsMetaData(
+            $processedAdsPaymentIds,
+            $processedPaymentsTotal,
+            $processedPaymentsForAds,
+        );
     }
 
     private function handleEventPaymentCandidate(AdsPayment $incomingPayment): void
@@ -205,24 +249,22 @@ private function handleEventPaymentCandidate(AdsPayment $incomingPayment): void
         }
     }
 
-    private function fetchTimestampsToUpdate(array $adsPaymentIds): array
+    private function fetchTimestampsToUpdate(array $paymentIds, string $paymentIdColumn): array
     {
-        if (empty($adsPaymentIds)) {
+        if (empty($paymentIds)) {
             return [];
         }
 
-        $whereInPlaceholder = str_repeat('?,', count($adsPaymentIds) - 1) . '?';
-        $query = sprintf(
+        $whereInPlaceholder = str_repeat('?,', count($paymentIds) - 1) . '?';
+        $query = str_replace(
+            [':paymentIdColumn', ':whereInPlaceholder'],
+            [$paymentIdColumn, $whereInPlaceholder],
             self::SQL_QUERY_SELECT_TIMESTAMPS_TO_UPDATE_TEMPLATE,
-            $whereInPlaceholder,
-            $whereInPlaceholder
         );
 
         return array_map(
-            function (stdClass $item) {
-                return (int)$item->pay_time;
-            },
-            DB::select($query, array_merge($adsPaymentIds, $adsPaymentIds))
+            fn(stdClass $item) => (int)$item->pay_time,
+            DB::select($query, array_merge($paymentIds, $paymentIds))
         );
     }
 

app/Http/Controllers/DemandController.php

@@ -41,6 +41,7 @@
 use Adshares\Common\Exception\RuntimeException;
 use Adshares\Demand\Application\Service\PaymentDetailsVerify;
 use DateTime;
+use DateTimeImmutable;
 use DateTimeInterface;
 use Illuminate\Database\Eloquent\ModelNotFoundException;
 use Illuminate\Http\Request;
@@ -472,9 +473,9 @@ public function paymentDetails(
     ): JsonResponse {
         $transactionIdDecoded = AdsUtils::decodeTxId($transactionId);
         $accountAddressDecoded = AdsUtils::decodeAddress($accountAddress);
-        $datetime = DateTime::createFromFormat(DateTimeInterface::ATOM, $date);
+        $datetime = DateTimeImmutable::createFromFormat(DateTimeInterface::ATOM, $date);
 
-        if ($transactionIdDecoded === null || $accountAddressDecoded === null) {
+        if (null === $transactionIdDecoded || null === $accountAddressDecoded || false === $datetime) {
             throw new BadRequestHttpException('Input data are invalid.');
         }
 
@@ -587,7 +588,7 @@ public function inventoryList(Request $request, TotalFeeReader $totalFeeReader):
                 'max_cpc' => $campaign->max_cpc,
                 'max_cpm' => $campaign->max_cpm,
                 'budget' => self::calculateBudgetAfterFees($campaign->budget, $totalFee),
-                'banners' => $banners,
+                'creatives' => $banners,
                 'targeting_requires' => (array)$campaign->targeting_requires,
                 'targeting_excludes' => (array)$campaign->targeting_excludes,
             ];

app/Http/Controllers/Manager/ServerConfigurationController.php

@@ -105,6 +105,8 @@ class ServerConfigurationController extends Controller
         Config::CRM_MAIL_ADDRESS_ON_USER_REGISTERED => 'nullable|email',
         Config::CURRENCY => 'nullable|appCurrency',
         Config::DEFAULT_USER_ROLES => 'nullable|notEmpty|list:userRole',
+        Config::DSP_BRIDGE_ACCOUNT_ADDRESS => 'nullable|accountId',
+        Config::DSP_BRIDGE_URL => 'nullable|url',
         Config::EMAIL_VERIFICATION_REQUIRED => 'nullable|boolean',
         Config::EXCHANGE_API_KEY => 'nullable',
         Config::EXCHANGE_API_SECRET => 'nullable',