From ee7b7987eed5fc6bd2465726440c6e34655fe2b6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82Podkalicki?= Date: Mon, 27 Feb 2023 13:30:44 +0100 Subject: [PATCH 01/55] Add OpenRTB provider to Config --- .../Controllers/Manager/ServerConfigurationController.php | 2 ++ app/Models/Config.php | 4 ++++ .../Controllers/Manager/ServerConfigurationControllerTest.php | 2 ++ 3 files changed, 8 insertions(+) diff --git a/app/Http/Controllers/Manager/ServerConfigurationController.php b/app/Http/Controllers/Manager/ServerConfigurationController.php index 01c060e18..35454d56b 100644 --- a/app/Http/Controllers/Manager/ServerConfigurationController.php +++ b/app/Http/Controllers/Manager/ServerConfigurationController.php @@ -147,6 +147,8 @@ class ServerConfigurationController extends Controller Config::NOW_PAYMENTS_IPN_SECRET => 'nullable', Config::NOW_PAYMENTS_MAX_AMOUNT => 'nullable|integer|min:0', Config::NOW_PAYMENTS_MIN_AMOUNT => 'nullable|integer|min:0', + Config::OPEN_RTB_PROVIDER_ACCOUNT_ADDRESS => 'nullable|accountId', + Config::OPEN_RTB_PROVIDER_URL => 'nullable|url', Config::OPERATOR_RX_FEE => 'nullable|commission', Config::OPERATOR_TX_FEE => 'nullable|commission', Config::PUBLISHER_APPLY_FORM_URL => 'nullable|url', diff --git a/app/Models/Config.php b/app/Models/Config.php index e4bfae3a0..8d1fc7711 100644 --- a/app/Models/Config.php +++ b/app/Models/Config.php @@ -150,6 +150,8 @@ class Config extends Model public const NOW_PAYMENTS_IPN_SECRET = 'now-payments-ipn-secret'; public const NOW_PAYMENTS_MAX_AMOUNT = 'now-payments-max-amount'; public const NOW_PAYMENTS_MIN_AMOUNT = 'now-payments-min-amount'; + public const OPEN_RTB_PROVIDER_ACCOUNT_ADDRESS = 'open-rtb-provider-account-address'; + public const OPEN_RTB_PROVIDER_URL = 'open-rtb-provider-url'; public const OPERATOR_RX_FEE = 'payment-rx-fee'; public const OPERATOR_TX_FEE = 'payment-tx-fee'; public const OPERATOR_WALLET_EMAIL_LAST_TIME = 'operator-wallet-transfer-email-time'; @@ -529,6 +531,8 @@ private static function getDefaultAdminSettings(array $fetched = []): array self::NOW_PAYMENTS_IPN_SECRET => '', self::NOW_PAYMENTS_MAX_AMOUNT => 1000, self::NOW_PAYMENTS_MIN_AMOUNT => 25, + self::OPEN_RTB_PROVIDER_ACCOUNT_ADDRESS => null, + self::OPEN_RTB_PROVIDER_URL => null, self::OPERATOR_RX_FEE => 0.01, self::OPERATOR_TX_FEE => 0.01, self::PUBLISHER_APPLY_FORM_URL => null, diff --git a/tests/app/Http/Controllers/Manager/ServerConfigurationControllerTest.php b/tests/app/Http/Controllers/Manager/ServerConfigurationControllerTest.php index b7aece9ad..4ebf26ceb 100644 --- a/tests/app/Http/Controllers/Manager/ServerConfigurationControllerTest.php +++ b/tests/app/Http/Controllers/Manager/ServerConfigurationControllerTest.php @@ -234,6 +234,8 @@ public function storeDataProvider(): array 'INVENTORY_FAILED_CONNECTION_LIMIT' => [Config::INVENTORY_FAILED_CONNECTION_LIMIT, '8'], 'LANDING_URL' => [Config::LANDING_URL, 'https://example.com'], 'MAX_INVALID_LOGIN_ATTEMPTS' => [Config::MAX_INVALID_LOGIN_ATTEMPTS, '8'], + 'OPEN_RTB_PROVIDER_ACCOUNT_ADDRESS' => [Config::OPEN_RTB_PROVIDER_ACCOUNT_ADDRESS, '0001-00000003-AB0C'], + 'OPEN_RTB_PROVIDER_URL' => [Config::OPEN_RTB_PROVIDER_URL, 'http://localhost:8015'], 'REFERRAL_REFUND_COMMISSION' => [Config::REFERRAL_REFUND_COMMISSION, '0'], 'REFERRAL_REFUND_ENABLED' => [Config::REFERRAL_REFUND_ENABLED, '1'], 'SUPPORT_EMAIL' => [Config::SUPPORT_EMAIL, 'sup@example.com'], From 4c1d84eada5dfe19ca57e190241413417f7542c5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82Podkalicki?= Date: Mon, 27 Feb 2023 13:37:03 +0100 Subject: [PATCH 02/55] Remove unused repository --- .../Supply/NetworkHostRepository.php | 32 ------------------- 1 file changed, 32 deletions(-) delete mode 100644 app/Repository/Supply/NetworkHostRepository.php diff --git a/app/Repository/Supply/NetworkHostRepository.php b/app/Repository/Supply/NetworkHostRepository.php deleted file mode 100644 index 72f9fe8b8..000000000 --- a/app/Repository/Supply/NetworkHostRepository.php +++ /dev/null @@ -1,32 +0,0 @@ - - */ - -namespace Adshares\Adserver\Repository\Supply; - -use Adshares\Adserver\Models\NetworkHost; - -class NetworkHostRepository -{ - public function find() - { - return (new NetworkHost())->get(); - } -} From 499995353915d78b5d5fa55441414ae5a40cf70b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82Podkalicki?= Date: Tue, 28 Feb 2023 07:52:50 +0100 Subject: [PATCH 03/55] Change deprecated field banners in inventory list --- app/Client/GuzzleDemandClient.php | 6 ++++-- app/Http/Controllers/DemandController.php | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/app/Client/GuzzleDemandClient.php b/app/Client/GuzzleDemandClient.php index 20f0fdb03..7d29333d1 100644 --- a/app/Client/GuzzleDemandClient.php +++ b/app/Client/GuzzleDemandClient.php @@ -261,7 +261,8 @@ private function processData( $classifiersRequired = $this->classifierRepository->fetchRequiredClassifiersNames(); $banners = []; - foreach ((array)$data['banners'] as $banner) { + $bannersInput = $data['creatives'] ?? $data['banners'];// legacy fallback, field 'banners' is deprecated + foreach ((array)$bannersInput as $banner) { $banner['demand_banner_id'] = Uuid::fromString($banner['id']); if (array_key_exists($banner['id'], $bannerDemandIdsToSupplyIds)) { @@ -358,7 +359,8 @@ private function getBannerDemandIdsToSupplyIds(array $campaigns, string $sourceA { $bannerDemandIds = []; foreach ($campaigns as $campaign) { - foreach ((array)$campaign['banners'] as $banner) { + $banners = $campaign['creatives'] ?? $campaign['banners'];// legacy fallback, field 'banners' is deprecated + foreach ((array)$banners as $banner) { $bannerDemandIds[] = $banner['id']; } } diff --git a/app/Http/Controllers/DemandController.php b/app/Http/Controllers/DemandController.php index 95801e183..cbd214505 100644 --- a/app/Http/Controllers/DemandController.php +++ b/app/Http/Controllers/DemandController.php @@ -584,7 +584,7 @@ public function inventoryList(Request $request, TotalFeeReader $totalFeeReader): 'max_cpc' => $campaign->max_cpc, 'max_cpm' => $campaign->max_cpm, 'budget' => self::calculateBudgetAfterFees($campaign->budget, $totalFee), - 'banners' => $banners, + 'creatives' => $banners, 'targeting_requires' => (array)$campaign->targeting_requires, 'targeting_excludes' => (array)$campaign->targeting_excludes, ]; From b41151ed1bc8adc21667e3a5ed960f119c9fce80 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82Podkalicki?= Date: Tue, 28 Feb 2023 11:34:20 +0100 Subject: [PATCH 04/55] Integrate OpenRTB provider (inventory) --- app/Client/GuzzleDemandClient.php | 19 +++++----- app/Console/Commands/AdsFetchHosts.php | 35 +++++++++++++++++++ .../Console/Commands/AdsFetchHostsTest.php | 24 +++++++++++++ 3 files changed, 70 insertions(+), 8 deletions(-) diff --git a/app/Client/GuzzleDemandClient.php b/app/Client/GuzzleDemandClient.php index 7d29333d1..6d98fce6a 100644 --- a/app/Client/GuzzleDemandClient.php +++ b/app/Client/GuzzleDemandClient.php @@ -259,6 +259,7 @@ private function processData( 'updated_at' => DateTime::createFromFormat(DateTimeInterface::ATOM, $data['updated_at']), ]; + $isOpenRtbProvider = $sourceAddress === config('app.open_rtb_provider_account_address'); $classifiersRequired = $this->classifierRepository->fetchRequiredClassifiersNames(); $banners = []; $bannersInput = $data['creatives'] ?? $data['banners'];// legacy fallback, field 'banners' is deprecated @@ -271,7 +272,9 @@ private function processData( unset($banner['id']); } - $banner['classification'] = $this->validateAndMapClassification($banner); + $banner['classification'] = $isOpenRtbProvider + ? self::flattenClassification($banner['classification'] ?? []) + : $this->validateAndMapClassification($banner); if ($this->missingRequiredClassifier($classifiersRequired, $banner['classification'])) { continue; } @@ -394,17 +397,17 @@ private function validateAndMapClassification(array $banner): array unset($classification[$invalidClassifier]); } + return self::flattenClassification($classification); + } + + private static function flattenClassification(mixed $classification): array + { $flatClassification = []; foreach ($classification as $classifier => $classificationItem) { $keywords = $classificationItem['keywords'] ?? []; - $keywords[SiteFilteringUpdater::KEYWORD_CLASSIFIED] = - SiteFilteringUpdater::KEYWORD_CLASSIFIED_VALUE; - - $flatClassification[$classifier] = AbstractFilterMapper::generateNestedStructure( - $keywords - ); + $keywords[SiteFilteringUpdater::KEYWORD_CLASSIFIED] = SiteFilteringUpdater::KEYWORD_CLASSIFIED_VALUE; + $flatClassification[$classifier] = AbstractFilterMapper::generateNestedStructure($keywords); } - return $flatClassification; } diff --git a/app/Console/Commands/AdsFetchHosts.php b/app/Console/Commands/AdsFetchHosts.php index 63973ad72..91f297166 100644 --- a/app/Console/Commands/AdsFetchHosts.php +++ b/app/Console/Commands/AdsFetchHosts.php @@ -32,8 +32,11 @@ use Adshares\Adserver\Http\Response\InfoResponse; use Adshares\Adserver\Models\NetworkHost; use Adshares\Adserver\ViewModel\ServerEventType; +use Adshares\Common\Domain\ValueObject\AccountId; +use Adshares\Common\Domain\ValueObject\Url; use Adshares\Common\Exception\RuntimeException; use Adshares\Config\AppMode; +use Adshares\Config\RegistrationMode; use Adshares\Network\BroadcastableUrl; use Adshares\Supply\Application\Dto\Info; use Adshares\Supply\Application\Service\DemandClient; @@ -84,6 +87,9 @@ public function handle(AdsClient $adsClient): void } $progressBar->finish(); $this->newLine(); + if ($this->registerOpenRtbProviderAsNetworkHost()) { + $found++; + } $this->comment('Cleaning up old hosts...'); $added = NetworkHost::all()->count() - $hostsCount; @@ -197,4 +203,33 @@ private function removeOldHosts(): int $period = new DateTimeImmutable(sprintf('-%d hours', config('app.hours_until_inactive_host_removal'))); return NetworkHost::deleteBroadcastedBefore($period); } + + private function registerOpenRtbProviderAsNetworkHost(): bool + { + if ( + null !== ($accountAddress = config('app.open_rtb_provider_account_address')) + && null !== ($url = config('app.open_rtb_provider_url')) + ) { + $info = new Info( + 'openrtb', + 'OpenRTB Provider', + '0.1.0', + new Url($url), + new Url($url), + new Url($url), + new Url($url . '/policies/privacy.html'), + new Url($url . '/policies/terms.html'), + new Url($url . '/adshares/inventory/list'), + new AccountId($accountAddress), + null, + [Info::CAPABILITY_ADVERTISER], + RegistrationMode::PRIVATE, + AppMode::OPERATIONAL + ); + $host = NetworkHost::registerHost($accountAddress, $url, $info, new DateTimeImmutable()); + Log::debug(sprintf('Stored %s as #%d', $url, $host->id)); + return true; + } + return false; + } } diff --git a/tests/app/Console/Commands/AdsFetchHostsTest.php b/tests/app/Console/Commands/AdsFetchHostsTest.php index 01dc31986..7377e1acc 100644 --- a/tests/app/Console/Commands/AdsFetchHostsTest.php +++ b/tests/app/Console/Commands/AdsFetchHostsTest.php @@ -241,6 +241,30 @@ public function testBroadcastError(): void self::assertServerEventDispatched(ServerEventType::HostBroadcastProcessed); } + public function testRegisterOpenRtbProvider(): void + { + $this->app->bind( + AdsClient::class, + function () { + $clientMock = self::createMock(AdsClient::class); + $clientMock->method('getBroadcast') + ->willReturn(new GetBroadcastResponse($this->getRawEmptyBroadcastData())); + return $clientMock; + } + ); + Config::updateAdminSettings([ + Config::OPEN_RTB_PROVIDER_ACCOUNT_ADDRESS => '0001-00000001-8B4E', + Config::OPEN_RTB_PROVIDER_URL => 'https://example.com', + ]); + + self::artisan(self::COMMAND_SIGNATURE)->assertExitCode(0); + + $host = NetworkHost::fetchByAddress('0001-00000001-8B4E'); + $this->assertNotNull($host); + $this->assertEquals('https://example.com', $host->host); + self::assertServerEventDispatched(ServerEventType::HostBroadcastProcessed); + } + private function setupAdsClient(): void { $this->app->bind( From 4817c0209d8e17c9f0fff0e7130eb9f3b08d2bef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82Podkalicki?= Date: Tue, 28 Feb 2023 11:59:04 +0100 Subject: [PATCH 05/55] Extract registrar --- app/Console/Commands/AdsFetchHosts.php | 54 ++++----------- .../Supply/OpenRtbProviderRegistrar.php | 65 +++++++++++++++++++ .../Console/Commands/AdsFetchHostsTest.php | 28 ++++++-- .../Supply/OpenRtbProviderRegistrarTest.php | 50 ++++++++++++++ 4 files changed, 152 insertions(+), 45 deletions(-) create mode 100644 app/Services/Supply/OpenRtbProviderRegistrar.php create mode 100644 tests/app/Services/Supply/OpenRtbProviderRegistrarTest.php diff --git a/app/Console/Commands/AdsFetchHosts.php b/app/Console/Commands/AdsFetchHosts.php index 91f297166..c8856bef6 100644 --- a/app/Console/Commands/AdsFetchHosts.php +++ b/app/Console/Commands/AdsFetchHosts.php @@ -31,12 +31,10 @@ use Adshares\Adserver\Events\ServerEvent; use Adshares\Adserver\Http\Response\InfoResponse; use Adshares\Adserver\Models\NetworkHost; +use Adshares\Adserver\Services\Supply\OpenRtbProviderRegistrar; use Adshares\Adserver\ViewModel\ServerEventType; -use Adshares\Common\Domain\ValueObject\AccountId; -use Adshares\Common\Domain\ValueObject\Url; use Adshares\Common\Exception\RuntimeException; use Adshares\Config\AppMode; -use Adshares\Config\RegistrationMode; use Adshares\Network\BroadcastableUrl; use Adshares\Supply\Application\Dto\Info; use Adshares\Supply\Application\Service\DemandClient; @@ -58,16 +56,20 @@ class AdsFetchHosts extends BaseCommand protected $signature = 'ads:fetch-hosts'; protected $description = 'Fetches Demand AdServers'; - public function __construct(Locker $locker, private readonly DemandClient $client) - { + public function __construct( + Locker $locker, + private readonly AdsClient $adsClient, + private readonly DemandClient $client, + private readonly OpenRtbProviderRegistrar $openRtbProviderRegistrar, + ) { parent::__construct($locker); } - public function handle(AdsClient $adsClient): void + public function handle(): int { if (!$this->lock()) { $this->info('Command ' . $this->signature . ' already running'); - return; + return self::FAILURE; } $this->info('Start command ' . $this->signature); @@ -81,13 +83,13 @@ public function handle(AdsClient $adsClient): void $progressBar->start(); while ($timeBlock <= $timeNow - self::BLOCK_TIME) { $blockId = dechex($timeBlock); - $found += $this->handleBlock($adsClient, $blockId); + $found += $this->handleBlock($blockId); $timeBlock += self::BLOCK_TIME; $progressBar->advance(); } $progressBar->finish(); $this->newLine(); - if ($this->registerOpenRtbProviderAsNetworkHost()) { + if ($this->openRtbProviderRegistrar->registerAsNetworkHost()) { $found++; } @@ -108,6 +110,7 @@ public function handle(AdsClient $adsClient): void $this->info($removed > 0 ? sprintf('Removed %d hosts', $removed) : 'Nothing to remove'); $this->info('Finished command ' . $this->signature); + return self::SUCCESS; } private function getTimeOfFirstBlock(int $timeNow): int @@ -124,11 +127,11 @@ private function getTimeOfFirstBlock(int $timeNow): int return $timeBlock; } - private function handleBlock(AdsClient $adsClient, string $blockId): int + private function handleBlock(string $blockId): int { $foundHosts = 0; try { - $response = $adsClient->getBroadcast($blockId); + $response = $this->adsClient->getBroadcast($blockId); $broadcastArray = $response->getBroadcast(); foreach ($broadcastArray as $broadcast) { @@ -203,33 +206,4 @@ private function removeOldHosts(): int $period = new DateTimeImmutable(sprintf('-%d hours', config('app.hours_until_inactive_host_removal'))); return NetworkHost::deleteBroadcastedBefore($period); } - - private function registerOpenRtbProviderAsNetworkHost(): bool - { - if ( - null !== ($accountAddress = config('app.open_rtb_provider_account_address')) - && null !== ($url = config('app.open_rtb_provider_url')) - ) { - $info = new Info( - 'openrtb', - 'OpenRTB Provider', - '0.1.0', - new Url($url), - new Url($url), - new Url($url), - new Url($url . '/policies/privacy.html'), - new Url($url . '/policies/terms.html'), - new Url($url . '/adshares/inventory/list'), - new AccountId($accountAddress), - null, - [Info::CAPABILITY_ADVERTISER], - RegistrationMode::PRIVATE, - AppMode::OPERATIONAL - ); - $host = NetworkHost::registerHost($accountAddress, $url, $info, new DateTimeImmutable()); - Log::debug(sprintf('Stored %s as #%d', $url, $host->id)); - return true; - } - return false; - } } diff --git a/app/Services/Supply/OpenRtbProviderRegistrar.php b/app/Services/Supply/OpenRtbProviderRegistrar.php new file mode 100644 index 000000000..15fc3516c --- /dev/null +++ b/app/Services/Supply/OpenRtbProviderRegistrar.php @@ -0,0 +1,65 @@ + + */ + +declare(strict_types=1); + +namespace Adshares\Adserver\Services\Supply; + +use Adshares\Adserver\Models\NetworkHost; +use Adshares\Common\Domain\ValueObject\AccountId; +use Adshares\Common\Domain\ValueObject\Url; +use Adshares\Config\AppMode; +use Adshares\Config\RegistrationMode; +use Adshares\Supply\Application\Dto\Info; +use DateTimeImmutable; +use Illuminate\Support\Facades\Log; + +class OpenRtbProviderRegistrar +{ + public function registerAsNetworkHost(): bool + { + if ( + null !== ($accountAddress = config('app.open_rtb_provider_account_address')) + && null !== ($url = config('app.open_rtb_provider_url')) + ) { + $info = new Info( + 'openrtb', + 'OpenRTB Provider', + '0.1.0', + new Url($url), + new Url($url), + new Url($url), + new Url($url . '/policies/privacy.html'), + new Url($url . '/policies/terms.html'), + new Url($url . '/adshares/inventory/list'), + new AccountId($accountAddress), + null, + [Info::CAPABILITY_ADVERTISER], + RegistrationMode::PRIVATE, + AppMode::OPERATIONAL + ); + $host = NetworkHost::registerHost($accountAddress, $url, $info, new DateTimeImmutable()); + Log::debug(sprintf('Stored %s as #%d', $url, $host->id)); + return true; + } + return false; + } +} diff --git a/tests/app/Console/Commands/AdsFetchHostsTest.php b/tests/app/Console/Commands/AdsFetchHostsTest.php index 7377e1acc..3ef872399 100644 --- a/tests/app/Console/Commands/AdsFetchHostsTest.php +++ b/tests/app/Console/Commands/AdsFetchHostsTest.php @@ -29,6 +29,7 @@ use Adshares\Adserver\Console\Locker; use Adshares\Adserver\Models\Config; use Adshares\Adserver\Models\NetworkHost; +use Adshares\Adserver\Services\Supply\OpenRtbProviderRegistrar; use Adshares\Adserver\Tests\Console\ConsoleTestCase; use Adshares\Adserver\ViewModel\ServerEventType; use Adshares\Config\AppMode; @@ -49,7 +50,7 @@ public function testLock(): void $lockerMock->expects(self::once())->method('lock')->willReturn(false); $this->instance(Locker::class, $lockerMock); - self::artisan(self::COMMAND_SIGNATURE)->assertExitCode(0); + self::artisan(self::COMMAND_SIGNATURE)->assertExitCode(1); } public function testFetchingHosts(): void @@ -252,6 +253,18 @@ function () { return $clientMock; } ); + $this->app->bind( + OpenRtbProviderRegistrar::class, + function () { + $mock = self::createMock(OpenRtbProviderRegistrar::class); + $mock->method('registerAsNetworkHost') + ->will($this->returnCallback(function () { + NetworkHost::factory()->create(); + return true; + })); + return $mock; + } + ); Config::updateAdminSettings([ Config::OPEN_RTB_PROVIDER_ACCOUNT_ADDRESS => '0001-00000001-8B4E', Config::OPEN_RTB_PROVIDER_URL => 'https://example.com', @@ -259,10 +272,15 @@ function () { self::artisan(self::COMMAND_SIGNATURE)->assertExitCode(0); - $host = NetworkHost::fetchByAddress('0001-00000001-8B4E'); - $this->assertNotNull($host); - $this->assertEquals('https://example.com', $host->host); - self::assertServerEventDispatched(ServerEventType::HostBroadcastProcessed); + self::assertServerEventDispatched( + ServerEventType::HostBroadcastProcessed, + [ + 'added' => 1, + 'found' => 1, + 'marked' => 0, + 'removed' => 0, + ] + ); } private function setupAdsClient(): void diff --git a/tests/app/Services/Supply/OpenRtbProviderRegistrarTest.php b/tests/app/Services/Supply/OpenRtbProviderRegistrarTest.php new file mode 100644 index 000000000..ba21cd224 --- /dev/null +++ b/tests/app/Services/Supply/OpenRtbProviderRegistrarTest.php @@ -0,0 +1,50 @@ + + */ + +declare(strict_types=1); + +namespace Adshares\Adserver\Tests\Services\Supply; + +use Adshares\Adserver\Models\Config; +use Adshares\Adserver\Models\NetworkHost; +use Adshares\Adserver\Services\Supply\OpenRtbProviderRegistrar; +use Adshares\Adserver\Tests\TestCase; +use Adshares\Adserver\Utilities\DatabaseConfigReader; + +class OpenRtbProviderRegistrarTest extends TestCase +{ + public function testRegisterAsNetworkHost(): void + { + Config::updateAdminSettings([ + Config::OPEN_RTB_PROVIDER_ACCOUNT_ADDRESS => '0001-00000001-8B4E', + Config::OPEN_RTB_PROVIDER_URL => 'https://example.com', + ]); + DatabaseConfigReader::overwriteAdministrationConfig(); + $registrar = new OpenRtbProviderRegistrar(); + + $registrar->registerAsNetworkHost(); + + self::assertDatabaseHas(NetworkHost::class, [ + 'address' => '0001-00000001-8B4E', + 'host' => 'https://example.com', + ]); + } +} From 8583a582289dfe3f933d345a8148a64d12eb9038 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82Podkalicki?= Date: Tue, 28 Feb 2023 12:37:35 +0100 Subject: [PATCH 06/55] Do not accept invalid date format --- app/Http/Controllers/DemandController.php | 5 +++-- .../Application/Service/PaymentDetailsVerify.php | 12 ++++++++---- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/app/Http/Controllers/DemandController.php b/app/Http/Controllers/DemandController.php index cbd214505..be9dd9952 100644 --- a/app/Http/Controllers/DemandController.php +++ b/app/Http/Controllers/DemandController.php @@ -41,6 +41,7 @@ use Adshares\Common\Exception\RuntimeException; use Adshares\Demand\Application\Service\PaymentDetailsVerify; use DateTime; +use DateTimeImmutable; use DateTimeInterface; use Illuminate\Database\Eloquent\ModelNotFoundException; use Illuminate\Http\Request; @@ -469,9 +470,9 @@ public function paymentDetails( ): JsonResponse { $transactionIdDecoded = AdsUtils::decodeTxId($transactionId); $accountAddressDecoded = AdsUtils::decodeAddress($accountAddress); - $datetime = DateTime::createFromFormat(DateTimeInterface::ATOM, $date); + $datetime = DateTimeImmutable::createFromFormat(DateTimeInterface::ATOM, $date); - if ($transactionIdDecoded === null || $accountAddressDecoded === null) { + if (null === $transactionIdDecoded || null === $accountAddressDecoded || false === $datetime) { throw new BadRequestHttpException('Input data are invalid.'); } diff --git a/src/Demand/Application/Service/PaymentDetailsVerify.php b/src/Demand/Application/Service/PaymentDetailsVerify.php index 80b175ee9..52a69b523 100644 --- a/src/Demand/Application/Service/PaymentDetailsVerify.php +++ b/src/Demand/Application/Service/PaymentDetailsVerify.php @@ -1,7 +1,7 @@ adsClient = $adsClient; } - public function verify(string $signature, string $transactionId, string $accountAddress, DateTime $date): bool - { + public function verify( + string $signature, + string $transactionId, + string $accountAddress, + DateTimeInterface $date, + ): bool { $publicKey = $this->adsClient->getPublicKeyByAccountAddress($accountAddress); return $this->signatureVerifier->verifyTransactionId( $publicKey, From 301589960bcbf0dffe1a9dc3f8555d0881c25713 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82Podkalicki?= Date: Wed, 1 Mar 2023 12:51:26 +0100 Subject: [PATCH 07/55] extract consts --- src/Common/Application/Factory/TaxonomyV2Factory.php | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/src/Common/Application/Factory/TaxonomyV2Factory.php b/src/Common/Application/Factory/TaxonomyV2Factory.php index 7cc1d92d8..3ef7169ae 100644 --- a/src/Common/Application/Factory/TaxonomyV2Factory.php +++ b/src/Common/Application/Factory/TaxonomyV2Factory.php @@ -1,7 +1,7 @@ $addValue, - 'path_fragments' => $pathFragments, + self::KEY_ADD_VALUE => $addValue, + self::KEY_PATH_FRAGMENTS => $pathFragments, ]; } @@ -218,7 +220,7 @@ private static function applyChange( ?array $value ): array { $temp = &$baseData[$key]; - foreach ($pathParsingResult['path_fragments'] as $pathFragment) { + foreach ($pathParsingResult[self::KEY_PATH_FRAGMENTS] as $pathFragment) { if (str_starts_with($pathFragment, '[?(@.') && str_ends_with($pathFragment, ')]')) { [$k, $v] = explode('=', substr($pathFragment, strlen('[?(@.'), -strlen(')]')), 2); for ($i = 0; $i < count($temp); $i++) { @@ -233,7 +235,7 @@ private static function applyChange( } $temp = &$temp[$pathFragment]; } - if ($pathParsingResult['add_value']) { + if ($pathParsingResult[self::KEY_ADD_VALUE]) { $temp[] = $value; } else { $temp = $value; From 539e3a867ed2659fa952a137381897174a93c49d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82Podkalicki?= Date: Fri, 3 Mar 2023 13:53:47 +0100 Subject: [PATCH 08/55] build info from response --- .../Supply/OpenRtbProviderRegistrar.php | 66 +++++--- .../Console/Commands/AdsFetchHostsTest.php | 2 +- .../ServerConfigurationControllerTest.php | 2 +- .../Supply/OpenRtbProviderRegistrarTest.php | 152 +++++++++++++++++- 4 files changed, 190 insertions(+), 32 deletions(-) diff --git a/app/Services/Supply/OpenRtbProviderRegistrar.php b/app/Services/Supply/OpenRtbProviderRegistrar.php index 15fc3516c..ff64266b8 100644 --- a/app/Services/Supply/OpenRtbProviderRegistrar.php +++ b/app/Services/Supply/OpenRtbProviderRegistrar.php @@ -26,40 +26,58 @@ use Adshares\Adserver\Models\NetworkHost; use Adshares\Common\Domain\ValueObject\AccountId; use Adshares\Common\Domain\ValueObject\Url; -use Adshares\Config\AppMode; -use Adshares\Config\RegistrationMode; -use Adshares\Supply\Application\Dto\Info; +use Adshares\Common\Exception\RuntimeException; +use Adshares\Supply\Application\Service\DemandClient; +use Adshares\Supply\Application\Service\Exception\UnexpectedClientResponseException; use DateTimeImmutable; use Illuminate\Support\Facades\Log; class OpenRtbProviderRegistrar { + private const OPEN_RTB_MODULE_NAME = 'openrtb'; + + public function __construct(private readonly DemandClient $demandClient) + { + } + public function registerAsNetworkHost(): bool { if ( - null !== ($accountAddress = config('app.open_rtb_provider_account_address')) - && null !== ($url = config('app.open_rtb_provider_url')) + null === ($accountAddress = config('app.open_rtb_provider_account_address')) + || null === ($url = config('app.open_rtb_provider_url')) ) { - $info = new Info( - 'openrtb', - 'OpenRTB Provider', - '0.1.0', - new Url($url), - new Url($url), - new Url($url), - new Url($url . '/policies/privacy.html'), - new Url($url . '/policies/terms.html'), - new Url($url . '/adshares/inventory/list'), - new AccountId($accountAddress), - null, - [Info::CAPABILITY_ADVERTISER], - RegistrationMode::PRIVATE, - AppMode::OPERATIONAL + return false; + } + + if (!AccountId::isValid($accountAddress, true)) { + Log::error('OpenRTB provider registration failed: configured account address is not valid'); + return false; + } + try { + $infoUrl = new Url($url); + } catch (RuntimeException $exception) { + Log::error(sprintf('OpenRTB provider registration failed: %s', $exception->getMessage())); + return false; + } + + try { + $info = $this->demandClient->fetchInfo($infoUrl); + } catch (UnexpectedClientResponseException $exception) { + Log::error(sprintf('OpenRTB provider registration failed: %s', $exception->getMessage())); + return false; + } + if ($info->getModule() !== self::OPEN_RTB_MODULE_NAME) { + Log::error( + sprintf('OpenRTB provider registration failed: Info for invalid module: %s', $info->getModule()) ); - $host = NetworkHost::registerHost($accountAddress, $url, $info, new DateTimeImmutable()); - Log::debug(sprintf('Stored %s as #%d', $url, $host->id)); - return true; + return false; + } + if ($info->getAdsAddress() !== $accountAddress) { + Log::error('OpenRTB provider registration failed: Info address does not match'); + return false; } - return false; + $host = NetworkHost::registerHost($accountAddress, $url, $info, new DateTimeImmutable()); + Log::debug(sprintf('Stored %s as #%d', $url, $host->id)); + return true; } } diff --git a/tests/app/Console/Commands/AdsFetchHostsTest.php b/tests/app/Console/Commands/AdsFetchHostsTest.php index 3ef872399..75c85aab7 100644 --- a/tests/app/Console/Commands/AdsFetchHostsTest.php +++ b/tests/app/Console/Commands/AdsFetchHostsTest.php @@ -267,7 +267,7 @@ function () { ); Config::updateAdminSettings([ Config::OPEN_RTB_PROVIDER_ACCOUNT_ADDRESS => '0001-00000001-8B4E', - Config::OPEN_RTB_PROVIDER_URL => 'https://example.com', + Config::OPEN_RTB_PROVIDER_URL => 'https://example.com/info.json', ]); self::artisan(self::COMMAND_SIGNATURE)->assertExitCode(0); diff --git a/tests/app/Http/Controllers/Manager/ServerConfigurationControllerTest.php b/tests/app/Http/Controllers/Manager/ServerConfigurationControllerTest.php index 4ebf26ceb..236991ab7 100644 --- a/tests/app/Http/Controllers/Manager/ServerConfigurationControllerTest.php +++ b/tests/app/Http/Controllers/Manager/ServerConfigurationControllerTest.php @@ -235,7 +235,7 @@ public function storeDataProvider(): array 'LANDING_URL' => [Config::LANDING_URL, 'https://example.com'], 'MAX_INVALID_LOGIN_ATTEMPTS' => [Config::MAX_INVALID_LOGIN_ATTEMPTS, '8'], 'OPEN_RTB_PROVIDER_ACCOUNT_ADDRESS' => [Config::OPEN_RTB_PROVIDER_ACCOUNT_ADDRESS, '0001-00000003-AB0C'], - 'OPEN_RTB_PROVIDER_URL' => [Config::OPEN_RTB_PROVIDER_URL, 'http://localhost:8015'], + 'OPEN_RTB_PROVIDER_URL' => [Config::OPEN_RTB_PROVIDER_URL, 'http://localhost:8015/info.json'], 'REFERRAL_REFUND_COMMISSION' => [Config::REFERRAL_REFUND_COMMISSION, '0'], 'REFERRAL_REFUND_ENABLED' => [Config::REFERRAL_REFUND_ENABLED, '1'], 'SUPPORT_EMAIL' => [Config::SUPPORT_EMAIL, 'sup@example.com'], diff --git a/tests/app/Services/Supply/OpenRtbProviderRegistrarTest.php b/tests/app/Services/Supply/OpenRtbProviderRegistrarTest.php index ba21cd224..8bccb2bed 100644 --- a/tests/app/Services/Supply/OpenRtbProviderRegistrarTest.php +++ b/tests/app/Services/Supply/OpenRtbProviderRegistrarTest.php @@ -28,23 +28,163 @@ use Adshares\Adserver\Services\Supply\OpenRtbProviderRegistrar; use Adshares\Adserver\Tests\TestCase; use Adshares\Adserver\Utilities\DatabaseConfigReader; +use Adshares\Common\Domain\ValueObject\AccountId; +use Adshares\Common\Domain\ValueObject\Url; +use Adshares\Config\AppMode; +use Adshares\Config\RegistrationMode; +use Adshares\Mock\Client\DummyDemandClient; +use Adshares\Supply\Application\Dto\Info; +use Adshares\Supply\Application\Service\DemandClient; +use Adshares\Supply\Application\Service\Exception\UnexpectedClientResponseException; +use PHPUnit\Framework\MockObject\MockObject; class OpenRtbProviderRegistrarTest extends TestCase { public function testRegisterAsNetworkHost(): void { Config::updateAdminSettings([ - Config::OPEN_RTB_PROVIDER_ACCOUNT_ADDRESS => '0001-00000001-8B4E', - Config::OPEN_RTB_PROVIDER_URL => 'https://example.com', + Config::OPEN_RTB_PROVIDER_ACCOUNT_ADDRESS => '0001-00000004-DBEB', + Config::OPEN_RTB_PROVIDER_URL => 'https://example.com/info.json', ]); DatabaseConfigReader::overwriteAdministrationConfig(); - $registrar = new OpenRtbProviderRegistrar(); + $registrar = new OpenRtbProviderRegistrar($this->getDemandClient()); - $registrar->registerAsNetworkHost(); + $result = $registrar->registerAsNetworkHost(); + self::assertTrue($result); self::assertDatabaseHas(NetworkHost::class, [ - 'address' => '0001-00000001-8B4E', - 'host' => 'https://example.com', + 'address' => '0001-00000004-DBEB', + 'host' => 'https://app.example.com', ]); } + + public function testRegisterAsNetworkHostFailWhileInvalidResponse(): void + { + Config::updateAdminSettings([ + Config::OPEN_RTB_PROVIDER_ACCOUNT_ADDRESS => '0001-00000004-DBEB', + Config::OPEN_RTB_PROVIDER_URL => 'https://example.com/info.json', + ]); + DatabaseConfigReader::overwriteAdministrationConfig(); + $clientMock = self::createMock(DemandClient::class); + $clientMock->method('fetchInfo')->willThrowException(new UnexpectedClientResponseException('test-exception')); + $registrar = new OpenRtbProviderRegistrar($clientMock); + + $result = $registrar->registerAsNetworkHost(); + + self::assertFalse($result); + } + + public function testRegisterAsNetworkHostFailWhileNoConfiguration(): void + { + $registrar = new OpenRtbProviderRegistrar($this->getDemandClient()); + + $result = $registrar->registerAsNetworkHost(); + + self::assertFalse($result); + } + + public function testRegisterAsNetworkHostFailWhileInvalidConfigurationAddress(): void + { + Config::updateAdminSettings([ + Config::OPEN_RTB_PROVIDER_ACCOUNT_ADDRESS => '0001-00000004', + Config::OPEN_RTB_PROVIDER_URL => 'https://example.com/info.json', + ]); + DatabaseConfigReader::overwriteAdministrationConfig(); + $registrar = new OpenRtbProviderRegistrar($this->getDemandClient()); + + $result = $registrar->registerAsNetworkHost(); + + self::assertFalse($result); + } + + public function testRegisterAsNetworkHostFailWhileInvalidConfigurationUrl(): void + { + Config::updateAdminSettings([ + Config::OPEN_RTB_PROVIDER_ACCOUNT_ADDRESS => '0001-00000004-DBEB', + Config::OPEN_RTB_PROVIDER_URL => 'example.com', + ]); + DatabaseConfigReader::overwriteAdministrationConfig(); + $registrar = new OpenRtbProviderRegistrar($this->getDemandClient()); + + $result = $registrar->registerAsNetworkHost(); + + self::assertFalse($result); + } + + public function testRegisterAsNetworkHostFailWhileInfoForAdserver(): void + { + Config::updateAdminSettings([ + Config::OPEN_RTB_PROVIDER_ACCOUNT_ADDRESS => '0001-00000004-DBEB', + Config::OPEN_RTB_PROVIDER_URL => 'https://example.com/info.json', + ]); + DatabaseConfigReader::overwriteAdministrationConfig(); + $registrar = new OpenRtbProviderRegistrar(new DummyDemandClient()); + + $result = $registrar->registerAsNetworkHost(); + + self::assertFalse($result); + self::assertDatabaseMissing(NetworkHost::class, [ + 'address' => '0001-00000004-DBEB', + 'host' => 'https://app.example.com', + ]); + } + + public function testRegisterAsNetworkHostFailWhileInfoForDifferentAddress(): void + { + Config::updateAdminSettings([ + Config::OPEN_RTB_PROVIDER_ACCOUNT_ADDRESS => '0001-00000004-DBEB', + Config::OPEN_RTB_PROVIDER_URL => 'https://example.com/info.json', + ]); + DatabaseConfigReader::overwriteAdministrationConfig(); + $info = new Info( + 'openrtb', + 'OpenRTB Provider ', + '0.1.0', + new Url('https://app.example.com'), + new Url('https://panel.example.com'), + new Url('https://example.com'), + new Url('https://example.com/privacy'), + new Url('https://example.com/terms'), + new Url('https://example.com/inventory'), + new AccountId('0001-00000005-CBCA'), + null, + [Info::CAPABILITY_ADVERTISER], + RegistrationMode::PRIVATE, + AppMode::OPERATIONAL + ); + $clientMock = self::createMock(DemandClient::class); + $clientMock->method('fetchInfo')->willReturn($info); + $registrar = new OpenRtbProviderRegistrar($clientMock); + + $result = $registrar->registerAsNetworkHost(); + + self::assertFalse($result); + self::assertDatabaseMissing(NetworkHost::class, [ + 'address' => '0001-00000004-DBEB', + 'host' => 'https://app.example.com', + ]); + } + + private function getDemandClient(): MockObject|DemandClient + { + $info = new Info( + 'openrtb', + 'OpenRTB Provider ', + '0.1.0', + new Url('https://app.example.com'), + new Url('https://panel.example.com'), + new Url('https://example.com'), + new Url('https://example.com/privacy'), + new Url('https://example.com/terms'), + new Url('https://example.com/inventory'), + new AccountId('0001-00000004-DBEB'), + null, + [Info::CAPABILITY_ADVERTISER], + RegistrationMode::PRIVATE, + AppMode::OPERATIONAL + ); + $clientMock = self::createMock(DemandClient::class); + $clientMock->method('fetchInfo')->willReturn($info); + return $clientMock; + } } From 42b99a5c89f05411c17d680bc409a717cfd2410d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82Podkalicki?= Date: Mon, 6 Mar 2023 12:42:31 +0100 Subject: [PATCH 09/55] fix dummy client --- tests/mock/Client/DummyAdSelectClient.php | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/mock/Client/DummyAdSelectClient.php b/tests/mock/Client/DummyAdSelectClient.php index e6604c9ab..2a0c1ed19 100644 --- a/tests/mock/Client/DummyAdSelectClient.php +++ b/tests/mock/Client/DummyAdSelectClient.php @@ -76,7 +76,7 @@ private function getBestBanners(array $zones, string $impressionId): array } $banners = []; - foreach ($bannerIds as $bannerId) { + foreach ($bannerIds as $index => $bannerId) { $banner = $bannerId ? NetworkBanner::where('uuid', hex2bin($bannerId))->first() : null; if (empty($banner)) { @@ -118,6 +118,7 @@ private function getBestBanners(array $zones, string $impressionId): array ), 'info_box' => true, 'rpm' => 0.5, + 'request_id' => (string)$index, ]; } } From ef4811bdaf1d09d546e609d70779c1d7c50de3c0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82Podkalicki?= Date: Mon, 6 Mar 2023 12:43:18 +0100 Subject: [PATCH 10/55] add serve url --- .../Manager/ServerConfigurationController.php | 1 + app/Models/Config.php | 2 + .../Supply/OpenRtbProviderRegistrar.php | 1 + .../Console/Commands/AdsFetchHostsTest.php | 1 + .../ServerConfigurationControllerTest.php | 1 + .../Supply/OpenRtbProviderRegistrarTest.php | 50 ++++++++----------- 6 files changed, 26 insertions(+), 30 deletions(-) diff --git a/app/Http/Controllers/Manager/ServerConfigurationController.php b/app/Http/Controllers/Manager/ServerConfigurationController.php index 35454d56b..170f3920f 100644 --- a/app/Http/Controllers/Manager/ServerConfigurationController.php +++ b/app/Http/Controllers/Manager/ServerConfigurationController.php @@ -148,6 +148,7 @@ class ServerConfigurationController extends Controller Config::NOW_PAYMENTS_MAX_AMOUNT => 'nullable|integer|min:0', Config::NOW_PAYMENTS_MIN_AMOUNT => 'nullable|integer|min:0', Config::OPEN_RTB_PROVIDER_ACCOUNT_ADDRESS => 'nullable|accountId', + Config::OPEN_RTB_PROVIDER_SERVE_URL => 'nullable|url', Config::OPEN_RTB_PROVIDER_URL => 'nullable|url', Config::OPERATOR_RX_FEE => 'nullable|commission', Config::OPERATOR_TX_FEE => 'nullable|commission', diff --git a/app/Models/Config.php b/app/Models/Config.php index 8d1fc7711..84f4e050a 100644 --- a/app/Models/Config.php +++ b/app/Models/Config.php @@ -151,6 +151,7 @@ class Config extends Model public const NOW_PAYMENTS_MAX_AMOUNT = 'now-payments-max-amount'; public const NOW_PAYMENTS_MIN_AMOUNT = 'now-payments-min-amount'; public const OPEN_RTB_PROVIDER_ACCOUNT_ADDRESS = 'open-rtb-provider-account-address'; + public const OPEN_RTB_PROVIDER_SERVE_URL = 'open-rtb-provider-serve-url'; public const OPEN_RTB_PROVIDER_URL = 'open-rtb-provider-url'; public const OPERATOR_RX_FEE = 'payment-rx-fee'; public const OPERATOR_TX_FEE = 'payment-tx-fee'; @@ -532,6 +533,7 @@ private static function getDefaultAdminSettings(array $fetched = []): array self::NOW_PAYMENTS_MAX_AMOUNT => 1000, self::NOW_PAYMENTS_MIN_AMOUNT => 25, self::OPEN_RTB_PROVIDER_ACCOUNT_ADDRESS => null, + self::OPEN_RTB_PROVIDER_SERVE_URL => null, self::OPEN_RTB_PROVIDER_URL => null, self::OPERATOR_RX_FEE => 0.01, self::OPERATOR_TX_FEE => 0.01, diff --git a/app/Services/Supply/OpenRtbProviderRegistrar.php b/app/Services/Supply/OpenRtbProviderRegistrar.php index ff64266b8..3abdb7e37 100644 --- a/app/Services/Supply/OpenRtbProviderRegistrar.php +++ b/app/Services/Supply/OpenRtbProviderRegistrar.php @@ -44,6 +44,7 @@ public function registerAsNetworkHost(): bool { if ( null === ($accountAddress = config('app.open_rtb_provider_account_address')) + || null === config('app.open_rtb_provider_serve_url') || null === ($url = config('app.open_rtb_provider_url')) ) { return false; diff --git a/tests/app/Console/Commands/AdsFetchHostsTest.php b/tests/app/Console/Commands/AdsFetchHostsTest.php index 75c85aab7..3fcef6336 100644 --- a/tests/app/Console/Commands/AdsFetchHostsTest.php +++ b/tests/app/Console/Commands/AdsFetchHostsTest.php @@ -267,6 +267,7 @@ function () { ); Config::updateAdminSettings([ Config::OPEN_RTB_PROVIDER_ACCOUNT_ADDRESS => '0001-00000001-8B4E', + Config::OPEN_RTB_PROVIDER_SERVE_URL => 'https://example.com/serve', Config::OPEN_RTB_PROVIDER_URL => 'https://example.com/info.json', ]); diff --git a/tests/app/Http/Controllers/Manager/ServerConfigurationControllerTest.php b/tests/app/Http/Controllers/Manager/ServerConfigurationControllerTest.php index 236991ab7..865c62791 100644 --- a/tests/app/Http/Controllers/Manager/ServerConfigurationControllerTest.php +++ b/tests/app/Http/Controllers/Manager/ServerConfigurationControllerTest.php @@ -235,6 +235,7 @@ public function storeDataProvider(): array 'LANDING_URL' => [Config::LANDING_URL, 'https://example.com'], 'MAX_INVALID_LOGIN_ATTEMPTS' => [Config::MAX_INVALID_LOGIN_ATTEMPTS, '8'], 'OPEN_RTB_PROVIDER_ACCOUNT_ADDRESS' => [Config::OPEN_RTB_PROVIDER_ACCOUNT_ADDRESS, '0001-00000003-AB0C'], + 'OPEN_RTB_PROVIDER_SERVE_URL' => [Config::OPEN_RTB_PROVIDER_SERVE_URL, 'http://localhost:8015/serve'], 'OPEN_RTB_PROVIDER_URL' => [Config::OPEN_RTB_PROVIDER_URL, 'http://localhost:8015/info.json'], 'REFERRAL_REFUND_COMMISSION' => [Config::REFERRAL_REFUND_COMMISSION, '0'], 'REFERRAL_REFUND_ENABLED' => [Config::REFERRAL_REFUND_ENABLED, '1'], diff --git a/tests/app/Services/Supply/OpenRtbProviderRegistrarTest.php b/tests/app/Services/Supply/OpenRtbProviderRegistrarTest.php index 8bccb2bed..a3a4df5fb 100644 --- a/tests/app/Services/Supply/OpenRtbProviderRegistrarTest.php +++ b/tests/app/Services/Supply/OpenRtbProviderRegistrarTest.php @@ -42,11 +42,7 @@ class OpenRtbProviderRegistrarTest extends TestCase { public function testRegisterAsNetworkHost(): void { - Config::updateAdminSettings([ - Config::OPEN_RTB_PROVIDER_ACCOUNT_ADDRESS => '0001-00000004-DBEB', - Config::OPEN_RTB_PROVIDER_URL => 'https://example.com/info.json', - ]); - DatabaseConfigReader::overwriteAdministrationConfig(); + $this->initOpenRtb(); $registrar = new OpenRtbProviderRegistrar($this->getDemandClient()); $result = $registrar->registerAsNetworkHost(); @@ -60,11 +56,7 @@ public function testRegisterAsNetworkHost(): void public function testRegisterAsNetworkHostFailWhileInvalidResponse(): void { - Config::updateAdminSettings([ - Config::OPEN_RTB_PROVIDER_ACCOUNT_ADDRESS => '0001-00000004-DBEB', - Config::OPEN_RTB_PROVIDER_URL => 'https://example.com/info.json', - ]); - DatabaseConfigReader::overwriteAdministrationConfig(); + $this->initOpenRtb(); $clientMock = self::createMock(DemandClient::class); $clientMock->method('fetchInfo')->willThrowException(new UnexpectedClientResponseException('test-exception')); $registrar = new OpenRtbProviderRegistrar($clientMock); @@ -85,11 +77,7 @@ public function testRegisterAsNetworkHostFailWhileNoConfiguration(): void public function testRegisterAsNetworkHostFailWhileInvalidConfigurationAddress(): void { - Config::updateAdminSettings([ - Config::OPEN_RTB_PROVIDER_ACCOUNT_ADDRESS => '0001-00000004', - Config::OPEN_RTB_PROVIDER_URL => 'https://example.com/info.json', - ]); - DatabaseConfigReader::overwriteAdministrationConfig(); + $this->initOpenRtb([Config::OPEN_RTB_PROVIDER_ACCOUNT_ADDRESS => '0001-00000004']); $registrar = new OpenRtbProviderRegistrar($this->getDemandClient()); $result = $registrar->registerAsNetworkHost(); @@ -99,11 +87,7 @@ public function testRegisterAsNetworkHostFailWhileInvalidConfigurationAddress(): public function testRegisterAsNetworkHostFailWhileInvalidConfigurationUrl(): void { - Config::updateAdminSettings([ - Config::OPEN_RTB_PROVIDER_ACCOUNT_ADDRESS => '0001-00000004-DBEB', - Config::OPEN_RTB_PROVIDER_URL => 'example.com', - ]); - DatabaseConfigReader::overwriteAdministrationConfig(); + $this->initOpenRtb([Config::OPEN_RTB_PROVIDER_URL => 'example.com']); $registrar = new OpenRtbProviderRegistrar($this->getDemandClient()); $result = $registrar->registerAsNetworkHost(); @@ -113,11 +97,7 @@ public function testRegisterAsNetworkHostFailWhileInvalidConfigurationUrl(): voi public function testRegisterAsNetworkHostFailWhileInfoForAdserver(): void { - Config::updateAdminSettings([ - Config::OPEN_RTB_PROVIDER_ACCOUNT_ADDRESS => '0001-00000004-DBEB', - Config::OPEN_RTB_PROVIDER_URL => 'https://example.com/info.json', - ]); - DatabaseConfigReader::overwriteAdministrationConfig(); + $this->initOpenRtb(); $registrar = new OpenRtbProviderRegistrar(new DummyDemandClient()); $result = $registrar->registerAsNetworkHost(); @@ -131,11 +111,7 @@ public function testRegisterAsNetworkHostFailWhileInfoForAdserver(): void public function testRegisterAsNetworkHostFailWhileInfoForDifferentAddress(): void { - Config::updateAdminSettings([ - Config::OPEN_RTB_PROVIDER_ACCOUNT_ADDRESS => '0001-00000004-DBEB', - Config::OPEN_RTB_PROVIDER_URL => 'https://example.com/info.json', - ]); - DatabaseConfigReader::overwriteAdministrationConfig(); + $this->initOpenRtb(); $info = new Info( 'openrtb', 'OpenRTB Provider ', @@ -187,4 +163,18 @@ private function getDemandClient(): MockObject|DemandClient $clientMock->method('fetchInfo')->willReturn($info); return $clientMock; } + + private function initOpenRtb(array $settings = []): void + { + $mergedSettings = array_merge( + [ + Config::OPEN_RTB_PROVIDER_ACCOUNT_ADDRESS => '0001-00000004-DBEB', + Config::OPEN_RTB_PROVIDER_SERVE_URL => 'https://example.com/serve', + Config::OPEN_RTB_PROVIDER_URL => 'https://example.com/info.json', + ], + $settings, + ); + Config::updateAdminSettings($mergedSettings); + DatabaseConfigReader::overwriteAdministrationConfig(); + } } From ca93f914b9dadf75e33c11cbf88cb8ec1f19a595 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82Podkalicki?= Date: Mon, 6 Mar 2023 13:01:21 +0100 Subject: [PATCH 11/55] replace open rtb banners --- app/Http/Controllers/SupplyController.php | 85 ++++++- .../Http/Controllers/SupplyControllerTest.php | 214 +++++++++++++++++- 2 files changed, 286 insertions(+), 13 deletions(-) diff --git a/app/Http/Controllers/SupplyController.php b/app/Http/Controllers/SupplyController.php index cd42e7bd2..8c93fa643 100644 --- a/app/Http/Controllers/SupplyController.php +++ b/app/Http/Controllers/SupplyController.php @@ -63,6 +63,7 @@ use Illuminate\Http\Request; use Illuminate\Http\Response; use Illuminate\Support\Facades\DB; +use Illuminate\Support\Facades\Http; use Illuminate\Support\Facades\Log; use Illuminate\Support\Facades\Storage; use Illuminate\View\View; @@ -433,7 +434,7 @@ private function findBanners( Request $request, Response $response, AdUser $contextProvider, - AdSelect $bannerFinder + AdSelect $bannerFinder, ): FoundBanners { $this->checkDecodedQueryData($decodedQueryData); $impressionId = $decodedQueryData['page']['iid']; @@ -476,8 +477,14 @@ private function findBanners( $context = Utils::mergeImpressionContextAndUserContext($impressionContext, $userContext); $impressionUuid = self::impressionIdToUuid($impressionId); $foundBanners = $bannerFinder->findBanners($zones, $context, $impressionUuid); + if ( + null !== config('app.open_rtb_provider_account_address') + && null !== config('app.open_rtb_provider_serve_url') + ) { + $foundBanners = $this->replaceOpenRtbBanners($foundBanners); + } - if ($foundBanners->exists(fn($key, $element) => $element != null)) { + if ($foundBanners->exists(fn($key, $element) => null !== $element)) { NetworkImpression::register( $impressionUuid, Utils::hexUuidFromBase64UrlWithChecksum($tid), @@ -1276,4 +1283,78 @@ private function buildCtx( } return Utils::encodeZones($ctx); } + + private function replaceOpenRtbBanners(FoundBanners $foundBanners): FoundBanners + { + $accountAddress = config('app.open_rtb_provider_account_address'); + $openBtbBanners = []; + foreach ($foundBanners as $index => $foundBanner) { + if (null !== $foundBanner && $accountAddress === $foundBanner['pay_from']) { + $openBtbBanners[(string)$index] = [ + 'request_id' => (string)$index, + 'serve_url' => $foundBanner['serve_url'], + ]; + } + } + if (empty($openBtbBanners)) { + return $foundBanners; + } + $response = Http::post(config('app.open_rtb_provider_serve_url'), $openBtbBanners); + if ( + BaseResponse::HTTP_OK !== $response->status() + || !$this->isOpenRtbAuctionResponseValid($content = $response->json(), $openBtbBanners) + ) { + foreach ($openBtbBanners as $index => $serveUrl) { + $foundBanners->set($index, null); + } + return $foundBanners; + } + foreach ($content as $entry) { + $foundBanner = array_merge( + $foundBanners->get((int)$entry['request_id']), + [ + 'click_url' => $entry['click_url'], + 'serve_url' => $entry['serve_url'], + 'view_url' => $entry['view_url'], + ] + ); + $foundBanners->set((int)$entry['request_id'], $foundBanner); + } + return $foundBanners; + } + + private function isOpenRtbAuctionResponseValid(mixed $content, array $openBtbBanners): bool + { + if (!is_array($content)) { + Log::error('Invalid OpenRTB response: body is not an array'); + return false; + } + foreach ($content as $entry) { + if (!is_array($entry)) { + Log::error('Invalid OpenRTB response: entry is not an array'); + return false; + } + $fields = [ + 'request_id', + 'click_url', + 'serve_url', + 'view_url', + ]; + foreach ($fields as $field) { + if (!isset($entry[$field])) { + Log::error(sprintf('Invalid OpenRTB response: missing key %s', $field)); + return false; + } + if (!is_string($entry[$field])) { + Log::error(sprintf('Invalid OpenRTB response: %s is not a string', $field)); + return false; + } + } + if (!array_key_exists($entry['request_id'], $openBtbBanners)) { + Log::error(sprintf('Invalid OpenRTB response: request %s is not known', $entry['request_id'])); + return false; + } + } + return true; + } } diff --git a/tests/app/Http/Controllers/SupplyControllerTest.php b/tests/app/Http/Controllers/SupplyControllerTest.php index d4c4c75e9..cbd7dd0bc 100644 --- a/tests/app/Http/Controllers/SupplyControllerTest.php +++ b/tests/app/Http/Controllers/SupplyControllerTest.php @@ -39,7 +39,9 @@ use Adshares\Adserver\Models\Zone; use Adshares\Adserver\Tests\TestCase; use Adshares\Adserver\Utilities\AdsAuthenticator; +use Adshares\Adserver\Utilities\AdsUtils; use Adshares\Common\Application\Service\AdUser; +use Adshares\Common\Domain\ValueObject\SecureUrl; use Adshares\Common\Domain\ValueObject\WalletAddress; use Adshares\Mock\Client\DummyAdUserClient; use Adshares\Supply\Application\Dto\FoundBanners; @@ -49,6 +51,7 @@ use GuzzleHttp\RequestOptions; use Illuminate\Support\Facades\Crypt; use Illuminate\Support\Facades\DB; +use Illuminate\Support\Facades\Http; use Illuminate\Support\Facades\Storage; use Psr\Http\Message\ResponseInterface; use Ramsey\Uuid\Uuid; @@ -237,17 +240,7 @@ private static function buildUriReportAd(string $caseId, string $bannerId): stri public function testFind(): void { $this->mockAdSelect(); - $adUser = self::createMock(AdUser::class); - $adUser->expects(self::once()) - ->method('getUserContext') - ->willReturnCallback(function ($context) { - self::assertInstanceOf(ImpressionContext::class, $context); - $contextArray = $context->toArray(); - self::assertEquals(1, $contextArray['device']['extensions']['metamask']); - self::assertEquals('good-user', $contextArray['user']['account']); - return (new DummyAdUserClient())->getUserContext($context); - }); - $this->instance(AdUser::class, $adUser); + $this->initAdUser(); /** @var User $user */ $user = User::factory()->create(['api_token' => '1234', 'auto_withdrawal' => 1e11]); /** @var Site $site */ @@ -277,6 +270,101 @@ public function testFind(): void $response->assertJsonPath('data.0.id', '3'); } + public function testFindOpenRTB(): void + { + Http::preventStrayRequests(); + Http::fake([ + 'example.com/serve' => Http::response([[ + 'request_id' => '0', + 'click_url' => 'https://example.com/click/1', + 'serve_url' => 'https://example.com/serve/1', + 'view_url' => 'https://example.com/view/1', + ]]), + ]); + $data = $this->initOpenRtb(); + + $response = $this->postJson(self::BANNER_FIND_URI, $data); + + $response->assertStatus(Response::HTTP_OK); + $response->assertJsonStructure(self::FIND_BANNER_STRUCTURE); + $response->assertJsonCount(1, 'data'); + $response->assertJsonPath('data.0.id', '3'); + Http::assertSentCount(1); + } + + public function testFindOpenRtbWhileInvalidStatus(): void + { + Http::preventStrayRequests(); + Http::fake(['example.com/serve' => Http::response(status: Response::HTTP_NOT_FOUND)]); + $data = $this->initOpenRtb(); + + $response = $this->postJson(self::BANNER_FIND_URI, $data); + + $response->assertStatus(Response::HTTP_OK); + $response->assertJsonStructure(self::FIND_BANNER_STRUCTURE); + $response->assertJsonCount(0, 'data'); + Http::assertSentCount(1); + } + + /** + * @dataProvider findOpenRtbWhileInvalidResponseProvider + */ + public function testFindOpenRtbWhileInvalidResponse(mixed $response): void + { + Http::preventStrayRequests(); + Http::fake([ + 'example.com/serve' => Http::response($response), + ]); + $data = $this->initOpenRtb(); + + $response = $this->postJson(self::BANNER_FIND_URI, $data); + + $response->assertStatus(Response::HTTP_OK); + $response->assertJsonStructure(self::FIND_BANNER_STRUCTURE); + $response->assertJsonCount(0, 'data'); + Http::assertSentCount(1); + } + + public function findOpenRtbWhileInvalidResponseProvider(): array + { + return [ + 'not existing request id' => [[[ + 'request_id' => '1', + 'click_url' => 'https://example.com/click/1', + 'serve_url' => 'https://example.com/serve/1', + 'view_url' => 'https://example.com/view/1', + ]]], + 'no request id' => [[[ + 'click_url' => 'https://example.com/click/1', + 'serve_url' => 'https://example.com/serve/1', + 'view_url' => 'https://example.com/view/1', + ]]], + 'no click url' => [[[ + 'request_id' => '0', + 'serve_url' => 'https://example.com/serve/1', + 'view_url' => 'https://example.com/view/1', + ]]], + 'no serve url' => [[[ + 'request_id' => '0', + 'click_url' => 'https://example.com/click/1', + 'view_url' => 'https://example.com/view/1', + ]]], + 'no view url' => [[[ + 'request_id' => '0', + 'click_url' => 'https://example.com/click/1', + 'serve_url' => 'https://example.com/serve/1', + ]]], + 'invalid serve url type' => [[[ + 'request_id' => '0', + 'click_url' => 'https://example.com/click/1', + 'serve_url' => 1234, + 'view_url' => 'https://example.com/view/1', + ]]], + 'entry is not array' => [['0']], + 'content is not array' => ['0'], + ]; + } + public function testFindWhileNoBanners(): void { $this->app->bind( @@ -960,4 +1048,108 @@ private static function initBeforeLoggingView(): array ]; return [$query, $banner, $zone]; } + + private function initAdUser(): void + { + $adUser = self::createMock(AdUser::class); + $adUser->expects(self::once()) + ->method('getUserContext') + ->willReturnCallback(function ($context) { + self::assertInstanceOf(ImpressionContext::class, $context); + $contextArray = $context->toArray(); + self::assertEquals(1, $contextArray['device']['extensions']['metamask']); + self::assertEquals('good-user', $contextArray['user']['account']); + return (new DummyAdUserClient())->getUserContext($context); + }); + $this->instance(AdUser::class, $adUser); + } + + private function initOpenRtb(): array + { + Config::updateAdminSettings([ + Config::OPEN_RTB_PROVIDER_ACCOUNT_ADDRESS => '0001-00000001-8B4E', + Config::OPEN_RTB_PROVIDER_SERVE_URL => 'https://example.com/serve', + Config::OPEN_RTB_PROVIDER_URL => 'https://example.com/info.json', + ]); + NetworkHost::factory()->create([ + 'address' => '0001-00000001-8B4E', + 'host' => 'https://example.com', + ]); + /** @var Zone $zone */ + $zone = Zone::factory()->create([ + 'site_id' => Site::factory()->create([ + 'user_id' => User::factory()->create(['api_token' => '1234', 'auto_withdrawal' => 1e11]), + 'status' => Site::STATUS_ACTIVE, + ]), + ]); + /** @var NetworkBanner $networkBanner */ + $networkBanner = NetworkBanner::factory()->create([ + 'network_campaign_id' => NetworkCampaign::factory()->create(), + 'serve_url' => 'https://example.com/serve/' . Uuid::uuid4()->toString(), + ]); + $impressionId = Uuid::uuid4(); + $adSelect = self::createMock(AdSelect::class); + $adSelect->method('findBanners')->willReturn( + new FoundBanners([ + [ + 'id' => $networkBanner->uuid, + 'publisher_id' => '0123456879ABCDEF0123456879ABCDEF', + 'zone_id' => $zone->uuid, + 'pay_from' => '0001-00000001-8B4E', + 'pay_to' => AdsUtils::normalizeAddress(config('app.adshares_address')), + 'type' => $networkBanner->type, + 'size' => $networkBanner->size, + 'serve_url' => $networkBanner->serve_url, + 'creative_sha1' => '', + 'click_url' => SecureUrl::change( + route( + 'log-network-click', + [ + 'id' => $networkBanner->uuid, + 'iid' => $impressionId, + 'r' => Utils::urlSafeBase64Encode($networkBanner->click_url), + 'zid' => $zone->uuid, + ] + ) + ), + 'view_url' => SecureUrl::change( + route( + 'log-network-view', + [ + 'id' => $networkBanner->uuid, + 'iid' => $impressionId, + 'r' => Utils::urlSafeBase64Encode($networkBanner->view_url), + 'zid' => $zone->uuid, + ] + ) + ), + 'info_box' => true, + 'rpm' => 0.5, + 'request_id' => '3', + ] + ]) + ); + $this->app->bind( + AdSelect::class, + static function () use ($adSelect) { + return $adSelect; + } + ); + $this->initAdUser(); + $data = [ + 'context' => [ + 'iid' => $impressionId, + 'url' => 'https://example.com', + 'metamask' => true, + 'uid' => 'good-user', + ], + 'placements' => [ + [ + 'id' => '3', + 'placementId' => $zone->uuid, + ], + ], + ]; + return $data; + } } From ea94f787ccb7808bae9c618a678345fe908ec1c7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82Podkalicki?= Date: Mon, 6 Mar 2023 13:26:49 +0100 Subject: [PATCH 12/55] fix test --- .../Http/Controllers/DemandControllerTest.php | 24 +++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/tests/app/Http/Controllers/DemandControllerTest.php b/tests/app/Http/Controllers/DemandControllerTest.php index b646d3653..9e7839a26 100644 --- a/tests/app/Http/Controllers/DemandControllerTest.php +++ b/tests/app/Http/Controllers/DemandControllerTest.php @@ -162,12 +162,12 @@ public function testInventoryList(): void $this->assertCount($activeCampaignsCount, $content); $this->assertEquals($campaignActive->uuid, $content[0]['id']); - $this->assertCount($activeBannersCount, $content[0]['banners']); - $this->assertEquals($bannerActive->uuid, $content[0]['banners'][0]['id']); - $this->assertEquals('829c3804401b0727f70f73d4415e162400cbe57b', $content[0]['banners'][0]['checksum']); + $this->assertCount($activeBannersCount, $content[0]['creatives']); + $this->assertEquals($bannerActive->uuid, $content[0]['creatives'][0]['id']); + $this->assertEquals('829c3804401b0727f70f73d4415e162400cbe57b', $content[0]['creatives'][0]['checksum']); $this->assertEquals( 'https://example.com/serve/x' . $bannerActive->uuid . '.doc?v=829c', - $content[0]['banners'][0]['serve_url'] + $content[0]['creatives'][0]['serve_url'] ); $this->assertEquals($campaignActive->medium, $content[0]['medium']); $this->assertEquals($campaignActive->vendor, $content[0]['vendor']); @@ -207,10 +207,10 @@ public function testInventoryListWithCdn(): void $this->assertCount(1, $content); $this->assertEquals($campaignActive->uuid, $content[0]['id']); - $this->assertCount(1, $content[0]['banners']); - $this->assertEquals($bannerActive->uuid, $content[0]['banners'][0]['id']); - $this->assertEquals('829c3804401b0727f70f73d4415e162400cbe57b', $content[0]['banners'][0]['checksum']); - $this->assertEquals('https://foo.com/file.png', $content[0]['banners'][0]['serve_url']); + $this->assertCount(1, $content[0]['creatives']); + $this->assertEquals($bannerActive->uuid, $content[0]['creatives'][0]['id']); + $this->assertEquals('829c3804401b0727f70f73d4415e162400cbe57b', $content[0]['creatives'][0]['checksum']); + $this->assertEquals('https://foo.com/file.png', $content[0]['creatives'][0]['serve_url']); //change content $bannerActive->creative_contents = 'foo content'; @@ -222,12 +222,12 @@ public function testInventoryListWithCdn(): void $this->assertCount(1, $content); $this->assertEquals($campaignActive->uuid, $content[0]['id']); - $this->assertCount(1, $content[0]['banners']); - $this->assertEquals($bannerActive->uuid, $content[0]['banners'][0]['id']); - $this->assertEquals('ec097bb2a51eb70410d13bbe94ef0319680accb6', $content[0]['banners'][0]['checksum']); + $this->assertCount(1, $content[0]['creatives']); + $this->assertEquals($bannerActive->uuid, $content[0]['creatives'][0]['id']); + $this->assertEquals('ec097bb2a51eb70410d13bbe94ef0319680accb6', $content[0]['creatives'][0]['checksum']); $this->assertEquals( 'https://example.com/serve/x' . $bannerActive->uuid . '.doc?v=ec09', - $content[0]['banners'][0]['serve_url'] + $content[0]['creatives'][0]['serve_url'] ); } From a39abc5403abfa45d5e87f4845379020d4676b7c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82Podkalicki?= Date: Mon, 6 Mar 2023 13:28:53 +0100 Subject: [PATCH 13/55] fix code style --- app/Models/Campaign.php | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/app/Models/Campaign.php b/app/Models/Campaign.php index 1783673f8..001eadfc4 100644 --- a/app/Models/Campaign.php +++ b/app/Models/Campaign.php @@ -513,11 +513,12 @@ public function isDirectDeal(): bool return false; } - if (MediumName::Metaverse->value === $this->medium) { - if (null !== ($vendor = MetaverseVendor::tryFrom($this->vendor))) { - $domains = $this->targeting_requires['site']['domain']; - return 1 !== count($domains) || $vendor->baseDomain() !== $domains[0]; - } + if ( + MediumName::Metaverse->value === $this->medium + && null !== ($vendor = MetaverseVendor::tryFrom($this->vendor)) + ) { + $domains = $this->targeting_requires['site']['domain']; + return 1 !== count($domains) || $vendor->baseDomain() !== $domains[0]; } return true; From 9cc30ff02f0cff6d420cbf130746c34e5993ebfe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82Podkalicki?= Date: Mon, 6 Mar 2023 14:07:48 +0100 Subject: [PATCH 14/55] null banner if not in response --- app/Http/Controllers/SupplyController.php | 4 ++++ .../Http/Controllers/SupplyControllerTest.php | 16 +++++++++++++++- 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/app/Http/Controllers/SupplyController.php b/app/Http/Controllers/SupplyController.php index 8c93fa643..e27e282af 100644 --- a/app/Http/Controllers/SupplyController.php +++ b/app/Http/Controllers/SupplyController.php @@ -1319,6 +1319,10 @@ private function replaceOpenRtbBanners(FoundBanners $foundBanners): FoundBanners ] ); $foundBanners->set((int)$entry['request_id'], $foundBanner); + unset($openBtbBanners[$entry['request_id']]); + } + foreach ($openBtbBanners as $index => $serveUrl) { + $foundBanners->set($index, null); } return $foundBanners; } diff --git a/tests/app/Http/Controllers/SupplyControllerTest.php b/tests/app/Http/Controllers/SupplyControllerTest.php index cbd7dd0bc..a8f56924e 100644 --- a/tests/app/Http/Controllers/SupplyControllerTest.php +++ b/tests/app/Http/Controllers/SupplyControllerTest.php @@ -270,7 +270,7 @@ public function testFind(): void $response->assertJsonPath('data.0.id', '3'); } - public function testFindOpenRTB(): void + public function testFindOpenRtb(): void { Http::preventStrayRequests(); Http::fake([ @@ -292,6 +292,20 @@ public function testFindOpenRTB(): void Http::assertSentCount(1); } + public function testFindOpenRtbWhileEmptyResponse(): void + { + Http::preventStrayRequests(); + Http::fake(['example.com/serve' => Http::response([])]); + $data = $this->initOpenRtb(); + + $response = $this->postJson(self::BANNER_FIND_URI, $data); + + $response->assertStatus(Response::HTTP_OK); + $response->assertJsonStructure(self::FIND_BANNER_STRUCTURE); + $response->assertJsonCount(0, 'data'); + Http::assertSentCount(1); + } + public function testFindOpenRtbWhileInvalidStatus(): void { Http::preventStrayRequests(); From 3d8eb136ffba0431642a71e0c859ac0f8e0bafd6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82Podkalicki?= Date: Wed, 8 Mar 2023 08:56:08 +0100 Subject: [PATCH 15/55] fetch by demand id --- app/Client/GuzzleAdSelectClient.php | 1 + app/Http/Controllers/SupplyController.php | 18 +++++++++--------- .../Http/Controllers/SupplyControllerTest.php | 1 + tests/mock/Client/DummyAdSelectClient.php | 1 + 4 files changed, 12 insertions(+), 9 deletions(-) diff --git a/app/Client/GuzzleAdSelectClient.php b/app/Client/GuzzleAdSelectClient.php index 7e74b8766..80cbb899b 100644 --- a/app/Client/GuzzleAdSelectClient.php +++ b/app/Client/GuzzleAdSelectClient.php @@ -303,6 +303,7 @@ private function fetchInOrderOfAppearance( $campaign = $banner->campaign; $data = [ 'id' => $bannerId, + 'demandId' => $banner->demand_banner_id, 'publisher_id' => $zone->site->user->uuid, 'zone_id' => $zone->uuid, 'pay_from' => $campaign->source_address, diff --git a/app/Http/Controllers/SupplyController.php b/app/Http/Controllers/SupplyController.php index e27e282af..a738f8c59 100644 --- a/app/Http/Controllers/SupplyController.php +++ b/app/Http/Controllers/SupplyController.php @@ -1287,24 +1287,24 @@ private function buildCtx( private function replaceOpenRtbBanners(FoundBanners $foundBanners): FoundBanners { $accountAddress = config('app.open_rtb_provider_account_address'); - $openBtbBanners = []; + $openRtbBanners = []; foreach ($foundBanners as $index => $foundBanner) { if (null !== $foundBanner && $accountAddress === $foundBanner['pay_from']) { - $openBtbBanners[(string)$index] = [ + $openRtbBanners[(string)$index] = [ 'request_id' => (string)$index, - 'serve_url' => $foundBanner['serve_url'], + 'id' => $foundBanner['demandId'], ]; } } - if (empty($openBtbBanners)) { + if (empty($openRtbBanners)) { return $foundBanners; } - $response = Http::post(config('app.open_rtb_provider_serve_url'), $openBtbBanners); + $response = Http::post(config('app.open_rtb_provider_serve_url'), $openRtbBanners); if ( BaseResponse::HTTP_OK !== $response->status() - || !$this->isOpenRtbAuctionResponseValid($content = $response->json(), $openBtbBanners) + || !$this->isOpenRtbAuctionResponseValid($content = $response->json(), $openRtbBanners) ) { - foreach ($openBtbBanners as $index => $serveUrl) { + foreach ($openRtbBanners as $index => $serveUrl) { $foundBanners->set($index, null); } return $foundBanners; @@ -1319,9 +1319,9 @@ private function replaceOpenRtbBanners(FoundBanners $foundBanners): FoundBanners ] ); $foundBanners->set((int)$entry['request_id'], $foundBanner); - unset($openBtbBanners[$entry['request_id']]); + unset($openRtbBanners[$entry['request_id']]); } - foreach ($openBtbBanners as $index => $serveUrl) { + foreach ($openRtbBanners as $index => $serveUrl) { $foundBanners->set($index, null); } return $foundBanners; diff --git a/tests/app/Http/Controllers/SupplyControllerTest.php b/tests/app/Http/Controllers/SupplyControllerTest.php index a8f56924e..a091e4c6a 100644 --- a/tests/app/Http/Controllers/SupplyControllerTest.php +++ b/tests/app/Http/Controllers/SupplyControllerTest.php @@ -1107,6 +1107,7 @@ private function initOpenRtb(): array new FoundBanners([ [ 'id' => $networkBanner->uuid, + 'demandId' => $networkBanner->demand_banner_id, 'publisher_id' => '0123456879ABCDEF0123456879ABCDEF', 'zone_id' => $zone->uuid, 'pay_from' => '0001-00000001-8B4E', diff --git a/tests/mock/Client/DummyAdSelectClient.php b/tests/mock/Client/DummyAdSelectClient.php index 2a0c1ed19..f265ef42a 100644 --- a/tests/mock/Client/DummyAdSelectClient.php +++ b/tests/mock/Client/DummyAdSelectClient.php @@ -86,6 +86,7 @@ private function getBestBanners(array $zones, string $impressionId): array $zoneId = $zoneData[$bannerId]['zone_id']; $banners[] = [ 'id' => $bannerId, + 'demandId' => $banner->demand_banner_id, 'publisher_id' => $zoneData[$bannerId]['publisher_id'], 'zone_id' => $zoneId, 'pay_from' => $campaign->source_address, From 6ac160edf994c338e583b9792e6b6a72107db2a3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82Podkalicki?= Date: Wed, 8 Mar 2023 09:01:10 +0100 Subject: [PATCH 16/55] rename provider to bridge --- app/Client/GuzzleDemandClient.php | 2 +- .../Manager/ServerConfigurationController.php | 6 +++--- app/Http/Controllers/SupplyController.php | 8 ++++---- app/Models/Config.php | 12 ++++++------ app/Services/Supply/OpenRtbProviderRegistrar.php | 6 +++--- tests/app/Console/Commands/AdsFetchHostsTest.php | 6 +++--- .../Manager/ServerConfigurationControllerTest.php | 6 +++--- tests/app/Http/Controllers/SupplyControllerTest.php | 6 +++--- .../Services/Supply/OpenRtbProviderRegistrarTest.php | 10 +++++----- 9 files changed, 31 insertions(+), 31 deletions(-) diff --git a/app/Client/GuzzleDemandClient.php b/app/Client/GuzzleDemandClient.php index 6d98fce6a..1cf332b6c 100644 --- a/app/Client/GuzzleDemandClient.php +++ b/app/Client/GuzzleDemandClient.php @@ -259,7 +259,7 @@ private function processData( 'updated_at' => DateTime::createFromFormat(DateTimeInterface::ATOM, $data['updated_at']), ]; - $isOpenRtbProvider = $sourceAddress === config('app.open_rtb_provider_account_address'); + $isOpenRtbProvider = $sourceAddress === config('app.open_rtb_bridge_account_address'); $classifiersRequired = $this->classifierRepository->fetchRequiredClassifiersNames(); $banners = []; $bannersInput = $data['creatives'] ?? $data['banners'];// legacy fallback, field 'banners' is deprecated diff --git a/app/Http/Controllers/Manager/ServerConfigurationController.php b/app/Http/Controllers/Manager/ServerConfigurationController.php index 170f3920f..797c26790 100644 --- a/app/Http/Controllers/Manager/ServerConfigurationController.php +++ b/app/Http/Controllers/Manager/ServerConfigurationController.php @@ -147,9 +147,9 @@ class ServerConfigurationController extends Controller Config::NOW_PAYMENTS_IPN_SECRET => 'nullable', Config::NOW_PAYMENTS_MAX_AMOUNT => 'nullable|integer|min:0', Config::NOW_PAYMENTS_MIN_AMOUNT => 'nullable|integer|min:0', - Config::OPEN_RTB_PROVIDER_ACCOUNT_ADDRESS => 'nullable|accountId', - Config::OPEN_RTB_PROVIDER_SERVE_URL => 'nullable|url', - Config::OPEN_RTB_PROVIDER_URL => 'nullable|url', + Config::OPEN_RTB_BRIDGE_ACCOUNT_ADDRESS => 'nullable|accountId', + Config::OPEN_RTB_BRIDGE_SERVE_URL => 'nullable|url', + Config::OPEN_RTB_BRIDGE_URL => 'nullable|url', Config::OPERATOR_RX_FEE => 'nullable|commission', Config::OPERATOR_TX_FEE => 'nullable|commission', Config::PUBLISHER_APPLY_FORM_URL => 'nullable|url', diff --git a/app/Http/Controllers/SupplyController.php b/app/Http/Controllers/SupplyController.php index a738f8c59..0b3b99feb 100644 --- a/app/Http/Controllers/SupplyController.php +++ b/app/Http/Controllers/SupplyController.php @@ -478,8 +478,8 @@ private function findBanners( $impressionUuid = self::impressionIdToUuid($impressionId); $foundBanners = $bannerFinder->findBanners($zones, $context, $impressionUuid); if ( - null !== config('app.open_rtb_provider_account_address') - && null !== config('app.open_rtb_provider_serve_url') + null !== config('app.open_rtb_bridge_account_address') + && null !== config('app.open_rtb_bridge_serve_url') ) { $foundBanners = $this->replaceOpenRtbBanners($foundBanners); } @@ -1286,7 +1286,7 @@ private function buildCtx( private function replaceOpenRtbBanners(FoundBanners $foundBanners): FoundBanners { - $accountAddress = config('app.open_rtb_provider_account_address'); + $accountAddress = config('app.open_rtb_bridge_account_address'); $openRtbBanners = []; foreach ($foundBanners as $index => $foundBanner) { if (null !== $foundBanner && $accountAddress === $foundBanner['pay_from']) { @@ -1299,7 +1299,7 @@ private function replaceOpenRtbBanners(FoundBanners $foundBanners): FoundBanners if (empty($openRtbBanners)) { return $foundBanners; } - $response = Http::post(config('app.open_rtb_provider_serve_url'), $openRtbBanners); + $response = Http::post(config('app.open_rtb_bridge_serve_url'), $openRtbBanners); if ( BaseResponse::HTTP_OK !== $response->status() || !$this->isOpenRtbAuctionResponseValid($content = $response->json(), $openRtbBanners) diff --git a/app/Models/Config.php b/app/Models/Config.php index 84f4e050a..354445f62 100644 --- a/app/Models/Config.php +++ b/app/Models/Config.php @@ -150,9 +150,9 @@ class Config extends Model public const NOW_PAYMENTS_IPN_SECRET = 'now-payments-ipn-secret'; public const NOW_PAYMENTS_MAX_AMOUNT = 'now-payments-max-amount'; public const NOW_PAYMENTS_MIN_AMOUNT = 'now-payments-min-amount'; - public const OPEN_RTB_PROVIDER_ACCOUNT_ADDRESS = 'open-rtb-provider-account-address'; - public const OPEN_RTB_PROVIDER_SERVE_URL = 'open-rtb-provider-serve-url'; - public const OPEN_RTB_PROVIDER_URL = 'open-rtb-provider-url'; + public const OPEN_RTB_BRIDGE_ACCOUNT_ADDRESS = 'open-rtb-bridge-account-address'; + public const OPEN_RTB_BRIDGE_SERVE_URL = 'open-rtb-bridge-serve-url'; + public const OPEN_RTB_BRIDGE_URL = 'open-rtb-bridge-url'; public const OPERATOR_RX_FEE = 'payment-rx-fee'; public const OPERATOR_TX_FEE = 'payment-tx-fee'; public const OPERATOR_WALLET_EMAIL_LAST_TIME = 'operator-wallet-transfer-email-time'; @@ -532,9 +532,9 @@ private static function getDefaultAdminSettings(array $fetched = []): array self::NOW_PAYMENTS_IPN_SECRET => '', self::NOW_PAYMENTS_MAX_AMOUNT => 1000, self::NOW_PAYMENTS_MIN_AMOUNT => 25, - self::OPEN_RTB_PROVIDER_ACCOUNT_ADDRESS => null, - self::OPEN_RTB_PROVIDER_SERVE_URL => null, - self::OPEN_RTB_PROVIDER_URL => null, + self::OPEN_RTB_BRIDGE_ACCOUNT_ADDRESS => null, + self::OPEN_RTB_BRIDGE_SERVE_URL => null, + self::OPEN_RTB_BRIDGE_URL => null, self::OPERATOR_RX_FEE => 0.01, self::OPERATOR_TX_FEE => 0.01, self::PUBLISHER_APPLY_FORM_URL => null, diff --git a/app/Services/Supply/OpenRtbProviderRegistrar.php b/app/Services/Supply/OpenRtbProviderRegistrar.php index 3abdb7e37..214633f8e 100644 --- a/app/Services/Supply/OpenRtbProviderRegistrar.php +++ b/app/Services/Supply/OpenRtbProviderRegistrar.php @@ -43,9 +43,9 @@ public function __construct(private readonly DemandClient $demandClient) public function registerAsNetworkHost(): bool { if ( - null === ($accountAddress = config('app.open_rtb_provider_account_address')) - || null === config('app.open_rtb_provider_serve_url') - || null === ($url = config('app.open_rtb_provider_url')) + null === ($accountAddress = config('app.open_rtb_bridge_account_address')) + || null === config('app.open_rtb_bridge_serve_url') + || null === ($url = config('app.open_rtb_bridge_url')) ) { return false; } diff --git a/tests/app/Console/Commands/AdsFetchHostsTest.php b/tests/app/Console/Commands/AdsFetchHostsTest.php index 3fcef6336..68195023a 100644 --- a/tests/app/Console/Commands/AdsFetchHostsTest.php +++ b/tests/app/Console/Commands/AdsFetchHostsTest.php @@ -266,9 +266,9 @@ function () { } ); Config::updateAdminSettings([ - Config::OPEN_RTB_PROVIDER_ACCOUNT_ADDRESS => '0001-00000001-8B4E', - Config::OPEN_RTB_PROVIDER_SERVE_URL => 'https://example.com/serve', - Config::OPEN_RTB_PROVIDER_URL => 'https://example.com/info.json', + Config::OPEN_RTB_BRIDGE_ACCOUNT_ADDRESS => '0001-00000001-8B4E', + Config::OPEN_RTB_BRIDGE_SERVE_URL => 'https://example.com/serve', + Config::OPEN_RTB_BRIDGE_URL => 'https://example.com/info.json', ]); self::artisan(self::COMMAND_SIGNATURE)->assertExitCode(0); diff --git a/tests/app/Http/Controllers/Manager/ServerConfigurationControllerTest.php b/tests/app/Http/Controllers/Manager/ServerConfigurationControllerTest.php index 865c62791..69a3ead44 100644 --- a/tests/app/Http/Controllers/Manager/ServerConfigurationControllerTest.php +++ b/tests/app/Http/Controllers/Manager/ServerConfigurationControllerTest.php @@ -234,9 +234,9 @@ public function storeDataProvider(): array 'INVENTORY_FAILED_CONNECTION_LIMIT' => [Config::INVENTORY_FAILED_CONNECTION_LIMIT, '8'], 'LANDING_URL' => [Config::LANDING_URL, 'https://example.com'], 'MAX_INVALID_LOGIN_ATTEMPTS' => [Config::MAX_INVALID_LOGIN_ATTEMPTS, '8'], - 'OPEN_RTB_PROVIDER_ACCOUNT_ADDRESS' => [Config::OPEN_RTB_PROVIDER_ACCOUNT_ADDRESS, '0001-00000003-AB0C'], - 'OPEN_RTB_PROVIDER_SERVE_URL' => [Config::OPEN_RTB_PROVIDER_SERVE_URL, 'http://localhost:8015/serve'], - 'OPEN_RTB_PROVIDER_URL' => [Config::OPEN_RTB_PROVIDER_URL, 'http://localhost:8015/info.json'], + 'OPEN_RTB_BRIDGE_ACCOUNT_ADDRESS' => [Config::OPEN_RTB_BRIDGE_ACCOUNT_ADDRESS, '0001-00000003-AB0C'], + 'OPEN_RTB_BRIDGE_SERVE_URL' => [Config::OPEN_RTB_BRIDGE_SERVE_URL, 'http://localhost:8015/serve'], + 'OPEN_RTB_BRIDGE_URL' => [Config::OPEN_RTB_BRIDGE_URL, 'http://localhost:8015/info.json'], 'REFERRAL_REFUND_COMMISSION' => [Config::REFERRAL_REFUND_COMMISSION, '0'], 'REFERRAL_REFUND_ENABLED' => [Config::REFERRAL_REFUND_ENABLED, '1'], 'SUPPORT_EMAIL' => [Config::SUPPORT_EMAIL, 'sup@example.com'], diff --git a/tests/app/Http/Controllers/SupplyControllerTest.php b/tests/app/Http/Controllers/SupplyControllerTest.php index a091e4c6a..71282eaf3 100644 --- a/tests/app/Http/Controllers/SupplyControllerTest.php +++ b/tests/app/Http/Controllers/SupplyControllerTest.php @@ -1081,9 +1081,9 @@ private function initAdUser(): void private function initOpenRtb(): array { Config::updateAdminSettings([ - Config::OPEN_RTB_PROVIDER_ACCOUNT_ADDRESS => '0001-00000001-8B4E', - Config::OPEN_RTB_PROVIDER_SERVE_URL => 'https://example.com/serve', - Config::OPEN_RTB_PROVIDER_URL => 'https://example.com/info.json', + Config::OPEN_RTB_BRIDGE_ACCOUNT_ADDRESS => '0001-00000001-8B4E', + Config::OPEN_RTB_BRIDGE_SERVE_URL => 'https://example.com/serve', + Config::OPEN_RTB_BRIDGE_URL => 'https://example.com/info.json', ]); NetworkHost::factory()->create([ 'address' => '0001-00000001-8B4E', diff --git a/tests/app/Services/Supply/OpenRtbProviderRegistrarTest.php b/tests/app/Services/Supply/OpenRtbProviderRegistrarTest.php index a3a4df5fb..83ee67b8c 100644 --- a/tests/app/Services/Supply/OpenRtbProviderRegistrarTest.php +++ b/tests/app/Services/Supply/OpenRtbProviderRegistrarTest.php @@ -77,7 +77,7 @@ public function testRegisterAsNetworkHostFailWhileNoConfiguration(): void public function testRegisterAsNetworkHostFailWhileInvalidConfigurationAddress(): void { - $this->initOpenRtb([Config::OPEN_RTB_PROVIDER_ACCOUNT_ADDRESS => '0001-00000004']); + $this->initOpenRtb([Config::OPEN_RTB_BRIDGE_ACCOUNT_ADDRESS => '0001-00000004']); $registrar = new OpenRtbProviderRegistrar($this->getDemandClient()); $result = $registrar->registerAsNetworkHost(); @@ -87,7 +87,7 @@ public function testRegisterAsNetworkHostFailWhileInvalidConfigurationAddress(): public function testRegisterAsNetworkHostFailWhileInvalidConfigurationUrl(): void { - $this->initOpenRtb([Config::OPEN_RTB_PROVIDER_URL => 'example.com']); + $this->initOpenRtb([Config::OPEN_RTB_BRIDGE_URL => 'example.com']); $registrar = new OpenRtbProviderRegistrar($this->getDemandClient()); $result = $registrar->registerAsNetworkHost(); @@ -168,9 +168,9 @@ private function initOpenRtb(array $settings = []): void { $mergedSettings = array_merge( [ - Config::OPEN_RTB_PROVIDER_ACCOUNT_ADDRESS => '0001-00000004-DBEB', - Config::OPEN_RTB_PROVIDER_SERVE_URL => 'https://example.com/serve', - Config::OPEN_RTB_PROVIDER_URL => 'https://example.com/info.json', + Config::OPEN_RTB_BRIDGE_ACCOUNT_ADDRESS => '0001-00000004-DBEB', + Config::OPEN_RTB_BRIDGE_SERVE_URL => 'https://example.com/serve', + Config::OPEN_RTB_BRIDGE_URL => 'https://example.com/info.json', ], $settings, ); From e9b13c25c8f1297e2376e4153eec5832eb19b400 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82Podkalicki?= Date: Wed, 8 Mar 2023 10:14:28 +0100 Subject: [PATCH 17/55] extract OpenRtbBridge --- app/Http/Controllers/SupplyController.php | 87 +--------------- app/Services/Demand/OpenRtbBridge.php | 121 ++++++++++++++++++++++ 2 files changed, 124 insertions(+), 84 deletions(-) create mode 100644 app/Services/Demand/OpenRtbBridge.php diff --git a/app/Http/Controllers/SupplyController.php b/app/Http/Controllers/SupplyController.php index 0b3b99feb..50ec7d4a4 100644 --- a/app/Http/Controllers/SupplyController.php +++ b/app/Http/Controllers/SupplyController.php @@ -35,6 +35,7 @@ use Adshares\Adserver\Models\User; use Adshares\Adserver\Models\Zone; use Adshares\Adserver\Rules\PayoutAddressRule; +use Adshares\Adserver\Services\Demand\OpenRtbBridge; use Adshares\Adserver\Utilities\AdsAuthenticator; use Adshares\Adserver\Utilities\AdsUtils; use Adshares\Adserver\Utilities\CssUtils; @@ -63,7 +64,6 @@ use Illuminate\Http\Request; use Illuminate\Http\Response; use Illuminate\Support\Facades\DB; -use Illuminate\Support\Facades\Http; use Illuminate\Support\Facades\Log; use Illuminate\Support\Facades\Storage; use Illuminate\View\View; @@ -477,11 +477,8 @@ private function findBanners( $context = Utils::mergeImpressionContextAndUserContext($impressionContext, $userContext); $impressionUuid = self::impressionIdToUuid($impressionId); $foundBanners = $bannerFinder->findBanners($zones, $context, $impressionUuid); - if ( - null !== config('app.open_rtb_bridge_account_address') - && null !== config('app.open_rtb_bridge_serve_url') - ) { - $foundBanners = $this->replaceOpenRtbBanners($foundBanners); + if (OpenRtbBridge::isActive()) { + $foundBanners = (new OpenRtbBridge())->replaceOpenRtbBanners($foundBanners, $context); } if ($foundBanners->exists(fn($key, $element) => null !== $element)) { @@ -1283,82 +1280,4 @@ private function buildCtx( } return Utils::encodeZones($ctx); } - - private function replaceOpenRtbBanners(FoundBanners $foundBanners): FoundBanners - { - $accountAddress = config('app.open_rtb_bridge_account_address'); - $openRtbBanners = []; - foreach ($foundBanners as $index => $foundBanner) { - if (null !== $foundBanner && $accountAddress === $foundBanner['pay_from']) { - $openRtbBanners[(string)$index] = [ - 'request_id' => (string)$index, - 'id' => $foundBanner['demandId'], - ]; - } - } - if (empty($openRtbBanners)) { - return $foundBanners; - } - $response = Http::post(config('app.open_rtb_bridge_serve_url'), $openRtbBanners); - if ( - BaseResponse::HTTP_OK !== $response->status() - || !$this->isOpenRtbAuctionResponseValid($content = $response->json(), $openRtbBanners) - ) { - foreach ($openRtbBanners as $index => $serveUrl) { - $foundBanners->set($index, null); - } - return $foundBanners; - } - foreach ($content as $entry) { - $foundBanner = array_merge( - $foundBanners->get((int)$entry['request_id']), - [ - 'click_url' => $entry['click_url'], - 'serve_url' => $entry['serve_url'], - 'view_url' => $entry['view_url'], - ] - ); - $foundBanners->set((int)$entry['request_id'], $foundBanner); - unset($openRtbBanners[$entry['request_id']]); - } - foreach ($openRtbBanners as $index => $serveUrl) { - $foundBanners->set($index, null); - } - return $foundBanners; - } - - private function isOpenRtbAuctionResponseValid(mixed $content, array $openBtbBanners): bool - { - if (!is_array($content)) { - Log::error('Invalid OpenRTB response: body is not an array'); - return false; - } - foreach ($content as $entry) { - if (!is_array($entry)) { - Log::error('Invalid OpenRTB response: entry is not an array'); - return false; - } - $fields = [ - 'request_id', - 'click_url', - 'serve_url', - 'view_url', - ]; - foreach ($fields as $field) { - if (!isset($entry[$field])) { - Log::error(sprintf('Invalid OpenRTB response: missing key %s', $field)); - return false; - } - if (!is_string($entry[$field])) { - Log::error(sprintf('Invalid OpenRTB response: %s is not a string', $field)); - return false; - } - } - if (!array_key_exists($entry['request_id'], $openBtbBanners)) { - Log::error(sprintf('Invalid OpenRTB response: request %s is not known', $entry['request_id'])); - return false; - } - } - return true; - } } diff --git a/app/Services/Demand/OpenRtbBridge.php b/app/Services/Demand/OpenRtbBridge.php new file mode 100644 index 000000000..4933d8422 --- /dev/null +++ b/app/Services/Demand/OpenRtbBridge.php @@ -0,0 +1,121 @@ + + */ + +namespace Adshares\Adserver\Services\Demand; + +use Adshares\Supply\Application\Dto\FoundBanners; +use Adshares\Supply\Application\Dto\ImpressionContext; +use Illuminate\Support\Facades\Http; +use Illuminate\Support\Facades\Log; +use Symfony\Component\HttpFoundation\Response as BaseResponse; + +class OpenRtbBridge +{ + public static function isActive(): bool + { + return null !== config('app.open_rtb_bridge_account_address') + && null !== config('app.open_rtb_bridge_serve_url'); + } + + public function replaceOpenRtbBanners(FoundBanners $foundBanners, ImpressionContext $context): FoundBanners + { + $accountAddress = config('app.open_rtb_bridge_account_address'); + $openRtbBanners = []; + foreach ($foundBanners as $index => $foundBanner) { + if (null !== $foundBanner && $accountAddress === $foundBanner['pay_from']) { + $openRtbBanners[(string)$index] = [ + 'request_id' => (string)$index, + 'id' => $foundBanner['demandId'], + ]; + } + } + if (empty($openRtbBanners)) { + return $foundBanners; + } + $response = Http::post( + config('app.open_rtb_bridge_serve_url'), + [ + 'context' => $context->toArray(), + 'requests' => $openRtbBanners + ], + ); + if ( + BaseResponse::HTTP_OK !== $response->status() + || !$this->isOpenRtbAuctionResponseValid($content = $response->json(), $openRtbBanners) + ) { + foreach ($openRtbBanners as $index => $serveUrl) { + $foundBanners->set($index, null); + } + return $foundBanners; + } + foreach ($content as $entry) { + $foundBanner = array_merge( + $foundBanners->get((int)$entry['request_id']), + [ + 'click_url' => $entry['click_url'], + 'serve_url' => $entry['serve_url'], + 'view_url' => $entry['view_url'], + ] + ); + $foundBanners->set((int)$entry['request_id'], $foundBanner); + unset($openRtbBanners[$entry['request_id']]); + } + foreach ($openRtbBanners as $index => $serveUrl) { + $foundBanners->set($index, null); + } + return $foundBanners; + } + + private function isOpenRtbAuctionResponseValid(mixed $content, array $openBtbBanners): bool + { + if (!is_array($content)) { + Log::error('Invalid OpenRTB response: body is not an array'); + return false; + } + foreach ($content as $entry) { + if (!is_array($entry)) { + Log::error('Invalid OpenRTB response: entry is not an array'); + return false; + } + $fields = [ + 'request_id', + 'click_url', + 'serve_url', + 'view_url', + ]; + foreach ($fields as $field) { + if (!isset($entry[$field])) { + Log::error(sprintf('Invalid OpenRTB response: missing key %s', $field)); + return false; + } + if (!is_string($entry[$field])) { + Log::error(sprintf('Invalid OpenRTB response: %s is not a string', $field)); + return false; + } + } + if (!array_key_exists($entry['request_id'], $openBtbBanners)) { + Log::error(sprintf('Invalid OpenRTB response: request %s is not known', $entry['request_id'])); + return false; + } + } + return true; + } +} From f7489e844e07c063989ce911c295ceb7106bab8b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82Podkalicki?= Date: Thu, 9 Mar 2023 12:36:34 +0100 Subject: [PATCH 18/55] change request --- app/Services/Demand/OpenRtbBridge.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/Services/Demand/OpenRtbBridge.php b/app/Services/Demand/OpenRtbBridge.php index 4933d8422..99223fd08 100644 --- a/app/Services/Demand/OpenRtbBridge.php +++ b/app/Services/Demand/OpenRtbBridge.php @@ -43,7 +43,7 @@ public function replaceOpenRtbBanners(FoundBanners $foundBanners, ImpressionCont if (null !== $foundBanner && $accountAddress === $foundBanner['pay_from']) { $openRtbBanners[(string)$index] = [ 'request_id' => (string)$index, - 'id' => $foundBanner['demandId'], + 'creative_id' => $foundBanner['demandId'], ]; } } From 70c4f836f8bf23b78d0d53a0461430a8f2dba813 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82Podkalicki?= Date: Fri, 10 Mar 2023 13:32:35 +0100 Subject: [PATCH 19/55] Remove serve url --- .../Controllers/Manager/ServerConfigurationController.php | 1 - app/Models/Config.php | 2 -- app/Services/Demand/OpenRtbBridge.php | 4 ++-- app/Services/Supply/OpenRtbProviderRegistrar.php | 2 +- tests/app/Console/Commands/AdsFetchHostsTest.php | 3 +-- .../Controllers/Manager/ServerConfigurationControllerTest.php | 3 +-- tests/app/Http/Controllers/SupplyControllerTest.php | 3 +-- tests/app/Services/Supply/OpenRtbProviderRegistrarTest.php | 3 +-- 8 files changed, 7 insertions(+), 14 deletions(-) diff --git a/app/Http/Controllers/Manager/ServerConfigurationController.php b/app/Http/Controllers/Manager/ServerConfigurationController.php index 797c26790..fc5d5864e 100644 --- a/app/Http/Controllers/Manager/ServerConfigurationController.php +++ b/app/Http/Controllers/Manager/ServerConfigurationController.php @@ -148,7 +148,6 @@ class ServerConfigurationController extends Controller Config::NOW_PAYMENTS_MAX_AMOUNT => 'nullable|integer|min:0', Config::NOW_PAYMENTS_MIN_AMOUNT => 'nullable|integer|min:0', Config::OPEN_RTB_BRIDGE_ACCOUNT_ADDRESS => 'nullable|accountId', - Config::OPEN_RTB_BRIDGE_SERVE_URL => 'nullable|url', Config::OPEN_RTB_BRIDGE_URL => 'nullable|url', Config::OPERATOR_RX_FEE => 'nullable|commission', Config::OPERATOR_TX_FEE => 'nullable|commission', diff --git a/app/Models/Config.php b/app/Models/Config.php index 354445f62..f7bb7b316 100644 --- a/app/Models/Config.php +++ b/app/Models/Config.php @@ -151,7 +151,6 @@ class Config extends Model public const NOW_PAYMENTS_MAX_AMOUNT = 'now-payments-max-amount'; public const NOW_PAYMENTS_MIN_AMOUNT = 'now-payments-min-amount'; public const OPEN_RTB_BRIDGE_ACCOUNT_ADDRESS = 'open-rtb-bridge-account-address'; - public const OPEN_RTB_BRIDGE_SERVE_URL = 'open-rtb-bridge-serve-url'; public const OPEN_RTB_BRIDGE_URL = 'open-rtb-bridge-url'; public const OPERATOR_RX_FEE = 'payment-rx-fee'; public const OPERATOR_TX_FEE = 'payment-tx-fee'; @@ -533,7 +532,6 @@ private static function getDefaultAdminSettings(array $fetched = []): array self::NOW_PAYMENTS_MAX_AMOUNT => 1000, self::NOW_PAYMENTS_MIN_AMOUNT => 25, self::OPEN_RTB_BRIDGE_ACCOUNT_ADDRESS => null, - self::OPEN_RTB_BRIDGE_SERVE_URL => null, self::OPEN_RTB_BRIDGE_URL => null, self::OPERATOR_RX_FEE => 0.01, self::OPERATOR_TX_FEE => 0.01, diff --git a/app/Services/Demand/OpenRtbBridge.php b/app/Services/Demand/OpenRtbBridge.php index 99223fd08..63c6e89f8 100644 --- a/app/Services/Demand/OpenRtbBridge.php +++ b/app/Services/Demand/OpenRtbBridge.php @@ -32,7 +32,7 @@ class OpenRtbBridge public static function isActive(): bool { return null !== config('app.open_rtb_bridge_account_address') - && null !== config('app.open_rtb_bridge_serve_url'); + && null !== config('app.open_rtb_bridge_url'); } public function replaceOpenRtbBanners(FoundBanners $foundBanners, ImpressionContext $context): FoundBanners @@ -51,7 +51,7 @@ public function replaceOpenRtbBanners(FoundBanners $foundBanners, ImpressionCont return $foundBanners; } $response = Http::post( - config('app.open_rtb_bridge_serve_url'), + config('app.open_rtb_bridge_url') . '/serve', [ 'context' => $context->toArray(), 'requests' => $openRtbBanners diff --git a/app/Services/Supply/OpenRtbProviderRegistrar.php b/app/Services/Supply/OpenRtbProviderRegistrar.php index 214633f8e..0604ebb48 100644 --- a/app/Services/Supply/OpenRtbProviderRegistrar.php +++ b/app/Services/Supply/OpenRtbProviderRegistrar.php @@ -44,11 +44,11 @@ public function registerAsNetworkHost(): bool { if ( null === ($accountAddress = config('app.open_rtb_bridge_account_address')) - || null === config('app.open_rtb_bridge_serve_url') || null === ($url = config('app.open_rtb_bridge_url')) ) { return false; } + $url = $url . '/info.json'; if (!AccountId::isValid($accountAddress, true)) { Log::error('OpenRTB provider registration failed: configured account address is not valid'); diff --git a/tests/app/Console/Commands/AdsFetchHostsTest.php b/tests/app/Console/Commands/AdsFetchHostsTest.php index 68195023a..eae1adc66 100644 --- a/tests/app/Console/Commands/AdsFetchHostsTest.php +++ b/tests/app/Console/Commands/AdsFetchHostsTest.php @@ -267,8 +267,7 @@ function () { ); Config::updateAdminSettings([ Config::OPEN_RTB_BRIDGE_ACCOUNT_ADDRESS => '0001-00000001-8B4E', - Config::OPEN_RTB_BRIDGE_SERVE_URL => 'https://example.com/serve', - Config::OPEN_RTB_BRIDGE_URL => 'https://example.com/info.json', + Config::OPEN_RTB_BRIDGE_URL => 'https://example.com', ]); self::artisan(self::COMMAND_SIGNATURE)->assertExitCode(0); diff --git a/tests/app/Http/Controllers/Manager/ServerConfigurationControllerTest.php b/tests/app/Http/Controllers/Manager/ServerConfigurationControllerTest.php index 69a3ead44..254ce72db 100644 --- a/tests/app/Http/Controllers/Manager/ServerConfigurationControllerTest.php +++ b/tests/app/Http/Controllers/Manager/ServerConfigurationControllerTest.php @@ -235,8 +235,7 @@ public function storeDataProvider(): array 'LANDING_URL' => [Config::LANDING_URL, 'https://example.com'], 'MAX_INVALID_LOGIN_ATTEMPTS' => [Config::MAX_INVALID_LOGIN_ATTEMPTS, '8'], 'OPEN_RTB_BRIDGE_ACCOUNT_ADDRESS' => [Config::OPEN_RTB_BRIDGE_ACCOUNT_ADDRESS, '0001-00000003-AB0C'], - 'OPEN_RTB_BRIDGE_SERVE_URL' => [Config::OPEN_RTB_BRIDGE_SERVE_URL, 'http://localhost:8015/serve'], - 'OPEN_RTB_BRIDGE_URL' => [Config::OPEN_RTB_BRIDGE_URL, 'http://localhost:8015/info.json'], + 'OPEN_RTB_BRIDGE_URL' => [Config::OPEN_RTB_BRIDGE_URL, 'http://localhost:8015'], 'REFERRAL_REFUND_COMMISSION' => [Config::REFERRAL_REFUND_COMMISSION, '0'], 'REFERRAL_REFUND_ENABLED' => [Config::REFERRAL_REFUND_ENABLED, '1'], 'SUPPORT_EMAIL' => [Config::SUPPORT_EMAIL, 'sup@example.com'], diff --git a/tests/app/Http/Controllers/SupplyControllerTest.php b/tests/app/Http/Controllers/SupplyControllerTest.php index 0d3e18da8..02a35df22 100644 --- a/tests/app/Http/Controllers/SupplyControllerTest.php +++ b/tests/app/Http/Controllers/SupplyControllerTest.php @@ -1117,8 +1117,7 @@ private function initOpenRtb(): array { Config::updateAdminSettings([ Config::OPEN_RTB_BRIDGE_ACCOUNT_ADDRESS => '0001-00000001-8B4E', - Config::OPEN_RTB_BRIDGE_SERVE_URL => 'https://example.com/serve', - Config::OPEN_RTB_BRIDGE_URL => 'https://example.com/info.json', + Config::OPEN_RTB_BRIDGE_URL => 'https://example.com', ]); NetworkHost::factory()->create([ 'address' => '0001-00000001-8B4E', diff --git a/tests/app/Services/Supply/OpenRtbProviderRegistrarTest.php b/tests/app/Services/Supply/OpenRtbProviderRegistrarTest.php index 83ee67b8c..35aee0a22 100644 --- a/tests/app/Services/Supply/OpenRtbProviderRegistrarTest.php +++ b/tests/app/Services/Supply/OpenRtbProviderRegistrarTest.php @@ -169,8 +169,7 @@ private function initOpenRtb(array $settings = []): void $mergedSettings = array_merge( [ Config::OPEN_RTB_BRIDGE_ACCOUNT_ADDRESS => '0001-00000004-DBEB', - Config::OPEN_RTB_BRIDGE_SERVE_URL => 'https://example.com/serve', - Config::OPEN_RTB_BRIDGE_URL => 'https://example.com/info.json', + Config::OPEN_RTB_BRIDGE_URL => 'https://example.com', ], $settings, ); From 67c1869b2d4d2c5477f18b284c5a6eb86fee3ea3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82Podkalicki?= Date: Fri, 10 Mar 2023 15:35:59 +0100 Subject: [PATCH 20/55] rename class --- app/Console/Commands/AdsFetchHosts.php | 4 +- app/Http/Controllers/SupplyController.php | 2 +- .../{Demand => Supply}/OpenRtbBridge.php | 6 +- ...gistrar.php => OpenRtbBridgeRegistrar.php} | 5 +- .../Console/Commands/AdsFetchHostsTest.php | 6 +- ...est.php => OpenRtbBridgeRegistrarTest.php} | 18 +++--- .../app/Services/Supply/OpenRtbBridgeTest.php | 57 +++++++++++++++++++ 7 files changed, 79 insertions(+), 19 deletions(-) rename app/Services/{Demand => Supply}/OpenRtbBridge.php (96%) rename app/Services/Supply/{OpenRtbProviderRegistrar.php => OpenRtbBridgeRegistrar.php} (96%) rename tests/app/Services/Supply/{OpenRtbProviderRegistrarTest.php => OpenRtbBridgeRegistrarTest.php} (89%) create mode 100644 tests/app/Services/Supply/OpenRtbBridgeTest.php diff --git a/app/Console/Commands/AdsFetchHosts.php b/app/Console/Commands/AdsFetchHosts.php index c8856bef6..bfa153246 100644 --- a/app/Console/Commands/AdsFetchHosts.php +++ b/app/Console/Commands/AdsFetchHosts.php @@ -31,7 +31,7 @@ use Adshares\Adserver\Events\ServerEvent; use Adshares\Adserver\Http\Response\InfoResponse; use Adshares\Adserver\Models\NetworkHost; -use Adshares\Adserver\Services\Supply\OpenRtbProviderRegistrar; +use Adshares\Adserver\Services\Supply\OpenRtbBridgeRegistrar; use Adshares\Adserver\ViewModel\ServerEventType; use Adshares\Common\Exception\RuntimeException; use Adshares\Config\AppMode; @@ -60,7 +60,7 @@ public function __construct( Locker $locker, private readonly AdsClient $adsClient, private readonly DemandClient $client, - private readonly OpenRtbProviderRegistrar $openRtbProviderRegistrar, + private readonly OpenRtbBridgeRegistrar $openRtbProviderRegistrar, ) { parent::__construct($locker); } diff --git a/app/Http/Controllers/SupplyController.php b/app/Http/Controllers/SupplyController.php index 9f2237c9d..6201b9d6a 100644 --- a/app/Http/Controllers/SupplyController.php +++ b/app/Http/Controllers/SupplyController.php @@ -35,7 +35,7 @@ use Adshares\Adserver\Models\User; use Adshares\Adserver\Models\Zone; use Adshares\Adserver\Rules\PayoutAddressRule; -use Adshares\Adserver\Services\Demand\OpenRtbBridge; +use Adshares\Adserver\Services\Supply\OpenRtbBridge; use Adshares\Adserver\Utilities\AdsAuthenticator; use Adshares\Adserver\Utilities\AdsUtils; use Adshares\Adserver\Utilities\CssUtils; diff --git a/app/Services/Demand/OpenRtbBridge.php b/app/Services/Supply/OpenRtbBridge.php similarity index 96% rename from app/Services/Demand/OpenRtbBridge.php rename to app/Services/Supply/OpenRtbBridge.php index 63c6e89f8..22cda71d5 100644 --- a/app/Services/Demand/OpenRtbBridge.php +++ b/app/Services/Supply/OpenRtbBridge.php @@ -19,7 +19,7 @@ * along with AdServer. If not, see */ -namespace Adshares\Adserver\Services\Demand; +namespace Adshares\Adserver\Services\Supply; use Adshares\Supply\Application\Dto\FoundBanners; use Adshares\Supply\Application\Dto\ImpressionContext; @@ -29,6 +29,8 @@ class OpenRtbBridge { + private const SERVE_PATH = '/serve'; + public static function isActive(): bool { return null !== config('app.open_rtb_bridge_account_address') @@ -51,7 +53,7 @@ public function replaceOpenRtbBanners(FoundBanners $foundBanners, ImpressionCont return $foundBanners; } $response = Http::post( - config('app.open_rtb_bridge_url') . '/serve', + config('app.open_rtb_bridge_url') . self::SERVE_PATH, [ 'context' => $context->toArray(), 'requests' => $openRtbBanners diff --git a/app/Services/Supply/OpenRtbProviderRegistrar.php b/app/Services/Supply/OpenRtbBridgeRegistrar.php similarity index 96% rename from app/Services/Supply/OpenRtbProviderRegistrar.php rename to app/Services/Supply/OpenRtbBridgeRegistrar.php index 0604ebb48..a5c67ff82 100644 --- a/app/Services/Supply/OpenRtbProviderRegistrar.php +++ b/app/Services/Supply/OpenRtbBridgeRegistrar.php @@ -32,8 +32,9 @@ use DateTimeImmutable; use Illuminate\Support\Facades\Log; -class OpenRtbProviderRegistrar +class OpenRtbBridgeRegistrar { + private const INFO_JSON_PATH = '/info.json'; private const OPEN_RTB_MODULE_NAME = 'openrtb'; public function __construct(private readonly DemandClient $demandClient) @@ -48,7 +49,7 @@ public function registerAsNetworkHost(): bool ) { return false; } - $url = $url . '/info.json'; + $url = $url . self::INFO_JSON_PATH; if (!AccountId::isValid($accountAddress, true)) { Log::error('OpenRTB provider registration failed: configured account address is not valid'); diff --git a/tests/app/Console/Commands/AdsFetchHostsTest.php b/tests/app/Console/Commands/AdsFetchHostsTest.php index eae1adc66..184c378c9 100644 --- a/tests/app/Console/Commands/AdsFetchHostsTest.php +++ b/tests/app/Console/Commands/AdsFetchHostsTest.php @@ -29,7 +29,7 @@ use Adshares\Adserver\Console\Locker; use Adshares\Adserver\Models\Config; use Adshares\Adserver\Models\NetworkHost; -use Adshares\Adserver\Services\Supply\OpenRtbProviderRegistrar; +use Adshares\Adserver\Services\Supply\OpenRtbBridgeRegistrar; use Adshares\Adserver\Tests\Console\ConsoleTestCase; use Adshares\Adserver\ViewModel\ServerEventType; use Adshares\Config\AppMode; @@ -254,9 +254,9 @@ function () { } ); $this->app->bind( - OpenRtbProviderRegistrar::class, + OpenRtbBridgeRegistrar::class, function () { - $mock = self::createMock(OpenRtbProviderRegistrar::class); + $mock = self::createMock(OpenRtbBridgeRegistrar::class); $mock->method('registerAsNetworkHost') ->will($this->returnCallback(function () { NetworkHost::factory()->create(); diff --git a/tests/app/Services/Supply/OpenRtbProviderRegistrarTest.php b/tests/app/Services/Supply/OpenRtbBridgeRegistrarTest.php similarity index 89% rename from tests/app/Services/Supply/OpenRtbProviderRegistrarTest.php rename to tests/app/Services/Supply/OpenRtbBridgeRegistrarTest.php index 35aee0a22..403976d97 100644 --- a/tests/app/Services/Supply/OpenRtbProviderRegistrarTest.php +++ b/tests/app/Services/Supply/OpenRtbBridgeRegistrarTest.php @@ -25,7 +25,7 @@ use Adshares\Adserver\Models\Config; use Adshares\Adserver\Models\NetworkHost; -use Adshares\Adserver\Services\Supply\OpenRtbProviderRegistrar; +use Adshares\Adserver\Services\Supply\OpenRtbBridgeRegistrar; use Adshares\Adserver\Tests\TestCase; use Adshares\Adserver\Utilities\DatabaseConfigReader; use Adshares\Common\Domain\ValueObject\AccountId; @@ -38,12 +38,12 @@ use Adshares\Supply\Application\Service\Exception\UnexpectedClientResponseException; use PHPUnit\Framework\MockObject\MockObject; -class OpenRtbProviderRegistrarTest extends TestCase +class OpenRtbBridgeRegistrarTest extends TestCase { public function testRegisterAsNetworkHost(): void { $this->initOpenRtb(); - $registrar = new OpenRtbProviderRegistrar($this->getDemandClient()); + $registrar = new OpenRtbBridgeRegistrar($this->getDemandClient()); $result = $registrar->registerAsNetworkHost(); @@ -59,7 +59,7 @@ public function testRegisterAsNetworkHostFailWhileInvalidResponse(): void $this->initOpenRtb(); $clientMock = self::createMock(DemandClient::class); $clientMock->method('fetchInfo')->willThrowException(new UnexpectedClientResponseException('test-exception')); - $registrar = new OpenRtbProviderRegistrar($clientMock); + $registrar = new OpenRtbBridgeRegistrar($clientMock); $result = $registrar->registerAsNetworkHost(); @@ -68,7 +68,7 @@ public function testRegisterAsNetworkHostFailWhileInvalidResponse(): void public function testRegisterAsNetworkHostFailWhileNoConfiguration(): void { - $registrar = new OpenRtbProviderRegistrar($this->getDemandClient()); + $registrar = new OpenRtbBridgeRegistrar($this->getDemandClient()); $result = $registrar->registerAsNetworkHost(); @@ -78,7 +78,7 @@ public function testRegisterAsNetworkHostFailWhileNoConfiguration(): void public function testRegisterAsNetworkHostFailWhileInvalidConfigurationAddress(): void { $this->initOpenRtb([Config::OPEN_RTB_BRIDGE_ACCOUNT_ADDRESS => '0001-00000004']); - $registrar = new OpenRtbProviderRegistrar($this->getDemandClient()); + $registrar = new OpenRtbBridgeRegistrar($this->getDemandClient()); $result = $registrar->registerAsNetworkHost(); @@ -88,7 +88,7 @@ public function testRegisterAsNetworkHostFailWhileInvalidConfigurationAddress(): public function testRegisterAsNetworkHostFailWhileInvalidConfigurationUrl(): void { $this->initOpenRtb([Config::OPEN_RTB_BRIDGE_URL => 'example.com']); - $registrar = new OpenRtbProviderRegistrar($this->getDemandClient()); + $registrar = new OpenRtbBridgeRegistrar($this->getDemandClient()); $result = $registrar->registerAsNetworkHost(); @@ -98,7 +98,7 @@ public function testRegisterAsNetworkHostFailWhileInvalidConfigurationUrl(): voi public function testRegisterAsNetworkHostFailWhileInfoForAdserver(): void { $this->initOpenRtb(); - $registrar = new OpenRtbProviderRegistrar(new DummyDemandClient()); + $registrar = new OpenRtbBridgeRegistrar(new DummyDemandClient()); $result = $registrar->registerAsNetworkHost(); @@ -130,7 +130,7 @@ public function testRegisterAsNetworkHostFailWhileInfoForDifferentAddress(): voi ); $clientMock = self::createMock(DemandClient::class); $clientMock->method('fetchInfo')->willReturn($info); - $registrar = new OpenRtbProviderRegistrar($clientMock); + $registrar = new OpenRtbBridgeRegistrar($clientMock); $result = $registrar->registerAsNetworkHost(); diff --git a/tests/app/Services/Supply/OpenRtbBridgeTest.php b/tests/app/Services/Supply/OpenRtbBridgeTest.php new file mode 100644 index 000000000..796d78ae3 --- /dev/null +++ b/tests/app/Services/Supply/OpenRtbBridgeTest.php @@ -0,0 +1,57 @@ + + */ + +declare(strict_types=1); + +namespace Adshares\Adserver\Tests\Services\Supply; + +use Adshares\Adserver\Models\Config; +use Adshares\Adserver\Services\Supply\OpenRtbBridge; +use Adshares\Adserver\Tests\TestCase; +use Adshares\Adserver\Utilities\DatabaseConfigReader; + +class OpenRtbBridgeTest extends TestCase +{ + public function testIsActiveWhileNotConfigured(): void + { + self::assertFalse(OpenRtbBridge::isActive()); + } + + public function testIsActiveWhileConfigured(): void + { + $this->initOpenRtb(); + + self::assertTrue(OpenRtbBridge::isActive()); + } + + private function initOpenRtb(array $settings = []): void + { + $mergedSettings = array_merge( + [ + Config::OPEN_RTB_BRIDGE_ACCOUNT_ADDRESS => '0001-00000004-DBEB', + Config::OPEN_RTB_BRIDGE_URL => 'https://example.com', + ], + $settings, + ); + Config::updateAdminSettings($mergedSettings); + DatabaseConfigReader::overwriteAdministrationConfig(); + } +} From a3055d9a9213b2081da36c49eb13459410805f67 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82Podkalicki?= Date: Fri, 10 Mar 2023 16:04:05 +0100 Subject: [PATCH 21/55] move tests --- .../Http/Controllers/SupplyControllerTest.php | 73 ------ .../app/Services/Supply/OpenRtbBridgeTest.php | 208 +++++++++++++++++- 2 files changed, 205 insertions(+), 76 deletions(-) diff --git a/tests/app/Http/Controllers/SupplyControllerTest.php b/tests/app/Http/Controllers/SupplyControllerTest.php index 02a35df22..f83d014ee 100644 --- a/tests/app/Http/Controllers/SupplyControllerTest.php +++ b/tests/app/Http/Controllers/SupplyControllerTest.php @@ -307,79 +307,6 @@ public function testFindOpenRtbWhileEmptyResponse(): void Http::assertSentCount(1); } - public function testFindOpenRtbWhileInvalidStatus(): void - { - Http::preventStrayRequests(); - Http::fake(['example.com/serve' => Http::response(status: Response::HTTP_NOT_FOUND)]); - $data = $this->initOpenRtb(); - - $response = $this->postJson(self::BANNER_FIND_URI, $data); - - $response->assertStatus(Response::HTTP_OK); - $response->assertJsonStructure(self::FIND_BANNER_STRUCTURE); - $response->assertJsonCount(0, 'data'); - Http::assertSentCount(1); - } - - /** - * @dataProvider findOpenRtbWhileInvalidResponseProvider - */ - public function testFindOpenRtbWhileInvalidResponse(mixed $response): void - { - Http::preventStrayRequests(); - Http::fake([ - 'example.com/serve' => Http::response($response), - ]); - $data = $this->initOpenRtb(); - - $response = $this->postJson(self::BANNER_FIND_URI, $data); - - $response->assertStatus(Response::HTTP_OK); - $response->assertJsonStructure(self::FIND_BANNER_STRUCTURE); - $response->assertJsonCount(0, 'data'); - Http::assertSentCount(1); - } - - public function findOpenRtbWhileInvalidResponseProvider(): array - { - return [ - 'not existing request id' => [[[ - 'request_id' => '1', - 'click_url' => 'https://example.com/click/1', - 'serve_url' => 'https://example.com/serve/1', - 'view_url' => 'https://example.com/view/1', - ]]], - 'no request id' => [[[ - 'click_url' => 'https://example.com/click/1', - 'serve_url' => 'https://example.com/serve/1', - 'view_url' => 'https://example.com/view/1', - ]]], - 'no click url' => [[[ - 'request_id' => '0', - 'serve_url' => 'https://example.com/serve/1', - 'view_url' => 'https://example.com/view/1', - ]]], - 'no serve url' => [[[ - 'request_id' => '0', - 'click_url' => 'https://example.com/click/1', - 'view_url' => 'https://example.com/view/1', - ]]], - 'no view url' => [[[ - 'request_id' => '0', - 'click_url' => 'https://example.com/click/1', - 'serve_url' => 'https://example.com/serve/1', - ]]], - 'invalid serve url type' => [[[ - 'request_id' => '0', - 'click_url' => 'https://example.com/click/1', - 'serve_url' => 1234, - 'view_url' => 'https://example.com/view/1', - ]]], - 'entry is not array' => [['0']], - 'content is not array' => ['0'], - ]; - } - public function testFindWhileNoBanners(): void { $this->app->bind( diff --git a/tests/app/Services/Supply/OpenRtbBridgeTest.php b/tests/app/Services/Supply/OpenRtbBridgeTest.php index 796d78ae3..00ebfa68c 100644 --- a/tests/app/Services/Supply/OpenRtbBridgeTest.php +++ b/tests/app/Services/Supply/OpenRtbBridgeTest.php @@ -23,10 +23,25 @@ namespace Adshares\Adserver\Tests\Services\Supply; +use Adshares\Adserver\Http\Utils; use Adshares\Adserver\Models\Config; +use Adshares\Adserver\Models\NetworkBanner; +use Adshares\Adserver\Models\NetworkCampaign; +use Adshares\Adserver\Models\NetworkHost; +use Adshares\Adserver\Models\Site; +use Adshares\Adserver\Models\User; +use Adshares\Adserver\Models\Zone; use Adshares\Adserver\Services\Supply\OpenRtbBridge; use Adshares\Adserver\Tests\TestCase; +use Adshares\Adserver\Utilities\AdsUtils; use Adshares\Adserver\Utilities\DatabaseConfigReader; +use Adshares\Common\Domain\ValueObject\SecureUrl; +use Adshares\Supply\Application\Dto\FoundBanners; +use Adshares\Supply\Application\Dto\ImpressionContext; +use Adshares\Supply\Application\Service\AdSelect; +use Illuminate\Support\Facades\Http; +use Ramsey\Uuid\Uuid; +use Symfony\Component\HttpFoundation\Response; class OpenRtbBridgeTest extends TestCase { @@ -37,16 +52,141 @@ public function testIsActiveWhileNotConfigured(): void public function testIsActiveWhileConfigured(): void { - $this->initOpenRtb(); + $this->initOpenRtbConfiguration(); self::assertTrue(OpenRtbBridge::isActive()); } - private function initOpenRtb(array $settings = []): void + public function testReplaceOpenRtbBanners(): void + { + Http::preventStrayRequests(); + Http::fake([ + 'example.com/serve' => Http::response([[ + 'request_id' => '0', + 'click_url' => 'https://example.com/click/1', + 'serve_url' => 'https://example.com/serve/1', + 'view_url' => 'https://example.com/view/1', + ]]), + ]); + $initiallyFoundBanners = $this->getFoundBanners(); + $context = new ImpressionContext([], [], []); + + $foundBanners = (new OpenRtbBridge())->replaceOpenRtbBanners($initiallyFoundBanners, $context); + + self::assertCount(1, $foundBanners); + self::assertEquals('3', $foundBanners->first()['request_id']); + Http::assertSentCount(1); + } + + public function testReplaceOpenRtbBannersWhileEmptyResponse(): void + { + Http::preventStrayRequests(); + Http::fake(['example.com/serve' => Http::response([])]); + $initiallyFoundBanners = $this->getFoundBanners(); + $context = new ImpressionContext([], [], []); + + $foundBanners = (new OpenRtbBridge())->replaceOpenRtbBanners($initiallyFoundBanners, $context); + + self::assertCount(1, $foundBanners); + self::assertNull($foundBanners->first()); + Http::assertSentCount(1); + } + + public function testReplaceOpenRtbBannersWhileNoBannersFromBridge(): void + { + Http::preventStrayRequests(); + Http::fake(); + $initiallyFoundBanners = $this->getFoundBanners(); + $banner = $initiallyFoundBanners->get(0); + $banner['pay_from'] = '0001-00000004-DBEB'; + $initiallyFoundBanners->set(0, $banner); + $context = new ImpressionContext([], [], []); + + $foundBanners = (new OpenRtbBridge())->replaceOpenRtbBanners($initiallyFoundBanners, $context); + + self::assertCount(1, $foundBanners); + self::assertEquals('3', $foundBanners->first()['request_id']); + Http::assertNothingSent(); + } + + public function testReplaceOpenRtbBannersWhileInvalidStatus(): void + { + Http::preventStrayRequests(); + Http::fake(['example.com/serve' => Http::response(status: Response::HTTP_NOT_FOUND)]); + $initiallyFoundBanners = $this->getFoundBanners(); + $context = new ImpressionContext([], [], []); + + $foundBanners = (new OpenRtbBridge())->replaceOpenRtbBanners($initiallyFoundBanners, $context); + + self::assertCount(1, $foundBanners); + self::assertNull($foundBanners->first()); + Http::assertSentCount(1); + } + + /** + * @dataProvider replaceOpenRtbBannersWhileInvalidResponseProvider + */ + public function testReplaceOpenRtbBannersWhileInvalidResponse(mixed $response): void + { + Http::preventStrayRequests(); + Http::fake([ + 'example.com/serve' => Http::response($response), + ]); + $initiallyFoundBanners = $this->getFoundBanners(); + $context = new ImpressionContext([], [], []); + + $foundBanners = (new OpenRtbBridge())->replaceOpenRtbBanners($initiallyFoundBanners, $context); + + self::assertCount(1, $foundBanners); + self::assertNull($foundBanners->first()); + Http::assertSentCount(1); + } + + public function replaceOpenRtbBannersWhileInvalidResponseProvider(): array + { + return [ + 'not existing request id' => [[[ + 'request_id' => '1', + 'click_url' => 'https://example.com/click/1', + 'serve_url' => 'https://example.com/serve/1', + 'view_url' => 'https://example.com/view/1', + ]]], + 'no request id' => [[[ + 'click_url' => 'https://example.com/click/1', + 'serve_url' => 'https://example.com/serve/1', + 'view_url' => 'https://example.com/view/1', + ]]], + 'no click url' => [[[ + 'request_id' => '0', + 'serve_url' => 'https://example.com/serve/1', + 'view_url' => 'https://example.com/view/1', + ]]], + 'no serve url' => [[[ + 'request_id' => '0', + 'click_url' => 'https://example.com/click/1', + 'view_url' => 'https://example.com/view/1', + ]]], + 'no view url' => [[[ + 'request_id' => '0', + 'click_url' => 'https://example.com/click/1', + 'serve_url' => 'https://example.com/serve/1', + ]]], + 'invalid serve url type' => [[[ + 'request_id' => '0', + 'click_url' => 'https://example.com/click/1', + 'serve_url' => 1234, + 'view_url' => 'https://example.com/view/1', + ]]], + 'entry is not array' => [['0']], + 'content is not array' => ['0'], + ]; + } + + private function initOpenRtbConfiguration(array $settings = []): void { $mergedSettings = array_merge( [ - Config::OPEN_RTB_BRIDGE_ACCOUNT_ADDRESS => '0001-00000004-DBEB', + Config::OPEN_RTB_BRIDGE_ACCOUNT_ADDRESS => '0001-00000001-8B4E', Config::OPEN_RTB_BRIDGE_URL => 'https://example.com', ], $settings, @@ -54,4 +194,66 @@ private function initOpenRtb(array $settings = []): void Config::updateAdminSettings($mergedSettings); DatabaseConfigReader::overwriteAdministrationConfig(); } + + private function getFoundBanners(): FoundBanners + { + $this->initOpenRtbConfiguration(); + NetworkHost::factory()->create([ + 'address' => '0001-00000001-8B4E', + 'host' => 'https://example.com', + ]); + /** @var Zone $zone */ + $zone = Zone::factory()->create([ + 'site_id' => Site::factory()->create([ + 'user_id' => User::factory()->create(['api_token' => '1234', 'auto_withdrawal' => 1e11]), + 'status' => Site::STATUS_ACTIVE, + ]), + ]); + /** @var NetworkBanner $networkBanner */ + $networkBanner = NetworkBanner::factory()->create([ + 'network_campaign_id' => NetworkCampaign::factory()->create(), + 'serve_url' => 'https://example.com/serve/' . Uuid::uuid4()->toString(), + ]); + $impressionId = Uuid::uuid4(); + + return new FoundBanners([ + [ + 'id' => $networkBanner->uuid, + 'demandId' => $networkBanner->demand_banner_id, + 'publisher_id' => '0123456879ABCDEF0123456879ABCDEF', + 'zone_id' => $zone->uuid, + 'pay_from' => '0001-00000001-8B4E', + 'pay_to' => AdsUtils::normalizeAddress(config('app.adshares_address')), + 'type' => $networkBanner->type, + 'size' => $networkBanner->size, + 'serve_url' => $networkBanner->serve_url, + 'creative_sha1' => '', + 'click_url' => SecureUrl::change( + route( + 'log-network-click', + [ + 'id' => $networkBanner->uuid, + 'iid' => $impressionId, + 'r' => Utils::urlSafeBase64Encode($networkBanner->click_url), + 'zid' => $zone->uuid, + ] + ) + ), + 'view_url' => SecureUrl::change( + route( + 'log-network-view', + [ + 'id' => $networkBanner->uuid, + 'iid' => $impressionId, + 'r' => Utils::urlSafeBase64Encode($networkBanner->view_url), + 'zid' => $zone->uuid, + ] + ) + ), + 'info_box' => true, + 'rpm' => 0.5, + 'request_id' => '3', + ] + ]); + } } From 3c745f4b476c388be3acc0f43951bf6237c7f7b2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82Podkalicki?= Date: Tue, 14 Mar 2023 12:59:56 +0100 Subject: [PATCH 22/55] add removing query params --- app/Http/Utils.php | 26 ++++++++++++++++++++++ tests/app/Http/UtilsTest.php | 42 ++++++++++++++++++++++++++++++++++++ 2 files changed, 68 insertions(+) diff --git a/app/Http/Utils.php b/app/Http/Utils.php index 132baa80c..747fbd899 100644 --- a/app/Http/Utils.php +++ b/app/Http/Utils.php @@ -255,6 +255,32 @@ public static function addUrlParameter($url, $name, $value): string } } + public static function removeUrlParameter($url, $name): string + { + $parsed = parse_url($url); + + parse_str($parsed['query'], $params); + unset($params[$name]); + $parsed['query'] = http_build_query($params) ?: null; + + return self::revertParseUrl($parsed); + } + + private static function revertParseUrl(array $parsed): string + { + $scheme = isset($parsed['scheme']) ? $parsed['scheme'] . '://' : ''; + $user = $parsed['user'] ?? ''; + $pass = isset($parsed['pass']) ? ':' . $parsed['pass'] : ''; + $pass = ($user || $pass) ? "$pass@" : ''; + $host = $parsed['host'] ?? ''; + $port = isset($parsed['port']) ? ':' . $parsed['port'] : ''; + $path = $parsed['path'] ?? ''; + $query = isset($parsed['query']) ? '?' . $parsed['query'] : ''; + $fragment = isset($parsed['fragment']) ? '#' . $parsed['fragment'] : ''; + + return "$scheme$user$pass$host$port$path$query$fragment"; + } + public static function attachOrProlongTrackingCookie( Request $request, Response $response, diff --git a/tests/app/Http/UtilsTest.php b/tests/app/Http/UtilsTest.php index 17641faf2..6cbdfa38c 100644 --- a/tests/app/Http/UtilsTest.php +++ b/tests/app/Http/UtilsTest.php @@ -139,4 +139,46 @@ public function testGetSiteContextOnInvalidUrl(): void ]; self::assertEquals('example.com', Utils::getSiteContext($request, $context)['domain']); } + + /** + * @dataProvider removeUrlParameterProvider + */ + public function testRemoveUrlParameter(string $url, string $expected): void + { + self::assertEquals($expected, Utils::removeUrlParameter($url, 'test')); + } + + public function removeUrlParameterProvider(): array + { + return [ + [ + 'https://example.com/1?test=', + 'https://example.com/1', + ], + [ + 'https://example.com/1?test=asdf', + 'https://example.com/1', + ], + [ + 'https://example.com/1?test[]=asdf&test[]=zxcv', + 'https://example.com/1', + ], + [ + 'https://example.com/1?a=1&test=asdf&b=1', + 'https://example.com/1?a=1&b=1', + ], + [ + 'https://example.com/1?a=1&b=1&test=asdf', + 'https://example.com/1?a=1&b=1', + ], + [ + 'https://example.com/1?a=1&test[]=asdf&b=1&test[]=zxcv', + 'https://example.com/1?a=1&b=1', + ], + [ + 'https://example.com?test=1#home', + 'https://example.com#home', + ], + ]; + } } From 17ef68a0d77d7e57a1594d1508bf340dc5ac8df1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82Podkalicki?= Date: Tue, 14 Mar 2023 13:32:07 +0100 Subject: [PATCH 23/55] extid, handle view/click --- app/Http/Controllers/SupplyController.php | 52 +++++++++++----- app/Services/Supply/OpenRtbBridge.php | 76 +++++++++++++++-------- routes/supply.php | 2 +- 3 files changed, 88 insertions(+), 42 deletions(-) diff --git a/app/Http/Controllers/SupplyController.php b/app/Http/Controllers/SupplyController.php index 6201b9d6a..262ef8e46 100644 --- a/app/Http/Controllers/SupplyController.php +++ b/app/Http/Controllers/SupplyController.php @@ -208,7 +208,7 @@ public function legacyFind( } elseif ('POST' === $request->getRealMethod()) { $data = (string)$request->getContent(); } elseif ('OPTIONS' === $request->getRealMethod()) { - $response->setStatusCode(Response::HTTP_NO_CONTENT); + $response->setStatusCode(BaseResponse::HTTP_NO_CONTENT); $response->headers->set('Access-Control-Max-Age', 1728000); return $response; } else { @@ -238,7 +238,7 @@ public function legacyFind( if (!$user) { if (config('app.auto_registration_enabled')) { if (!in_array(UserRole::PUBLISHER, config('app.default_user_roles'))) { - throw new HttpException(Response::HTTP_FORBIDDEN, 'Cannot register publisher'); + throw new HttpException(BaseResponse::HTTP_FORBIDDEN, 'Cannot register publisher'); } $user = User::registerWithWallet($payoutAddress, true); if (config('app.auto_confirmation_enabled')) { @@ -250,7 +250,7 @@ public function legacyFind( } } if (!$user->isPublisher()) { - throw new HttpException(Response::HTTP_FORBIDDEN, 'Forbidden'); + throw new HttpException(BaseResponse::HTTP_FORBIDDEN, 'Forbidden'); } $site = Site::fetchOrCreate( $user->id, @@ -579,7 +579,7 @@ static function () use ($jsPath, $params) { return $response; } - public function logNetworkSimpleClick(Request $request): RedirectResponse + public function logNetworkSimpleClick(Request $request): BaseResponse { $impressionId = self::impressionIdToUuid($request->query->get('iid')); $networkImpression = NetworkImpression::fetchByImpressionId($impressionId); @@ -606,19 +606,21 @@ public function logNetworkSimpleClick(Request $request): RedirectResponse return $this->logNetworkClick($request, $networkImpression->context->banner_id); } - public function logNetworkClick(Request $request, string $bannerId): RedirectResponse + public function logNetworkClick(Request $request, string $bannerId): BaseResponse { $this->validateEventRequest($request); + $isDspBridge = $request->query->has('extid'); $url = $this->getRedirectionUrlFromQuery($request); - if (!$url) { if (null === ($banner = NetworkBanner::fetchByPublicId($bannerId))) { throw new NotFoundHttpException(); } $url = $banner->click_url; } - + if (!$url) { + throw new UnprocessableEntityHttpException(); + } $url = $this->addQueryStringToUrl($request, $url); $caseId = str_replace('-', '', $request->query->get('cid')); @@ -647,7 +649,13 @@ public function logNetworkClick(Request $request, string $bannerId): RedirectRes $url = Utils::addUrlParameter($url, 'ctx', $ctx); } - $response = new RedirectResponse($url); + if ($isDspBridge) { + $redirectUrl = (new OpenRtbBridge())->getEventRedirectUrl($url) + ?: route('why', ['bid' => $bannerId, 'cid' => $caseId]); + $response = new RedirectResponse($redirectUrl); + } else { + $response = new RedirectResponse($url); + } $response->send(); try { @@ -699,7 +707,7 @@ private function addQueryStringToUrl(Request $request, string $url): string return $url; } - public function logNetworkSimpleView(Request $request): RedirectResponse + public function logNetworkSimpleView(Request $request): BaseResponse { $impressionId = self::impressionIdToUuid($request->query->get('iid')); $networkImpression = NetworkImpression::fetchByImpressionId($impressionId); @@ -726,7 +734,7 @@ public function logNetworkSimpleView(Request $request): RedirectResponse return $this->logNetworkView($request, $networkImpression->context->banner_id); } - public function logNetworkView(Request $request, string $bannerId): RedirectResponse + public function logNetworkView(Request $request, string $bannerId): BaseResponse { $this->validateEventRequest($request); @@ -739,10 +747,18 @@ public function logNetworkView(Request $request, string $bannerId): RedirectResp throw new NotFoundHttpException(); } + $isDspBridge = $request->query->has('extid'); $url = $this->getRedirectionUrlFromQuery($request); - if ($url) { - $url = $this->addQueryStringToUrl($request, $url); + if (!$url) { + if (null === ($banner = NetworkBanner::fetchByPublicId($bannerId))) { + throw new NotFoundHttpException(); + } + $url = $banner->view_url; + } + if (!$url) { + throw new UnprocessableEntityHttpException(); } + $url = $this->addQueryStringToUrl($request, $url); $payTo = AdsUtils::normalizeAddress(config('app.adshares_address')); @@ -764,7 +780,15 @@ public function logNetworkView(Request $request, string $bannerId): RedirectResp $ctx = $this->buildCtx($networkImpression, $impressionId, $zoneId); $url = Utils::addUrlParameter($url, 'ctx', $ctx); } - $response = new RedirectResponse($url); + + if ($isDspBridge) { + $redirectUrl = (new OpenRtbBridge())->getEventRedirectUrl($url); + $response = null !== $redirectUrl + ? new RedirectResponse($redirectUrl) + : new BaseResponse(status: BaseResponse::HTTP_NO_CONTENT); + } else { + $response = new RedirectResponse($url); + } if ($request->headers->has('Origin')) { $response->headers->set('Access-Control-Allow-Origin', $request->headers->get('Origin')); @@ -791,7 +815,7 @@ public function logNetworkView(Request $request, string $bannerId): RedirectResp private function validateEventRequest(Request $request): void { if ( - !$request->query->has('r') + !($request->query->has('r') || $request->query->has('extid')) || !($request->query->has('ctx') || (Uuid::isValid($request->query->get('zid', '')))) || !Uuid::isValid($request->query->get('cid', '')) ) { diff --git a/app/Services/Supply/OpenRtbBridge.php b/app/Services/Supply/OpenRtbBridge.php index 22cda71d5..1e029531e 100644 --- a/app/Services/Supply/OpenRtbBridge.php +++ b/app/Services/Supply/OpenRtbBridge.php @@ -21,8 +21,10 @@ namespace Adshares\Adserver\Services\Supply; +use Adshares\Adserver\Http\Utils; use Adshares\Supply\Application\Dto\FoundBanners; use Adshares\Supply\Application\Dto\ImpressionContext; +use Illuminate\Http\Client\HttpClientException; use Illuminate\Support\Facades\Http; use Illuminate\Support\Facades\Log; use Symfony\Component\HttpFoundation\Response as BaseResponse; @@ -52,33 +54,35 @@ public function replaceOpenRtbBanners(FoundBanners $foundBanners, ImpressionCont if (empty($openRtbBanners)) { return $foundBanners; } - $response = Http::post( - config('app.open_rtb_bridge_url') . self::SERVE_PATH, - [ - 'context' => $context->toArray(), - 'requests' => $openRtbBanners - ], - ); - if ( - BaseResponse::HTTP_OK !== $response->status() - || !$this->isOpenRtbAuctionResponseValid($content = $response->json(), $openRtbBanners) - ) { - foreach ($openRtbBanners as $index => $serveUrl) { - $foundBanners->set($index, null); - } - return $foundBanners; - } - foreach ($content as $entry) { - $foundBanner = array_merge( - $foundBanners->get((int)$entry['request_id']), + try { + $response = Http::post( + config('app.open_rtb_bridge_url') . self::SERVE_PATH, [ - 'click_url' => $entry['click_url'], - 'serve_url' => $entry['serve_url'], - 'view_url' => $entry['view_url'], - ] + 'context' => $context->toArray(), + 'requests' => $openRtbBanners, + ], ); - $foundBanners->set((int)$entry['request_id'], $foundBanner); - unset($openRtbBanners[$entry['request_id']]); + if ( + BaseResponse::HTTP_OK === $response->status() + && $this->isOpenRtbAuctionResponseValid($content = $response->json(), $openRtbBanners) + ) { + foreach ($content as $entry) { + $externalId = $entry['ext_id']; + $foundBanner = $foundBanners->get((int)$entry['request_id']); + foreach (['click_url', 'view_url'] as $field) { + $foundBanner[$field] = Utils::addUrlParameter( + Utils::removeUrlParameter($foundBanner[$field], 'r'), + 'extid', + $externalId, + ); + } + $foundBanner['serve_url'] = $entry['serve_url']; + $foundBanners->set((int)$entry['request_id'], $foundBanner); + unset($openRtbBanners[$entry['request_id']]); + } + } + } catch (HttpClientException $exception) { + Log::error(sprintf('Replacing OpenRtb banner failed: %s', $exception->getMessage())); } foreach ($openRtbBanners as $index => $serveUrl) { $foundBanners->set($index, null); @@ -98,10 +102,9 @@ private function isOpenRtbAuctionResponseValid(mixed $content, array $openBtbBan return false; } $fields = [ + 'ext_id', 'request_id', - 'click_url', 'serve_url', - 'view_url', ]; foreach ($fields as $field) { if (!isset($entry[$field])) { @@ -120,4 +123,23 @@ private function isOpenRtbAuctionResponseValid(mixed $content, array $openBtbBan } return true; } + + public function getEventRedirectUrl(string $url): ?string + { + $redirectUrl = null; + try { + $response = Http::get($url); + $statusCode = $response->status(); + if (BaseResponse::HTTP_OK === $statusCode) { + $redirectUrl = $response->json('redirect_url'); + } else { + if (BaseResponse::HTTP_NO_CONTENT !== $statusCode) { + Log::error(sprintf('DSP bridge event notification failed: %d: %s', $statusCode, $response->body())); + } + } + } catch (HttpClientException $exception) { + Log::error(sprintf('DSP bridge event notification failed: client exception: %s', $exception->getMessage())); + } + return $redirectUrl; + } } diff --git a/routes/supply.php b/routes/supply.php index 52d88623c..11ab9cf33 100644 --- a/routes/supply.php +++ b/routes/supply.php @@ -44,7 +44,7 @@ Route::get('/supply/targeting-reach', [SupplyController::class, 'targetingReachList']); # WHY PAGE -Route::get('/supply/why', [SupplyController::class, 'why']); +Route::get('/supply/why', [SupplyController::class, 'why'])->name('why'); Route::get('/supply/ad/report/{case_id}/{banner_id}', [SupplyController::class, 'reportAd']) ->name('report-ad'); From 5e2a5e5006a8dfe22a965df42f62a13035bbfa63 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82Podkalicki?= Date: Tue, 14 Mar 2023 14:50:06 +0100 Subject: [PATCH 24/55] fix tests --- .../Http/Controllers/SupplyControllerTest.php | 3 +- .../app/Services/Supply/OpenRtbBridgeTest.php | 78 ++++++++++++++----- 2 files changed, 60 insertions(+), 21 deletions(-) diff --git a/tests/app/Http/Controllers/SupplyControllerTest.php b/tests/app/Http/Controllers/SupplyControllerTest.php index f83d014ee..88efb10e9 100644 --- a/tests/app/Http/Controllers/SupplyControllerTest.php +++ b/tests/app/Http/Controllers/SupplyControllerTest.php @@ -277,9 +277,8 @@ public function testFindOpenRtb(): void Http::fake([ 'example.com/serve' => Http::response([[ 'request_id' => '0', - 'click_url' => 'https://example.com/click/1', + 'ext_id' => '1', 'serve_url' => 'https://example.com/serve/1', - 'view_url' => 'https://example.com/view/1', ]]), ]); $data = $this->initOpenRtb(); diff --git a/tests/app/Services/Supply/OpenRtbBridgeTest.php b/tests/app/Services/Supply/OpenRtbBridgeTest.php index 00ebfa68c..e1c0bdd9c 100644 --- a/tests/app/Services/Supply/OpenRtbBridgeTest.php +++ b/tests/app/Services/Supply/OpenRtbBridgeTest.php @@ -38,7 +38,9 @@ use Adshares\Common\Domain\ValueObject\SecureUrl; use Adshares\Supply\Application\Dto\FoundBanners; use Adshares\Supply\Application\Dto\ImpressionContext; -use Adshares\Supply\Application\Service\AdSelect; +use Closure; +use GuzzleHttp\Promise\PromiseInterface; +use Illuminate\Http\Client\ConnectionException; use Illuminate\Support\Facades\Http; use Ramsey\Uuid\Uuid; use Symfony\Component\HttpFoundation\Response; @@ -62,10 +64,9 @@ public function testReplaceOpenRtbBanners(): void Http::preventStrayRequests(); Http::fake([ 'example.com/serve' => Http::response([[ + 'ext_id' => '1', 'request_id' => '0', - 'click_url' => 'https://example.com/click/1', 'serve_url' => 'https://example.com/serve/1', - 'view_url' => 'https://example.com/view/1', ]]), ]); $initiallyFoundBanners = $this->getFoundBanners(); @@ -74,7 +75,10 @@ public function testReplaceOpenRtbBanners(): void $foundBanners = (new OpenRtbBridge())->replaceOpenRtbBanners($initiallyFoundBanners, $context); self::assertCount(1, $foundBanners); - self::assertEquals('3', $foundBanners->first()['request_id']); + $foundBanner = $foundBanners->first(); + self::assertEquals('3', $foundBanner['request_id']); + self::assertStringContainsString('extid=1', $foundBanner['click_url']); + self::assertStringContainsString('extid=1', $foundBanner['view_url']); Http::assertSentCount(1); } @@ -123,6 +127,18 @@ public function testReplaceOpenRtbBannersWhileInvalidStatus(): void Http::assertSentCount(1); } + public function testReplaceOpenRtbBannersWhileConnectionException(): void + { + Http::fake(fn() => throw new ConnectionException('test-exception')); + $initiallyFoundBanners = $this->getFoundBanners(); + $context = new ImpressionContext([], [], []); + + $foundBanners = (new OpenRtbBridge())->replaceOpenRtbBanners($initiallyFoundBanners, $context); + + self::assertCount(1, $foundBanners); + self::assertNull($foundBanners->first()); + } + /** * @dataProvider replaceOpenRtbBannersWhileInvalidResponseProvider */ @@ -147,41 +163,65 @@ public function replaceOpenRtbBannersWhileInvalidResponseProvider(): array return [ 'not existing request id' => [[[ 'request_id' => '1', - 'click_url' => 'https://example.com/click/1', + 'ext_id' => '1', 'serve_url' => 'https://example.com/serve/1', - 'view_url' => 'https://example.com/view/1', ]]], 'no request id' => [[[ - 'click_url' => 'https://example.com/click/1', + 'ext_id' => '1', 'serve_url' => 'https://example.com/serve/1', - 'view_url' => 'https://example.com/view/1', ]]], - 'no click url' => [[[ + 'no ext id' => [[[ 'request_id' => '0', 'serve_url' => 'https://example.com/serve/1', - 'view_url' => 'https://example.com/view/1', ]]], 'no serve url' => [[[ 'request_id' => '0', - 'click_url' => 'https://example.com/click/1', - 'view_url' => 'https://example.com/view/1', - ]]], - 'no view url' => [[[ - 'request_id' => '0', - 'click_url' => 'https://example.com/click/1', - 'serve_url' => 'https://example.com/serve/1', + 'ext_id' => '1', ]]], 'invalid serve url type' => [[[ 'request_id' => '0', - 'click_url' => 'https://example.com/click/1', + 'ext_id' => '1', 'serve_url' => 1234, - 'view_url' => 'https://example.com/view/1', ]]], 'entry is not array' => [['0']], 'content is not array' => ['0'], ]; } + /** + * @dataProvider getEventRedirectUrlProvider + */ + public function testGetEventRedirectUrl(Closure $responseClosure, ?string $expectedUrl): void + { + Http::preventStrayRequests(); + Http::fake(['example.com' => $responseClosure()]); + + $url = (new OpenRtbBridge())->getEventRedirectUrl('https://example.com'); + + self::assertEquals($expectedUrl, $url); + Http::assertSentCount(1); + } + + public function getEventRedirectUrlProvider(): array + { + return [ + 'redirection' => [ + fn() => Http::response(['redirect_url' => 'https://adshares.net']), + 'https://adshares.net', + ], + 'no redirection' => [fn() => Http::response(status: Response::HTTP_NO_CONTENT), null], + 'unsupported response format' => [fn() => Http::response(['url' => 'https://adshares.net']), null], + ]; + } + public function testGetEventRedirectUrlWhileConnectionException(): void + { + Http::fake(fn() => throw new ConnectionException('test-exception')); + + $url = (new OpenRtbBridge())->getEventRedirectUrl('https://example.com'); + + self::assertEquals(null, $url); + } + private function initOpenRtbConfiguration(array $settings = []): void { $mergedSettings = array_merge( From 85819f3cb1b701bb181143db4228a9a49bed6f4b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82Podkalicki?= Date: Tue, 14 Mar 2023 15:19:00 +0100 Subject: [PATCH 25/55] add view tests --- app/Http/Controllers/SupplyController.php | 6 ++- .../Http/Controllers/SupplyControllerTest.php | 48 ++++++++++++++++++- 2 files changed, 52 insertions(+), 2 deletions(-) diff --git a/app/Http/Controllers/SupplyController.php b/app/Http/Controllers/SupplyController.php index 262ef8e46..1102840aa 100644 --- a/app/Http/Controllers/SupplyController.php +++ b/app/Http/Controllers/SupplyController.php @@ -756,7 +756,11 @@ public function logNetworkView(Request $request, string $bannerId): BaseResponse $url = $banner->view_url; } if (!$url) { - throw new UnprocessableEntityHttpException(); + $response = new BaseResponse(status: BaseResponse::HTTP_NO_CONTENT); + if ($request->headers->has('Origin')) { + $response->headers->set('Access-Control-Allow-Origin', $request->headers->get('Origin')); + } + return $response; } $url = $this->addQueryStringToUrl($request, $url); diff --git a/tests/app/Http/Controllers/SupplyControllerTest.php b/tests/app/Http/Controllers/SupplyControllerTest.php index 88efb10e9..9bca5badf 100644 --- a/tests/app/Http/Controllers/SupplyControllerTest.php +++ b/tests/app/Http/Controllers/SupplyControllerTest.php @@ -745,9 +745,10 @@ public function testLogNetworkView(): void { [$query, $banner, $zone] = self::initBeforeLoggingView(); - $response = $this->get(self::buildLogViewUri($banner->uuid, $query)); + $response = $this->get(self::buildLogViewUri($banner->uuid, $query), ['Origin' => 'https://example.com']); $response->assertStatus(Response::HTTP_FOUND); + $response->assertHeader('Access-Control-Allow-Origin', 'https://example.com'); $response->assertHeader('Location'); $location = $response->headers->get('Location'); self::assertStringStartsWith('https://example.com/view', $location); @@ -853,6 +854,51 @@ public function testLogNetworkViewFailWhileImpressionIdIsInvalid(): void $response->assertStatus(Response::HTTP_NOT_FOUND); } + public function testLogNetworkViewWhileBannerFromBridgeRedirection(): void + { + [$query, $banner, $zone] = self::initBeforeLoggingView(); + unset($query['r']); + $query['extid'] = '12'; + Http::preventStrayRequests(); + Http::fake(['example.com/view?*' => Http::response(['redirect_url' => 'https://adshares.net/view'])]); + + $response = $this->get(self::buildLogViewUri($banner->uuid, $query)); + + $response->assertStatus(Response::HTTP_FOUND); + $response->assertHeader('Location'); + $location = $response->headers->get('Location'); + self::assertStringStartsWith('https://adshares.net/view', $location); + } + + public function testLogNetworkViewWhileBannerFromBridgeNoRedirection(): void + { + [$query, $banner, $zone] = self::initBeforeLoggingView(); + unset($query['r']); + $query['extid'] = '12'; + Http::preventStrayRequests(); + Http::fake(['example.com/view?*' => Http::response(status: Response::HTTP_NO_CONTENT)]); + + $response = $this->get(self::buildLogViewUri($banner->uuid, $query)); + + $response->assertStatus(Response::HTTP_NO_CONTENT); + } + + public function testLogNetworkViewWhileBannerFromBridgeNoUrl(): void + { + [$query, $banner, $zone] = self::initBeforeLoggingView(); + $banner->view_url = ''; + $banner->saveOrFail(); + unset($query['r']); + $query['extid'] = '12'; + Http::preventStrayRequests(); + Http::fake(['example.com/view?*' => Http::response(status: Response::HTTP_NO_CONTENT)]); + + $response = $this->get(self::buildLogViewUri($banner->uuid, $query), ['Origin' => 'https://example.com']); + + $response->assertStatus(Response::HTTP_NO_CONTENT); + $response->assertHeader('Access-Control-Allow-Origin', 'https://example.com'); + } + public function testRegister(): void { Config::updateAdminSettings([Config::ADUSER_SERVE_SUBDOMAIN => 'au']); From eb030a02f1d8e329907b8742b2ebdbd425ea5e52 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82Podkalicki?= Date: Tue, 14 Mar 2023 15:30:45 +0100 Subject: [PATCH 26/55] add click tests --- .../Http/Controllers/SupplyControllerTest.php | 56 ++++++++++++++++++- 1 file changed, 55 insertions(+), 1 deletion(-) diff --git a/tests/app/Http/Controllers/SupplyControllerTest.php b/tests/app/Http/Controllers/SupplyControllerTest.php index 9bca5badf..b8dff7b19 100644 --- a/tests/app/Http/Controllers/SupplyControllerTest.php +++ b/tests/app/Http/Controllers/SupplyControllerTest.php @@ -741,6 +741,57 @@ public function testLogNetworkClickFailWhileInvalidRedirectUrlAndBannerId(): voi $response->assertStatus(Response::HTTP_NOT_FOUND); } + public function testLogNetworkClickWhileBannerFromBridgeRedirection(): void + { + [$query, $banner, $zone] = self::initBeforeLoggingClick(); + unset($query['r']); + $query['extid'] = '12'; + Http::preventStrayRequests(); + Http::fake(['example.com/click?*' => Http::response(['redirect_url' => 'https://adshares.net/click'])]); + + $response = $this->get(self::buildLogClickUri($banner->uuid, $query)); + + $response->assertStatus(Response::HTTP_FOUND); + $response->assertHeader('Location'); + $location = $response->headers->get('Location'); + self::assertStringStartsWith('https://adshares.net/click', $location); + self::assertDatabaseCount(NetworkCaseClick::class, 1); + Http::assertSentCount(1); + } + + public function testLogNetworkClickWhileBannerFromBridgeNoRedirection(): void + { + [$query, $banner, $zone] = self::initBeforeLoggingClick(); + unset($query['r']); + $query['extid'] = '12'; + Http::preventStrayRequests(); + Http::fake(['example.com/click?*' => Http::response(status: Response::HTTP_NO_CONTENT)]); + + $response = $this->get(self::buildLogClickUri($banner->uuid, $query)); + + $response->assertStatus(Response::HTTP_FOUND); + $response->assertHeader('Location'); + $location = $response->headers->get('Location'); + self::assertStringContainsString('/supply/why?', $location); + self::assertDatabaseCount(NetworkCaseClick::class, 1); + Http::assertSentCount(1); + } + + public function testLogNetworkClickWhileBannerFromBridgeNoUrl(): void + { + [$query, $banner, $zone] = self::initBeforeLoggingClick(); + unset($query['r']); + $query['extid'] = '12'; + $banner->click_url = ''; + $banner->saveOrFail(); + Http::preventStrayRequests(); + + $response = $this->get(self::buildLogClickUri($banner->uuid, $query)); + + $response->assertStatus(Response::HTTP_UNPROCESSABLE_ENTITY); + Http::assertNothingSent(); + } + public function testLogNetworkView(): void { [$query, $banner, $zone] = self::initBeforeLoggingView(); @@ -868,6 +919,7 @@ public function testLogNetworkViewWhileBannerFromBridgeRedirection(): void $response->assertHeader('Location'); $location = $response->headers->get('Location'); self::assertStringStartsWith('https://adshares.net/view', $location); + Http::assertSentCount(1); } public function testLogNetworkViewWhileBannerFromBridgeNoRedirection(): void @@ -881,6 +933,7 @@ public function testLogNetworkViewWhileBannerFromBridgeNoRedirection(): void $response = $this->get(self::buildLogViewUri($banner->uuid, $query)); $response->assertStatus(Response::HTTP_NO_CONTENT); + Http::assertSentCount(1); } public function testLogNetworkViewWhileBannerFromBridgeNoUrl(): void @@ -891,12 +944,12 @@ public function testLogNetworkViewWhileBannerFromBridgeNoUrl(): void unset($query['r']); $query['extid'] = '12'; Http::preventStrayRequests(); - Http::fake(['example.com/view?*' => Http::response(status: Response::HTTP_NO_CONTENT)]); $response = $this->get(self::buildLogViewUri($banner->uuid, $query), ['Origin' => 'https://example.com']); $response->assertStatus(Response::HTTP_NO_CONTENT); $response->assertHeader('Access-Control-Allow-Origin', 'https://example.com'); + Http::assertNothingSent(); } public function testRegister(): void @@ -1039,6 +1092,7 @@ private static function initBeforeLoggingView(): array $campaign = NetworkCampaign::factory()->create(); /** @var NetworkBanner $banner */ $banner = NetworkBanner::factory()->create([ + 'click_url' => 'https://example.com/click', 'network_campaign_id' => $campaign, 'view_url' => 'https://example.com/view', ]); From 220055d51191faed78c4f1a3c0e049e6d22713a7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82Podkalicki?= Date: Tue, 14 Mar 2023 16:08:46 +0100 Subject: [PATCH 27/55] test invalid banner id --- .../Http/Controllers/SupplyControllerTest.php | 21 +++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/tests/app/Http/Controllers/SupplyControllerTest.php b/tests/app/Http/Controllers/SupplyControllerTest.php index b8dff7b19..9ffb806d4 100644 --- a/tests/app/Http/Controllers/SupplyControllerTest.php +++ b/tests/app/Http/Controllers/SupplyControllerTest.php @@ -409,6 +409,17 @@ public function testFindDynamicWithoutExistingUser(): void self::assertNotNull(User::firstOrFail()->admin_confirmed_at); } + public function testFindDynamicWithoutExistingUserWhileInvalidPublisherId(): void + { + Config::updateAdminSettings([Config::AUTO_CONFIRMATION_ENABLED => '1']); + $this->mockAdSelect(); + $data = self::getDynamicFindData(['context' => self::getContextData(['publisher' => '0001-00000001-8B4E'])]); + + $response = $this->postJson(self::BANNER_FIND_URI, $data); + + $response->assertStatus(Response::HTTP_UNPROCESSABLE_ENTITY); + } + public function testFindDynamicWhileSiteApprovalRequired(): void { Config::updateAdminSettings([Config::SITE_APPROVAL_REQUIRED => '*']); @@ -885,6 +896,16 @@ public function testLogNetworkViewWhileCaseIdAndImpressionIdAndZoneIdAreUuidV4() self::assertEquals('0001-00000005-CBCA', $locationQuery['pto']); } + public function testLogNetworkViewFailWhileInvalidRedirectUrlAndBannerId(): void + { + [$query, $banner, $zone] = self::initBeforeLoggingView(); + $query['r'] = ''; + + $response = $this->get(self::buildLogViewUri('invalid', $query)); + + $response->assertStatus(Response::HTTP_NOT_FOUND); + } + public function testLogNetworkViewFailWhileImpressionIdIsMissing(): void { [$query, $banner, $zone] = self::initBeforeLoggingView(); From 8fe969aa60dbba02381fc09732586064b497665d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82Podkalicki?= Date: Fri, 17 Mar 2023 18:28:31 +0100 Subject: [PATCH 28/55] fetch payment metadata --- app/Models/BridgePayment.php | 72 ++++++ app/Services/Supply/OpenRtbBridge.php | 81 ++++++ database/factories/BridgePaymentFactory.php | 44 ++++ ...17_144439_create_bridge_payments_table.php | 49 ++++ .../app/Services/Supply/OpenRtbBridgeTest.php | 240 ++++++++++++++++-- 5 files changed, 458 insertions(+), 28 deletions(-) create mode 100644 app/Models/BridgePayment.php create mode 100644 database/factories/BridgePaymentFactory.php create mode 100644 database/migrations/2023_03_17_144439_create_bridge_payments_table.php diff --git a/app/Models/BridgePayment.php b/app/Models/BridgePayment.php new file mode 100644 index 000000000..a52ecd0f5 --- /dev/null +++ b/app/Models/BridgePayment.php @@ -0,0 +1,72 @@ + + */ + +namespace Adshares\Adserver\Models; + +use DateTimeImmutable; +use Illuminate\Database\Eloquent\Builder; +use Illuminate\Database\Eloquent\Collection; +use Illuminate\Database\Eloquent\Factories\HasFactory; +use Illuminate\Database\Eloquent\Model; +use Illuminate\Support\Carbon; + +/** + * @property int $id + * @property Carbon created_at + * @property Carbon updated_at + * @property string payment_id + * @property Carbon payment_time + * @property int|null amount + * @property int status + * @property string address + * @property int last_offset + * @mixin Builder + */ +class BridgePayment extends Model +{ + use HasFactory; + + public const STATUS_INVALID = -1; + public const STATUS_NEW = 0; + public const STATUS_RETRY = 1; + public const STATUS_DONE = 2; + + public static function fetchByAddressAndPaymentIds(string $address, array $paymentIds): Collection + { + return (new self())->where('address', $address)->whereIn('payment_id', $paymentIds)->get(); + } + + public static function register( + string $address, + string $paymentId, + DateTimeImmutable $paymentTime, + ?int $amount, + int $status, + ): void { + $bridgePayment = new self(); + $bridgePayment->address = $address; + $bridgePayment->payment_id = $paymentId; + $bridgePayment->payment_time = $paymentTime; + $bridgePayment->amount = $amount; + $bridgePayment->status = $status; + $bridgePayment->saveOrFail(); + } +} diff --git a/app/Services/Supply/OpenRtbBridge.php b/app/Services/Supply/OpenRtbBridge.php index 1e029531e..910ec4de5 100644 --- a/app/Services/Supply/OpenRtbBridge.php +++ b/app/Services/Supply/OpenRtbBridge.php @@ -22,8 +22,11 @@ namespace Adshares\Adserver\Services\Supply; use Adshares\Adserver\Http\Utils; +use Adshares\Adserver\Models\BridgePayment; use Adshares\Supply\Application\Dto\FoundBanners; use Adshares\Supply\Application\Dto\ImpressionContext; +use DateTimeImmutable; +use DateTimeInterface; use Illuminate\Http\Client\HttpClientException; use Illuminate\Support\Facades\Http; use Illuminate\Support\Facades\Log; @@ -31,6 +34,7 @@ class OpenRtbBridge { + private const PAYMENTS_PATH = '/payment-reports'; private const SERVE_PATH = '/serve'; public static function isActive(): bool @@ -90,6 +94,83 @@ public function replaceOpenRtbBanners(FoundBanners $foundBanners, ImpressionCont return $foundBanners; } + public function fetchAndStorePayments(): void + { + try { + $response = Http::get(config('app.open_rtb_bridge_url') . self::PAYMENTS_PATH); + if ( + BaseResponse::HTTP_OK === $response->status() + && $this->isPaymentResponseValid($content = $response->json()) + ) { + if (empty($content)) { + return; + } + $paymentIds = array_map(fn (array $entry) => $entry['id'], $content); + $accountAddress = config('app.open_rtb_bridge_account_address'); + $bridgePayments = BridgePayment::fetchByAddressAndPaymentIds($accountAddress, $paymentIds) + ->keyBy('payment_id'); + foreach ($content as $entry) { + $paymentId = $entry['id']; + if (null === ($bridgePayment = $bridgePayments->get($paymentId))) { + BridgePayment::register( + $accountAddress, + $paymentId, + DateTimeImmutable::createFromFormat(DateTimeInterface::ATOM, $entry['created_at']), + $entry['value'], + 'done' === $entry['status'] ? BridgePayment::STATUS_NEW : BridgePayment::STATUS_RETRY, + ); + } else { + /** @var $bridgePayment BridgePayment */ + if (BridgePayment::STATUS_RETRY === $bridgePayment->status && 'done' === $entry['status']) { + $bridgePayment->payment_time = + DateTimeImmutable::createFromFormat(DateTimeInterface::ATOM, $entry['created_at']); + $bridgePayment->amount = $entry['value']; + $bridgePayment->status = BridgePayment::STATUS_NEW; + $bridgePayment->saveOrFail(); + } + } + } + } + } catch (HttpClientException $exception) { + Log::error(sprintf('Fetching payments from bridge failed: %s', $exception->getMessage())); + } + } + + private function isPaymentResponseValid(mixed $content): bool + { + if (!is_array($content)) { + Log::error('Invalid bridge payments response: body is not an array'); + return false; + } + foreach ($content as $entry) { + if (!is_array($entry)) { + Log::error('Invalid bridge payments response: entry is not an array'); + return false; + } + $fields = [ + 'id', + 'created_at', + 'status', + 'value', + ]; + foreach ($fields as $field) { + if (!array_key_exists($field, $entry)) { + Log::error(sprintf('Invalid bridge payments response: missing key %s', $field)); + return false; + } + } + if (!is_string($entry['status'])) { + Log::error('Invalid bridge payments response: status is not a string'); + return false; + } + if (false === DateTimeImmutable::createFromFormat(DateTimeInterface::ATOM, $entry['created_at'])) { + Log::error('Invalid bridge payments response: created_at is not in ISO8601 format'); + return false; + } + } + return true; + } + private function isOpenRtbAuctionResponseValid(mixed $content, array $openBtbBanners): bool { if (!is_array($content)) { diff --git a/database/factories/BridgePaymentFactory.php b/database/factories/BridgePaymentFactory.php new file mode 100644 index 000000000..177bca3e2 --- /dev/null +++ b/database/factories/BridgePaymentFactory.php @@ -0,0 +1,44 @@ + + */ + +declare(strict_types=1); + +namespace Database\Factories; + +use DateTimeImmutable; +use Illuminate\Database\Eloquent\Factories\Factory; + +class BridgePaymentFactory extends Factory +{ + public function definition(): array + { + return [ + 'created_at' => new DateTimeImmutable(), + 'updated_at' => new DateTimeImmutable(), + 'address' => '0001-00000001-8B4E', + 'payment_id' => '0', + 'payment_time' => new DateTimeImmutable('-1 hour'), + 'amount' => 1e11, + 'status' => 0, + 'last_offset' => 0, + ]; + } +} diff --git a/database/migrations/2023_03_17_144439_create_bridge_payments_table.php b/database/migrations/2023_03_17_144439_create_bridge_payments_table.php new file mode 100644 index 000000000..c79a0e045 --- /dev/null +++ b/database/migrations/2023_03_17_144439_create_bridge_payments_table.php @@ -0,0 +1,49 @@ + + */ + +use Illuminate\Database\Migrations\Migration; +use Illuminate\Database\Schema\Blueprint; +use Illuminate\Support\Facades\Schema; + +return new class extends Migration +{ + public function up(): void + { + Schema::create('bridge_payments', function (Blueprint $table) { + $table->id(); + $table->timestamps(); + $table->string('address', 18); + $table->string('payment_id'); + $table->timestamp('payment_time'); + $table->bigInteger('amount')->nullable(); + $table->tinyInteger('status')->default(0); + $table->integer('last_offset')->default(0); + + $table->unique(['address', 'payment_id']); + $table->index('status', 'bridge_payments_status_index'); + }); + } + + public function down(): void + { + Schema::dropIfExists('bridge_payments'); + } +}; diff --git a/tests/app/Services/Supply/OpenRtbBridgeTest.php b/tests/app/Services/Supply/OpenRtbBridgeTest.php index e1c0bdd9c..4b53d396b 100644 --- a/tests/app/Services/Supply/OpenRtbBridgeTest.php +++ b/tests/app/Services/Supply/OpenRtbBridgeTest.php @@ -24,6 +24,7 @@ namespace Adshares\Adserver\Tests\Services\Supply; use Adshares\Adserver\Http\Utils; +use Adshares\Adserver\Models\BridgePayment; use Adshares\Adserver\Models\Config; use Adshares\Adserver\Models\NetworkBanner; use Adshares\Adserver\Models\NetworkCampaign; @@ -39,9 +40,9 @@ use Adshares\Supply\Application\Dto\FoundBanners; use Adshares\Supply\Application\Dto\ImpressionContext; use Closure; -use GuzzleHttp\Promise\PromiseInterface; use Illuminate\Http\Client\ConnectionException; use Illuminate\Support\Facades\Http; +use Illuminate\Support\Facades\Log; use Ramsey\Uuid\Uuid; use Symfony\Component\HttpFoundation\Response; @@ -63,11 +64,13 @@ public function testReplaceOpenRtbBanners(): void { Http::preventStrayRequests(); Http::fake([ - 'example.com/serve' => Http::response([[ - 'ext_id' => '1', - 'request_id' => '0', - 'serve_url' => 'https://example.com/serve/1', - ]]), + 'example.com/serve' => Http::response([ + [ + 'ext_id' => '1', + 'request_id' => '0', + 'serve_url' => 'https://example.com/serve/1', + ] + ]), ]); $initiallyFoundBanners = $this->getFoundBanners(); $context = new ImpressionContext([], [], []); @@ -161,28 +164,48 @@ public function testReplaceOpenRtbBannersWhileInvalidResponse(mixed $response): public function replaceOpenRtbBannersWhileInvalidResponseProvider(): array { return [ - 'not existing request id' => [[[ - 'request_id' => '1', - 'ext_id' => '1', - 'serve_url' => 'https://example.com/serve/1', - ]]], - 'no request id' => [[[ - 'ext_id' => '1', - 'serve_url' => 'https://example.com/serve/1', - ]]], - 'no ext id' => [[[ - 'request_id' => '0', - 'serve_url' => 'https://example.com/serve/1', - ]]], - 'no serve url' => [[[ - 'request_id' => '0', - 'ext_id' => '1', - ]]], - 'invalid serve url type' => [[[ - 'request_id' => '0', - 'ext_id' => '1', - 'serve_url' => 1234, - ]]], + 'not existing request id' => [ + [ + [ + 'request_id' => '1', + 'ext_id' => '1', + 'serve_url' => 'https://example.com/serve/1', + ] + ] + ], + 'no request id' => [ + [ + [ + 'ext_id' => '1', + 'serve_url' => 'https://example.com/serve/1', + ] + ] + ], + 'no ext id' => [ + [ + [ + 'request_id' => '0', + 'serve_url' => 'https://example.com/serve/1', + ] + ] + ], + 'no serve url' => [ + [ + [ + 'request_id' => '0', + 'ext_id' => '1', + ] + ] + ], + 'invalid serve url type' => [ + [ + [ + 'request_id' => '0', + 'ext_id' => '1', + 'serve_url' => 1234, + ] + ] + ], 'entry is not array' => [['0']], 'content is not array' => ['0'], ]; @@ -213,6 +236,7 @@ public function getEventRedirectUrlProvider(): array 'unsupported response format' => [fn() => Http::response(['url' => 'https://adshares.net']), null], ]; } + public function testGetEventRedirectUrlWhileConnectionException(): void { Http::fake(fn() => throw new ConnectionException('test-exception')); @@ -222,6 +246,151 @@ public function testGetEventRedirectUrlWhileConnectionException(): void self::assertEquals(null, $url); } + public function testFetchAndStorePayments(): void + { + BridgePayment::factory()->create([ + 'payment_id' => '1678957200', + 'payment_time' => '2023-03-17 14:00:00', + 'status' => BridgePayment::STATUS_RETRY, + 'amount' => null, + ]); + $responseData = [ + [ + 'id' => 1678953600, + 'created_at' => '2023-03-17T16:04:33+00:00', + 'updated_at' => '2023-03-17T16:04:33+00:00', + 'status' => 'done', + 'value' => 100_000_000_000, + ], + [ + 'id' => 1678957200, + 'created_at' => '2023-03-17T16:04:33+00:00', + 'updated_at' => '2023-03-17T16:04:33+00:00', + 'status' => 'done', + 'value' => 123_400_000_000, + ], + [ + 'id' => 1678960800, + 'created_at' => '2023-03-17T16:04:33+00:00', + 'updated_at' => '2023-03-17T16:04:33+00:00', + 'status' => 'error', + 'value' => null, + ], + [ + 'id' => 1678964400, + 'created_at' => '2023-03-17T16:04:33+00:00', + 'updated_at' => '2023-03-17T16:04:33+00:00', + 'status' => 'processing', + 'value' => null, + ], + ]; + $this->initOpenRtbConfiguration(); + Http::preventStrayRequests(); + Http::fake(['example.com/payment-reports' => Http::response($responseData)]); + + (new OpenRtbBridge())->fetchAndStorePayments(); + + self::assertDatabaseHas( + BridgePayment::class, + [ + 'payment_id' => '1678957200', + 'payment_time' => '2023-03-17 16:04:33', + 'status' => BridgePayment::STATUS_NEW, + 'amount' => 123_400_000_000, + ], + ); + self::assertDatabaseHas( + BridgePayment::class, + [ + 'payment_id' => '1678964400', + 'payment_time' => '2023-03-17 16:04:33', + 'status' => BridgePayment::STATUS_RETRY, + 'amount' => null, + ], + ); + Http::assertSentCount(1); + } + + public function testFetchAndStorePaymentsWhileResponseIsEmpty(): void + { + $responseData = []; + $this->initOpenRtbConfiguration(); + Http::preventStrayRequests(); + Http::fake(['example.com/payment-reports' => Http::response($responseData)]); + + (new OpenRtbBridge())->fetchAndStorePayments(); + + self::assertDatabaseEmpty(BridgePayment::class); + Http::assertSentCount(1); + } + + public function testFetchAndStorePaymentsWhileConnectionException(): void + { + Log::spy(); + $this->initOpenRtbConfiguration(); + Http::fake(fn() => throw new ConnectionException('test-exception')); + + (new OpenRtbBridge())->fetchAndStorePayments(); + + self::assertDatabaseEmpty(BridgePayment::class); + Log::shouldHaveReceived('error') + ->with('Fetching payments from bridge failed: test-exception') + ->once(); + } + + /** + * @dataProvider fetchAndStorePaymentsWhileInvalidResponseProvider + */ + public function testFetchAndStorePaymentsWhileInvalidResponse(mixed $response, string $errorMessage): void + { + Log::spy(); + Http::preventStrayRequests(); + Http::fake([ + 'example.com/payment-reports' => Http::response($response), + ]); + $this->initOpenRtbConfiguration(); + + (new OpenRtbBridge())->fetchAndStorePayments(); + + self::assertDatabaseEmpty(BridgePayment::class); + Http::assertSentCount(1); + Log::shouldHaveReceived('error') + ->with($errorMessage) + ->once(); + } + + public function fetchAndStorePaymentsWhileInvalidResponseProvider(): array + { + return [ + 'no id' => [ + [self::validPaymentResponseEntry(remove: 'id')], + 'Invalid bridge payments response: missing key id', + ], + 'no created_at' => [ + [self::validPaymentResponseEntry(remove: 'created_at')], + 'Invalid bridge payments response: missing key created_at', + ], + 'no status' => [ + [self::validPaymentResponseEntry(remove: 'status')], + 'Invalid bridge payments response: missing key status', + ], + 'no value' => [ + [self::validPaymentResponseEntry(remove: 'value')], + 'Invalid bridge payments response: missing key value', + ], + 'invalid created_at format' => [ + [self::validPaymentResponseEntry(['created_at' => '2023-01-01'])], + 'Invalid bridge payments response: created_at is not in ISO8601 format', + ], + 'invalid status type' => [ + [self::validPaymentResponseEntry(['status' => 0])], + 'Invalid bridge payments response: status is not a string', + ], + 'entry is not array' => [['0'], 'Invalid bridge payments response: entry is not an array'], + 'content is not array' => ['0', 'Invalid bridge payments response: body is not an array'], + ]; + } + private function initOpenRtbConfiguration(array $settings = []): void { $mergedSettings = array_merge( @@ -296,4 +465,19 @@ private function getFoundBanners(): FoundBanners ] ]); } + + private static function validPaymentResponseEntry(array $merge = [], string $remove = null): array + { + $data = array_merge([ + 'id' => 1678953600, + 'created_at' => '2023-03-17T16:04:33+00:00', + 'updated_at' => '2023-03-17T16:04:33+00:00', + 'status' => 'done', + 'value' => 100_000_000_000, + ], $merge); + if (null !== $remove) { + unset($data[$remove]); + } + return $data; + } } From f54fc6bc4789a12498fd8514111cecbcce414bc5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82Podkalicki?= Date: Mon, 20 Mar 2023 12:10:00 +0100 Subject: [PATCH 29/55] add processing payments --- .../Commands/SupplyProcessPayments.php | 10 ++ app/Models/BridgePayment.php | 9 ++ app/Models/NetworkCasePayment.php | 38 ++++- app/Services/PaymentDetailsProcessor.php | 84 ++++++++++- app/Services/Supply/OpenRtbBridge.php | 80 +++++++++- database/factories/BridgePaymentFactory.php | 3 +- ...17_144439_create_bridge_payments_table.php | 12 ++ .../Services/PaymentDetailsProcessorTest.php | 108 +++++++++++-- .../app/Services/Supply/OpenRtbBridgeTest.php | 142 ++++++++++++++++++ 9 files changed, 458 insertions(+), 28 deletions(-) diff --git a/app/Console/Commands/SupplyProcessPayments.php b/app/Console/Commands/SupplyProcessPayments.php index 8bcd8d5cb..41a813ba8 100644 --- a/app/Console/Commands/SupplyProcessPayments.php +++ b/app/Console/Commands/SupplyProcessPayments.php @@ -31,6 +31,7 @@ use Adshares\Adserver\Services\Dto\PaymentProcessingResult; use Adshares\Adserver\Services\LicenseFeeSender; use Adshares\Adserver\Services\PaymentDetailsProcessor; +use Adshares\Adserver\Services\Supply\OpenRtbBridge; use Adshares\Adserver\ViewModel\ServerEventType; use Adshares\Common\Infrastructure\Service\LicenseReader; use Adshares\Supply\Application\Service\DemandClient; @@ -126,6 +127,15 @@ public function handle(): void } } + $bridge = new OpenRtbBridge(); + $processed = $bridge->processPayments( + $this->demandClient, + $this->paymentDetailsProcessor, + (int)$this->option('chunkSize'), + ); + $processedPaymentsForAds += $processed; + $processedPaymentsTotal += $processed; + $timestamps = $this->fetchTimestampsToUpdate($processedAdsPaymentIds); foreach ($timestamps as $timestamp) { NetworkCaseLogsHourlyMeta::invalidate($timestamp); diff --git a/app/Models/BridgePayment.php b/app/Models/BridgePayment.php index a52ecd0f5..4718b9f0a 100644 --- a/app/Models/BridgePayment.php +++ b/app/Models/BridgePayment.php @@ -49,11 +49,20 @@ class BridgePayment extends Model public const STATUS_RETRY = 1; public const STATUS_DONE = 2; + protected $dates = [ + 'payment_time', + ]; + public static function fetchByAddressAndPaymentIds(string $address, array $paymentIds): Collection { return (new self())->where('address', $address)->whereIn('payment_id', $paymentIds)->get(); } + public static function fetchNew(): Collection + { + return (new self())->where('status', self::STATUS_NEW)->get(); + } + public static function register( string $address, string $paymentId, diff --git a/app/Models/NetworkCasePayment.php b/app/Models/NetworkCasePayment.php index c6f98f121..8a9e7d25b 100644 --- a/app/Models/NetworkCasePayment.php +++ b/app/Models/NetworkCasePayment.php @@ -1,7 +1,7 @@ $bridgePaymentId, + 'exchange_rate' => $exchangeRate, + 'license_fee' => 0, + 'operator_fee' => 0, + 'paid_amount' => $totalAmount, + 'paid_amount_currency' => $paidAmountCurrency, + 'pay_time' => $payTime, + 'total_amount' => $totalAmount, + ] + ); + } + + public static function fetchPaymentsForPublishersByPaymentId( + int $paymentId, + bool $usePaidAmountCurrency, + bool $isAdsPayment = true, ): Collection { $paidAmountColumn = $usePaidAmountCurrency ? 'paid_amount_currency' : 'paid_amount'; + $paymentIdColumn = $isAdsPayment ? 'ads_payment_id' : 'bridge_payment_id'; return self::select( [ 'publisher_id', @@ -127,7 +151,7 @@ public static function fetchPaymentsForPublishersByAdsPaymentId( function (JoinClause $join) { $join->on('network_case_payments.network_case_id', '=', 'network_cases.id'); } - )->where('ads_payment_id', $adsPaymentId)->groupBy('publisher_id')->get(); + )->where($paymentIdColumn, $paymentId)->groupBy('publisher_id')->get(); } public static function fetchPaymentsToExport( diff --git a/app/Services/PaymentDetailsProcessor.php b/app/Services/PaymentDetailsProcessor.php index e3dba129d..3dff82571 100644 --- a/app/Services/PaymentDetailsProcessor.php +++ b/app/Services/PaymentDetailsProcessor.php @@ -1,7 +1,7 @@ id; + $availableAmount = $payment->amount - $carriedEventValueSum; + + $exchangeRate = $this->fetchExchangeRate(); + $exchangeRateValue = $exchangeRate->getValue(); + $totalEventValue = 0; + $cases = $this->fetchNetworkCasesForPaymentDetails($paymentDetails); + + foreach ($paymentDetails as $paymentDetail) { + /** @var NetworkCase $case */ + if (null === ($case = $cases->get($paymentDetail['case_id']))) { + continue; + } + + $spendableAmount = max(0, $availableAmount - $totalEventValue); + $eventValue = (int)min($spendableAmount, $paymentDetail['event_value']); + + $networkCasePayment = NetworkCasePayment::createByBridgePaymentId( + $bridgePaymentId, + $transactionTime, + $eventValue, + $exchangeRateValue, + $exchangeRate->fromClick($eventValue), + ); + $case->networkCasePayments()->save($networkCasePayment); + + $totalEventValue += $eventValue; + } + + return new PaymentProcessingResult($totalEventValue, 0); + } + public function addAdIncomeToUserLedger(AdsPayment $adsPayment): void { $adServerAddress = config('app.adshares_address'); $usePaidAmountCurrency = Currency::ADS !== Currency::from(config('app.currency')); - $splitPayments = NetworkCasePayment::fetchPaymentsForPublishersByAdsPaymentId( + $splitPayments = NetworkCasePayment::fetchPaymentsForPublishersByPaymentId( $adsPayment->id, $usePaidAmountCurrency ); @@ -111,7 +150,6 @@ public function addAdIncomeToUserLedger(AdsPayment $adsPayment): void $splitPayment->paid_amount ) ); - continue; } @@ -128,6 +166,42 @@ public function addAdIncomeToUserLedger(AdsPayment $adsPayment): void } } + public function addBridgeAdIncomeToUserLedger(BridgePayment $payment): void + { + $adServerAddress = config('app.adshares_address'); + $usePaidAmountCurrency = Currency::ADS !== Currency::from(config('app.currency')); + $splitPayments = NetworkCasePayment::fetchPaymentsForPublishersByPaymentId( + $payment->id, + $usePaidAmountCurrency, + false, + ); + + foreach ($splitPayments as $splitPayment) { + if (null === ($user = User::fetchByUuid($splitPayment->publisher_id))) { + Log::warning( + sprintf( + '[PaymentDetailsProcessor] User id (%s) does not exist. BridgePayment id (%s). Amount (%s).', + $splitPayment->publisher_id, + $payment->id, + $splitPayment->paid_amount + ) + ); + continue; + } + + $amount = (int)$splitPayment->paid_amount; + UserLedgerEntry::constructWithAddressAndTransaction( + $user->id, + $amount, + UserLedgerEntry::STATUS_ACCEPTED, + UserLedgerEntry::TYPE_AD_INCOME, + $payment->address, + $adServerAddress, + '', + )->save(); + } + } + private function fetchExchangeRate(): ExchangeRate { $appCurrency = Currency::from(config('app.currency')); diff --git a/app/Services/Supply/OpenRtbBridge.php b/app/Services/Supply/OpenRtbBridge.php index 910ec4de5..68c0500f9 100644 --- a/app/Services/Supply/OpenRtbBridge.php +++ b/app/Services/Supply/OpenRtbBridge.php @@ -21,21 +21,34 @@ namespace Adshares\Adserver\Services\Supply; +use Adshares\Adserver\Facades\DB; use Adshares\Adserver\Http\Utils; use Adshares\Adserver\Models\BridgePayment; +use Adshares\Adserver\Models\NetworkHost; +use Adshares\Adserver\Services\PaymentDetailsProcessor; use Adshares\Supply\Application\Dto\FoundBanners; use Adshares\Supply\Application\Dto\ImpressionContext; +use Adshares\Supply\Application\Service\DemandClient; +use Adshares\Supply\Application\Service\Exception\EmptyInventoryException; +use Adshares\Supply\Application\Service\Exception\UnexpectedClientResponseException; use DateTimeImmutable; use DateTimeInterface; use Illuminate\Http\Client\HttpClientException; +use Illuminate\Support\Collection; use Illuminate\Support\Facades\Http; use Illuminate\Support\Facades\Log; use Symfony\Component\HttpFoundation\Response as BaseResponse; class OpenRtbBridge { + private const PAYMENT_REPORT_READY_STATUS = 'done'; private const PAYMENTS_PATH = '/payment-reports'; private const SERVE_PATH = '/serve'; + private const SQL_QUERY_GET_PROCESSED_PAYMENTS_AMOUNT = << $entry['id'], $content); + $paymentIds = array_map(fn(array $entry) => $entry['id'], $content); $accountAddress = config('app.open_rtb_bridge_account_address'); $bridgePayments = BridgePayment::fetchByAddressAndPaymentIds($accountAddress, $paymentIds) ->keyBy('payment_id'); @@ -117,11 +130,15 @@ public function fetchAndStorePayments(): void $paymentId, DateTimeImmutable::createFromFormat(DateTimeInterface::ATOM, $entry['created_at']), $entry['value'], - 'done' === $entry['status'] ? BridgePayment::STATUS_NEW : BridgePayment::STATUS_RETRY, + self::PAYMENT_REPORT_READY_STATUS === $entry['status'] ? BridgePayment::STATUS_NEW + : BridgePayment::STATUS_RETRY, ); } else { /** @var $bridgePayment BridgePayment */ - if (BridgePayment::STATUS_RETRY === $bridgePayment->status && 'done' === $entry['status']) { + if ( + BridgePayment::STATUS_RETRY === $bridgePayment->status + && self::PAYMENT_REPORT_READY_STATUS === $entry['status'] + ) { $bridgePayment->payment_time = DateTimeImmutable::createFromFormat(DateTimeInterface::ATOM, $entry['created_at']); $bridgePayment->amount = $entry['value']; @@ -136,6 +153,63 @@ public function fetchAndStorePayments(): void } } + public function processPayments( + DemandClient $demandClient, + PaymentDetailsProcessor $paymentDetailsProcessor, + int $limit = 500, + ): int { + $processedPaymentsCount = 0; + /** @var Collection $bridgePayments */ + $bridgePayments = BridgePayment::fetchNew(); + foreach ($bridgePayments as $bridgePayment) { + if (null === ($networkHost = NetworkHost::fetchByAddress($bridgePayment->address))) { + $bridgePayment->status = BridgePayment::STATUS_INVALID; + $bridgePayment->save(); + ++$processedPaymentsCount; + continue; + } + + $offset = $bridgePayment->last_offset ?? 0; + $eventValueSum = 0; + if ($offset > 0) { + $paymentSum = DB::selectOne(self::SQL_QUERY_GET_PROCESSED_PAYMENTS_AMOUNT, [0]); + $eventValueSum = $paymentSum->total_amount ?? 0; + } + $transactionTime = $bridgePayment->payment_time; + + do { + try { + $paymentDetails = $demandClient->fetchPaymentDetails( + $networkHost->host, + $bridgePayment->payment_id, + $limit, + $offset, + ); + } catch (EmptyInventoryException | UnexpectedClientResponseException) { + $bridgePayment->save(); + break 2; + } + + $processPaymentDetails = $paymentDetailsProcessor->processEventsPaidByBridge( + $bridgePayment, + $transactionTime, + $paymentDetails, + $eventValueSum, + ); + $eventValueSum += $processPaymentDetails->eventValuePartialSum(); + + $bridgePayment->last_offset = $offset += $limit; + } while (count($paymentDetails) === $limit); + + $paymentDetailsProcessor->addBridgeAdIncomeToUserLedger($bridgePayment); + + $bridgePayment->status = BridgePayment::STATUS_DONE; + $bridgePayment->save(); + ++$processedPaymentsCount; + } + return $processedPaymentsCount; + } + private function isPaymentResponseValid(mixed $content): bool { if (!is_array($content)) { diff --git a/database/factories/BridgePaymentFactory.php b/database/factories/BridgePaymentFactory.php index 177bca3e2..53fbb74c5 100644 --- a/database/factories/BridgePaymentFactory.php +++ b/database/factories/BridgePaymentFactory.php @@ -23,6 +23,7 @@ namespace Database\Factories; +use Adshares\Adserver\Models\BridgePayment; use DateTimeImmutable; use Illuminate\Database\Eloquent\Factories\Factory; @@ -37,7 +38,7 @@ public function definition(): array 'payment_id' => '0', 'payment_time' => new DateTimeImmutable('-1 hour'), 'amount' => 1e11, - 'status' => 0, + 'status' => BridgePayment::STATUS_NEW, 'last_offset' => 0, ]; } diff --git a/database/migrations/2023_03_17_144439_create_bridge_payments_table.php b/database/migrations/2023_03_17_144439_create_bridge_payments_table.php index c79a0e045..04328557a 100644 --- a/database/migrations/2023_03_17_144439_create_bridge_payments_table.php +++ b/database/migrations/2023_03_17_144439_create_bridge_payments_table.php @@ -21,6 +21,7 @@ use Illuminate\Database\Migrations\Migration; use Illuminate\Database\Schema\Blueprint; +use Illuminate\Support\Facades\DB; use Illuminate\Support\Facades\Schema; return new class extends Migration @@ -40,10 +41,21 @@ public function up(): void $table->unique(['address', 'payment_id']); $table->index('status', 'bridge_payments_status_index'); }); + + Schema::table('network_case_payments', function (Blueprint $table) { + $table->bigInteger('ads_payment_id')->nullable()->change(); + $table->bigInteger('bridge_payment_id')->after('ads_payment_id')->nullable()->index(); + }); } public function down(): void { + DB::delete('DELETE FROM network_case_payments WHERE ads_payment_id IS NULL'); + Schema::table('network_case_payments', function (Blueprint $table) { + $table->bigInteger('ads_payment_id')->nullable(false)->change(); + $table->dropColumn('bridge_payment_id'); + }); + Schema::dropIfExists('bridge_payments'); } }; diff --git a/tests/app/Services/PaymentDetailsProcessorTest.php b/tests/app/Services/PaymentDetailsProcessorTest.php index d95fda104..e7c94fd93 100644 --- a/tests/app/Services/PaymentDetailsProcessorTest.php +++ b/tests/app/Services/PaymentDetailsProcessorTest.php @@ -1,7 +1,7 @@ getPaymentDetailsProcessor(); - $adsPayment = $this->createAdsPayment(10000); + $paymentDetails = [ + [ + 'case_id' => '0123456789ABCDEF0123456789ABCDEF', + 'event_value' => 10000, + ], + ]; - $paymentDetailsProcessor->processPaidEvents($adsPayment, new DateTime(), [], 0); + $paymentDetailsProcessor->processPaidEvents($adsPayment, new DateTimeImmutable(), $paymentDetails, 0); $this->assertCount(0, NetworkPayment::all()); } - public function testProcessingDetails(): void + public function testProcessPaidEvents(): void { $totalPayment = 10000; $paidEventsCount = 2; @@ -70,6 +78,7 @@ public function testProcessingDetails(): void /** @var NetworkImpression $networkImpression */ $networkImpression = NetworkImpression::factory()->create(); + /** @var Collection $networkCases */ $networkCases = NetworkCase::factory()->times($paidEventsCount)->create( ['network_impression_id' => $networkImpression->id, 'publisher_id' => $userUuid] ); @@ -80,16 +89,11 @@ public function testProcessingDetails(): void foreach ($networkCases as $networkCase) { $paymentDetails[] = [ 'case_id' => $networkCase->case_id, - 'event_id' => $networkCase->event_id, - 'event_type' => $networkCase->event_type, - 'banner_id' => $networkCase->banner_id, - 'zone_id' => $networkCase->zone_id, - 'publisher_id' => $userUuid, 'event_value' => $totalPayment / $paidEventsCount, ]; } - $result = $paymentDetailsProcessor->processPaidEvents($adsPayment, new DateTime(), $paymentDetails, 0); + $result = $paymentDetailsProcessor->processPaidEvents($adsPayment, new DateTimeImmutable(), $paymentDetails, 0); $expectedLicenseAmount = 0; $expectedOperatorAmount = 0; @@ -103,7 +107,57 @@ public function testProcessingDetails(): void $this->assertEquals($totalPayment, $result->eventValuePartialSum()); $this->assertEquals($expectedLicenseAmount, $result->licenseFeePartialSum()); - $this->assertEquals($expectedAdIncome, NetworkCasePayment::sum('paid_amount')); + $this->assertEquals($expectedAdIncome, (new NetworkCasePayment())->sum('paid_amount')); + } + + public function testProcessEventsPaidByBridge(): void + { + $totalPayment = 1e11; + $paidEventsCount = 2; + $expectedAdIncome = $totalPayment; + $expectedLicenseAmount = 0; + /** @var User $publisher */ + $publisher = User::factory()->create(); + $publisherUuid = $publisher->uuid; + $networkImpression = NetworkImpression::factory()->create(); + /** @var Collection $networkCases */ + $networkCases = NetworkCase::factory()->times($paidEventsCount)->create( + [ + 'network_impression_id' => $networkImpression, + 'publisher_id' => $publisherUuid, + ] + ); + $paymentDetails = []; + foreach ($networkCases as $networkCase) { + $paymentDetails[] = [ + 'case_id' => $networkCase->case_id, + 'event_value' => (int)($totalPayment / $paidEventsCount), + ]; + } + $entryWithNotExistingCaseId = [ + 'case_id' => '0123456789ABCDEF0123456789ABCDEF', + 'event_value' => $totalPayment, + ]; + $paymentDetails[] = $entryWithNotExistingCaseId; + /** @var BridgePayment $payment */ + $payment = BridgePayment::factory()->create(); + $paymentDetailsProcessor = $this->getPaymentDetailsProcessor(); + + $result = $paymentDetailsProcessor->processEventsPaidByBridge( + $payment, + new DateTimeImmutable(), + $paymentDetails, + 0, + ); + + self::assertEquals($totalPayment, $result->eventValuePartialSum()); + self::assertEquals($expectedLicenseAmount, $result->licenseFeePartialSum()); + self::assertEquals($expectedAdIncome, (new NetworkCasePayment())->sum('paid_amount')); + self::assertDatabaseCount(NetworkCasePayment::class, 2); + self::assertDatabaseHas(NetworkCasePayment::class, [ + 'bridge_payment_id' => $payment->id, + 'network_case_id' => $networkCases->first()->id, + ]); } /** @@ -153,6 +207,36 @@ public function currencyProvider(): array ]; } + public function testBridgeAdIncomeToUserLedger(): void + { + $payment = BridgePayment::factory()->create(); + /** @var User $user */ + $user = User::factory()->create(); + /** @var NetworkCase $networkCase */ + $networkCase = NetworkCase::factory()->create(['publisher_id' => $user->uuid]); + $paidAmount = 100_000_000; + $rate = 5.0; + $paidAmountCurrency = (int)floor($paidAmount * $rate); + NetworkCasePayment::factory()->create([ + 'bridge_payment_id' => $payment->id, + 'exchange_rate' => $rate, + 'license_fee' => 0, + 'network_case_id' => $networkCase->id, + 'operator_fee' => 0, + 'paid_amount' => $paidAmount, + 'paid_amount_currency' => $paidAmountCurrency, + 'total_amount' => $paidAmount, + ]); + $expectedPaidAmount = $paidAmount; + $paymentDetailsProcessor = $this->getPaymentDetailsProcessor(); + + $paymentDetailsProcessor->addBridgeAdIncomeToUserLedger($payment); + + $entries = UserLedgerEntry::all(); + self::assertCount(1, $entries); + self::assertEquals($expectedPaidAmount, $entries->first()->amount); + } + public function testAddAdIncomeToUserLedgerWhenNoUser(): void { $adsPayment = $this->createAdsPayment(100_000_000_000); diff --git a/tests/app/Services/Supply/OpenRtbBridgeTest.php b/tests/app/Services/Supply/OpenRtbBridgeTest.php index 4b53d396b..fe438da20 100644 --- a/tests/app/Services/Supply/OpenRtbBridgeTest.php +++ b/tests/app/Services/Supply/OpenRtbBridgeTest.php @@ -28,19 +28,34 @@ use Adshares\Adserver\Models\Config; use Adshares\Adserver\Models\NetworkBanner; use Adshares\Adserver\Models\NetworkCampaign; +use Adshares\Adserver\Models\NetworkCase; +use Adshares\Adserver\Models\NetworkCasePayment; use Adshares\Adserver\Models\NetworkHost; +use Adshares\Adserver\Models\NetworkImpression; +use Adshares\Adserver\Models\NetworkPayment; use Adshares\Adserver\Models\Site; use Adshares\Adserver\Models\User; +use Adshares\Adserver\Models\UserLedgerEntry; use Adshares\Adserver\Models\Zone; +use Adshares\Adserver\Services\PaymentDetailsProcessor; use Adshares\Adserver\Services\Supply\OpenRtbBridge; use Adshares\Adserver\Tests\TestCase; use Adshares\Adserver\Utilities\AdsUtils; use Adshares\Adserver\Utilities\DatabaseConfigReader; +use Adshares\Common\Application\Dto\ExchangeRate; +use Adshares\Common\Domain\ValueObject\NullUrl; use Adshares\Common\Domain\ValueObject\SecureUrl; +use Adshares\Common\Infrastructure\Service\ExchangeRateReader; +use Adshares\Common\Infrastructure\Service\LicenseReader; +use Adshares\Mock\Client\DummyDemandClient; use Adshares\Supply\Application\Dto\FoundBanners; use Adshares\Supply\Application\Dto\ImpressionContext; +use Adshares\Supply\Application\Service\DemandClient; +use Adshares\Supply\Application\Service\Exception\UnexpectedClientResponseException; use Closure; +use DateTime; use Illuminate\Http\Client\ConnectionException; +use Illuminate\Support\Collection; use Illuminate\Support\Facades\Http; use Illuminate\Support\Facades\Log; use Ramsey\Uuid\Uuid; @@ -391,6 +406,123 @@ public function fetchAndStorePaymentsWhileInvalidResponseProvider(): array ]; } + public function testProcessPayments(): void + { + $networkHost = self::registerHost(new DummyDemandClient()); + /** @var BridgePayment $bridgePayment */ + $bridgePayment = BridgePayment::factory()->create(['address' => $networkHost->address]); + $exchangeRateReader = self::createMock(ExchangeRateReader::class); + $exchangeRateReader->method('fetchExchangeRate') + ->willReturn(new ExchangeRate(new DateTime(), 1, 'USD')); + $licenseReader = self::createMock(LicenseReader::class); + $paymentDetailsProcessor = new PaymentDetailsProcessor($exchangeRateReader, $licenseReader); + $networkImpression = NetworkImpression::factory()->create(); + /** @var User $publisher */ + $publisher = User::factory()->create(); + $caseCount = 2; + /** @var Collection $networkCases */ + $networkCases = NetworkCase::factory()->times($caseCount)->create([ + 'network_impression_id' => $networkImpression, + 'publisher_id' => $publisher->uuid, + ]); + $expectedPaidAmount = 1e11; + $expectedLicenseFee = 0; + $demandClient = self::createMock(DemandClient::class); + $demandClient->expects(self::once())->method('fetchPaymentDetails')->willReturn( + $networkCases->map( + fn($case) => [ + 'case_id' => $case->case_id, + 'event_value' => (int)($expectedPaidAmount / $caseCount), + ], + )->toArray() + ); + + (new OpenRtbBridge())->processPayments($demandClient, $paymentDetailsProcessor); + + self::assertDatabaseCount(BridgePayment::class, 1); + self::assertDatabaseHas(BridgePayment::class, ['status' => BridgePayment::STATUS_DONE]); + self::assertEquals($expectedPaidAmount, (new NetworkCasePayment())->sum('total_amount')); + self::assertEquals($expectedLicenseFee, (new NetworkPayment())->sum('amount')); + self::assertDatabaseHas(UserLedgerEntry::class, [ + 'amount' => $expectedPaidAmount, + 'user_id' => $publisher->id, + ]); + self::assertDatabaseCount(NetworkCasePayment::class, $caseCount); + self::assertDatabaseHas(NetworkCasePayment::class, [ + 'bridge_payment_id' => $bridgePayment->id, + 'network_case_id' => $networkCases->first()->id, + ]); + } + + public function testProcessPaymentsRetryWhileDemandClientFailed(): void + { + $networkHost = self::registerHost(new DummyDemandClient()); + /** @var BridgePayment $bridgePayment */ + $bridgePayment = BridgePayment::factory()->create(['address' => $networkHost->address]); + $exchangeRateReader = self::createMock(ExchangeRateReader::class); + $exchangeRateReader->method('fetchExchangeRate') + ->willReturn(new ExchangeRate(new DateTime(), 1, 'USD')); + $licenseReader = self::createMock(LicenseReader::class); + $paymentDetailsProcessor = new PaymentDetailsProcessor($exchangeRateReader, $licenseReader); + $networkImpression = NetworkImpression::factory()->create(); + /** @var User $publisher */ + $publisher = User::factory()->create(); + $caseCount = 2; + /** @var Collection $networkCases */ + $networkCases = NetworkCase::factory()->times($caseCount)->create([ + 'network_impression_id' => $networkImpression, + 'publisher_id' => $publisher->uuid, + ]); + $expectedPaidAmount = 1e11; + $expectedLicenseFee = 0; + $demandClient = self::createMock(DemandClient::class); + $demandClient->expects(self::exactly(4))->method('fetchPaymentDetails')->willReturnOnConsecutiveCalls( + [[ + 'case_id' => $networkCases->first()->case_id, + 'event_value' => (int)($expectedPaidAmount / $caseCount), + ]], + $this->throwException(new UnexpectedClientResponseException('test-exception')), + [[ + 'case_id' => $networkCases->last()->case_id, + 'event_value' => (int)($expectedPaidAmount / $caseCount), + ]], + [], + ); + + (new OpenRtbBridge())->processPayments($demandClient, $paymentDetailsProcessor, 1); + (new OpenRtbBridge())->processPayments($demandClient, $paymentDetailsProcessor, 1); + + self::assertDatabaseCount(BridgePayment::class, 1); + self::assertDatabaseHas(BridgePayment::class, ['status' => BridgePayment::STATUS_DONE]); + self::assertEquals($expectedPaidAmount, (new NetworkCasePayment())->sum('total_amount')); + self::assertEquals($expectedLicenseFee, (new NetworkPayment())->sum('amount')); + self::assertDatabaseHas(UserLedgerEntry::class, [ + 'amount' => $expectedPaidAmount, + 'user_id' => $publisher->id, + ]); + self::assertDatabaseCount(NetworkCasePayment::class, $caseCount); + self::assertDatabaseHas(NetworkCasePayment::class, [ + 'bridge_payment_id' => $bridgePayment->id, + 'network_case_id' => $networkCases->first()->id, + ]); + } + + public function testProcessPaymentsWhileHostDeleted(): void + { + $networkHost = self::registerHost(new DummyDemandClient()); + BridgePayment::factory()->create(['address' => $networkHost->address]); + $networkHost->delete(); + $paymentDetailsProcessor = self::createMock(PaymentDetailsProcessor::class); + $demandClient = self::createMock(DemandClient::class); + + (new OpenRtbBridge())->processPayments($demandClient, $paymentDetailsProcessor); + + self::assertDatabaseCount(BridgePayment::class, 1); + self::assertDatabaseHas(BridgePayment::class, ['status' => BridgePayment::STATUS_INVALID]); + self::assertDatabaseEmpty(UserLedgerEntry::class); + self::assertDatabaseEmpty(NetworkCasePayment::class); + } + private function initOpenRtbConfiguration(array $settings = []): void { $mergedSettings = array_merge( @@ -480,4 +612,14 @@ private static function validPaymentResponseEntry(array $merge = [], string $rem } return $data; } + + private function registerHost(DummyDemandClient $demandClient): NetworkHost + { + $info = $demandClient->fetchInfo(new NullUrl()); + return NetworkHost::factory()->create([ + 'address' => '0001-00000000-9B6F', + 'info' => $info, + 'info_url' => $info->getServerUrl() . 'info.json', + ]); + } } From 10098fe21e3614b1ad147ea92c866794d9e4cfd6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82Podkalicki?= Date: Mon, 20 Mar 2023 16:05:36 +0100 Subject: [PATCH 30/55] invalidate statistics for bridge payments --- .../Commands/SupplyProcessPayments.php | 100 ++++++++++++------ .../Dto/ProcessedPaymentsMetaData.php | 49 +++++++++ app/Services/Supply/OpenRtbBridge.php | 9 +- .../Commands/SupplyProcessPaymentsTest.php | 49 +++++++++ 4 files changed, 170 insertions(+), 37 deletions(-) create mode 100644 app/Services/Dto/ProcessedPaymentsMetaData.php diff --git a/app/Console/Commands/SupplyProcessPayments.php b/app/Console/Commands/SupplyProcessPayments.php index 41a813ba8..be5fa726b 100644 --- a/app/Console/Commands/SupplyProcessPayments.php +++ b/app/Console/Commands/SupplyProcessPayments.php @@ -29,6 +29,7 @@ use Adshares\Adserver\Models\NetworkCaseLogsHourlyMeta; use Adshares\Adserver\Models\NetworkHost; use Adshares\Adserver\Services\Dto\PaymentProcessingResult; +use Adshares\Adserver\Services\Dto\ProcessedPaymentsMetaData; use Adshares\Adserver\Services\LicenseFeeSender; use Adshares\Adserver\Services\PaymentDetailsProcessor; use Adshares\Adserver\Services\Supply\OpenRtbBridge; @@ -43,6 +44,8 @@ class SupplyProcessPayments extends BaseCommand { + private const COLUMN_ADS_PAYMENT_ID = 'ads_payment_id'; + private const COLUMN_BRIDGE_PAYMENT_ID = 'bridge_payment_id'; private const TRY_OUT_PERIOD_FOR_EVENT_PAYMENT = '-24 hours'; private const SQL_QUERY_GET_PROCESSED_PAYMENTS_AMOUNT = <<info('Start command ' . $this->getName()); - $adsPayments = AdsPayment::fetchByStatus(AdsPayment::STATUS_EVENT_PAYMENT_CANDIDATE); + $processedAdsPaymentMetaData = $this->processAdsPayments(); + $processedBridgePaymentMetaData = (new OpenRtbBridge())->processPayments( + $this->demandClient, + $this->paymentDetailsProcessor, + (int)$this->option('chunkSize'), + ); + $processedPaymentsForAds = $processedAdsPaymentMetaData->getProcessedPaymentsForAds() + + $processedBridgePaymentMetaData->getProcessedPaymentsForAds(); + $processedPaymentsTotal = $processedAdsPaymentMetaData->getProcessedPaymentsTotal() + + $processedBridgePaymentMetaData->getProcessedPaymentsTotal(); - $earliestTryOutDateTime = new DateTimeImmutable(self::TRY_OUT_PERIOD_FOR_EVENT_PAYMENT); + $this->invalidateUpdatedCasesStatistics($processedAdsPaymentMetaData, $processedBridgePaymentMetaData); + ServerEvent::dispatch(ServerEventType::IncomingAdPaymentProcessed, [ + 'adsPaymentCount' => $processedPaymentsForAds, + 'totalPaymentCount' => $processedPaymentsTotal, + ]); + + $this->info('End command ' . $this->getName()); + } + + private function processAdsPayments(): ProcessedPaymentsMetaData + { $processedAdsPaymentIds = []; $processedPaymentsTotal = 0; $processedPaymentsForAds = 0; + $adsPayments = AdsPayment::fetchByStatus(AdsPayment::STATUS_EVENT_PAYMENT_CANDIDATE); + $earliestTryOutDateTime = new DateTimeImmutable(self::TRY_OUT_PERIOD_FOR_EVENT_PAYMENT); + /** @var AdsPayment $adsPayment */ foreach ($adsPayments as $adsPayment) { if ($adsPayment->created_at < $earliestTryOutDateTime) { @@ -126,26 +154,11 @@ public function handle(): void ); } } - - $bridge = new OpenRtbBridge(); - $processed = $bridge->processPayments( - $this->demandClient, - $this->paymentDetailsProcessor, - (int)$this->option('chunkSize'), + return new ProcessedPaymentsMetaData( + $processedAdsPaymentIds, + $processedPaymentsTotal, + $processedPaymentsForAds, ); - $processedPaymentsForAds += $processed; - $processedPaymentsTotal += $processed; - - $timestamps = $this->fetchTimestampsToUpdate($processedAdsPaymentIds); - foreach ($timestamps as $timestamp) { - NetworkCaseLogsHourlyMeta::invalidate($timestamp); - } - ServerEvent::dispatch(ServerEventType::IncomingAdPaymentProcessed, [ - 'adsPaymentCount' => $processedPaymentsForAds, - 'totalPaymentCount' => $processedPaymentsTotal, - ]); - - $this->info('End command ' . $this->getName()); } private function handleEventPaymentCandidate(AdsPayment $incomingPayment): void @@ -211,24 +224,43 @@ private function handleEventPaymentCandidate(AdsPayment $incomingPayment): void } } - private function fetchTimestampsToUpdate(array $adsPaymentIds): array + private function invalidateUpdatedCasesStatistics( + ProcessedPaymentsMetaData $processedAdsPaymentMetaData, + ProcessedPaymentsMetaData $processedBridgePaymentMetaData, + ): void { + $timestamps = array_unique( + array_merge( + $this->fetchTimestampsToUpdate( + $processedAdsPaymentMetaData->getPaymentIds(), + self::COLUMN_ADS_PAYMENT_ID, + ), + $this->fetchTimestampsToUpdate( + $processedBridgePaymentMetaData->getPaymentIds(), + self::COLUMN_BRIDGE_PAYMENT_ID, + ), + ), + ); + foreach ($timestamps as $timestamp) { + NetworkCaseLogsHourlyMeta::invalidate($timestamp); + } + } + + private function fetchTimestampsToUpdate(array $paymentIds, string $paymentIdColumn): array { - if (empty($adsPaymentIds)) { + if (empty($paymentIds)) { return []; } - $whereInPlaceholder = str_repeat('?,', count($adsPaymentIds) - 1) . '?'; - $query = sprintf( + $whereInPlaceholder = str_repeat('?,', count($paymentIds) - 1) . '?'; + $query = str_replace( + [':paymentIdColumn', ':whereInPlaceholder'], + [$paymentIdColumn, $whereInPlaceholder], self::SQL_QUERY_SELECT_TIMESTAMPS_TO_UPDATE_TEMPLATE, - $whereInPlaceholder, - $whereInPlaceholder ); return array_map( - function (stdClass $item) { - return (int)$item->pay_time; - }, - DB::select($query, array_merge($adsPaymentIds, $adsPaymentIds)) + fn(stdClass $item) => (int)$item->pay_time, + DB::select($query, array_merge($paymentIds, $paymentIds)) ); } } diff --git a/app/Services/Dto/ProcessedPaymentsMetaData.php b/app/Services/Dto/ProcessedPaymentsMetaData.php new file mode 100644 index 000000000..9c8c6da3c --- /dev/null +++ b/app/Services/Dto/ProcessedPaymentsMetaData.php @@ -0,0 +1,49 @@ + + */ + +declare(strict_types=1); + +namespace Adshares\Adserver\Services\Dto; + +final class ProcessedPaymentsMetaData +{ + public function __construct( + private readonly array $paymentIds, + private readonly int $processedPaymentsTotal, + private readonly int $processedPaymentsForAds, + ) { + } + + public function getPaymentIds(): array + { + return $this->paymentIds; + } + + public function getProcessedPaymentsTotal(): int + { + return $this->processedPaymentsTotal; + } + + public function getProcessedPaymentsForAds(): int + { + return $this->processedPaymentsForAds; + } +} diff --git a/app/Services/Supply/OpenRtbBridge.php b/app/Services/Supply/OpenRtbBridge.php index 68c0500f9..52a2231c2 100644 --- a/app/Services/Supply/OpenRtbBridge.php +++ b/app/Services/Supply/OpenRtbBridge.php @@ -25,6 +25,7 @@ use Adshares\Adserver\Http\Utils; use Adshares\Adserver\Models\BridgePayment; use Adshares\Adserver\Models\NetworkHost; +use Adshares\Adserver\Services\Dto\ProcessedPaymentsMetaData; use Adshares\Adserver\Services\PaymentDetailsProcessor; use Adshares\Supply\Application\Dto\FoundBanners; use Adshares\Supply\Application\Dto\ImpressionContext; @@ -156,8 +157,9 @@ public function fetchAndStorePayments(): void public function processPayments( DemandClient $demandClient, PaymentDetailsProcessor $paymentDetailsProcessor, - int $limit = 500, - ): int { + int $limit = 5000, + ): ProcessedPaymentsMetaData { + $paymentIds = []; $processedPaymentsCount = 0; /** @var Collection $bridgePayments */ $bridgePayments = BridgePayment::fetchNew(); @@ -205,9 +207,10 @@ public function processPayments( $bridgePayment->status = BridgePayment::STATUS_DONE; $bridgePayment->save(); + $paymentIds[] = $bridgePayment->id; ++$processedPaymentsCount; } - return $processedPaymentsCount; + return new ProcessedPaymentsMetaData($paymentIds, $processedPaymentsCount, $processedPaymentsCount); } private function isPaymentResponseValid(mixed $content): bool diff --git a/tests/app/Console/Commands/SupplyProcessPaymentsTest.php b/tests/app/Console/Commands/SupplyProcessPaymentsTest.php index 2a6224f93..794a54f7f 100644 --- a/tests/app/Console/Commands/SupplyProcessPaymentsTest.php +++ b/tests/app/Console/Commands/SupplyProcessPaymentsTest.php @@ -23,6 +23,7 @@ use Adshares\Adserver\Console\Locker; use Adshares\Adserver\Models\AdsPayment; +use Adshares\Adserver\Models\BridgePayment; use Adshares\Adserver\Models\NetworkCase; use Adshares\Adserver\Models\NetworkCaseLogsHourlyMeta; use Adshares\Adserver\Models\NetworkCasePayment; @@ -30,6 +31,7 @@ use Adshares\Adserver\Models\NetworkImpression; use Adshares\Adserver\Models\NetworkPayment; use Adshares\Adserver\Models\User; +use Adshares\Adserver\Models\UserLedgerEntry; use Adshares\Adserver\Services\PaymentDetailsProcessor; use Adshares\Adserver\Tests\Console\ConsoleTestCase; use Adshares\Adserver\ViewModel\ServerEventType; @@ -42,6 +44,7 @@ use Adshares\Supply\Application\Service\Exception\UnexpectedClientResponseException; use DateTimeImmutable; use Illuminate\Http\Response; +use Illuminate\Support\Collection; class SupplyProcessPaymentsTest extends ConsoleTestCase { @@ -165,6 +168,52 @@ public function testAdsProcessEventPayment(): void self::assertAdPaymentProcessedEventDispatched(1); } + public function testHandleBridgePayment(): void + { + $networkHost = self::registerHost(new DummyDemandClient()); + /** @var BridgePayment $bridgePayment */ + $bridgePayment = BridgePayment::factory()->create(['address' => $networkHost->address]); + $networkImpression = NetworkImpression::factory()->create(); + /** @var User $publisher */ + $publisher = User::factory()->create(); + $caseCount = 2; + /** @var Collection $networkCases */ + $networkCases = NetworkCase::factory()->times($caseCount)->create([ + 'network_impression_id' => $networkImpression, + 'publisher_id' => $publisher->uuid, + ]); + $expectedPaidAmount = 1e11; + $expectedLicenseFee = 0; + $demandClient = self::createMock(DemandClient::class); + $demandClient->expects(self::once())->method('fetchPaymentDetails')->willReturn( + $networkCases->map( + fn($case) => [ + 'case_id' => $case->case_id, + 'event_value' => (int)($expectedPaidAmount / $caseCount), + ], + )->toArray() + ); + $this->app->bind(DemandClient::class, fn() => $demandClient); + + $this->artisan(self::SIGNATURE, ['--chunkSize' => 500])->assertExitCode(0); + + self::assertDatabaseCount(BridgePayment::class, 1); + self::assertDatabaseHas(BridgePayment::class, ['status' => BridgePayment::STATUS_DONE]); + self::assertEquals($expectedPaidAmount, (new NetworkCasePayment())->sum('total_amount')); + self::assertEquals($expectedLicenseFee, (new NetworkPayment())->sum('amount')); + self::assertGreaterThan(0, NetworkCaseLogsHourlyMeta::fetchInvalid()->count()); + self::assertAdPaymentProcessedEventDispatched(1); + self::assertDatabaseHas(UserLedgerEntry::class, [ + 'amount' => $expectedPaidAmount, + 'user_id' => $publisher->id, + ]); + self::assertDatabaseCount(NetworkCasePayment::class, $caseCount); + self::assertDatabaseHas(NetworkCasePayment::class, [ + 'bridge_payment_id' => $bridgePayment->id, + 'network_case_id' => $networkCases->first()->id, + ]); + } + public function testAdsProcessEventPaymentWithPaymentProcessorError(): void { $paymentDetailsProcessor = self::createMock(PaymentDetailsProcessor::class); From f2c25bf3d91441affc3cc4e2081d669ca4490c79 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82Podkalicki?= Date: Mon, 20 Mar 2023 16:50:48 +0100 Subject: [PATCH 31/55] add transaction --- .../Commands/SupplyProcessPayments.php | 2 +- app/Services/Supply/OpenRtbBridge.php | 68 +++++++---- .../app/Services/Supply/OpenRtbBridgeTest.php | 115 +++++++++++++----- 3 files changed, 127 insertions(+), 58 deletions(-) diff --git a/app/Console/Commands/SupplyProcessPayments.php b/app/Console/Commands/SupplyProcessPayments.php index be5fa726b..3d930cc1a 100644 --- a/app/Console/Commands/SupplyProcessPayments.php +++ b/app/Console/Commands/SupplyProcessPayments.php @@ -147,7 +147,7 @@ private function processAdsPayments(): ProcessedPaymentsMetaData DB::rollBack(); $this->error( sprintf( - 'Error during handling paid events for id=%d (%s)', + 'Error during handling paid events for ads payment id=%d (%s)', $adsPayment->id, $throwable->getMessage() ) diff --git a/app/Services/Supply/OpenRtbBridge.php b/app/Services/Supply/OpenRtbBridge.php index 52a2231c2..585763e75 100644 --- a/app/Services/Supply/OpenRtbBridge.php +++ b/app/Services/Supply/OpenRtbBridge.php @@ -39,6 +39,7 @@ use Illuminate\Support\Facades\Http; use Illuminate\Support\Facades\Log; use Symfony\Component\HttpFoundation\Response as BaseResponse; +use Throwable; class OpenRtbBridge { @@ -179,36 +180,49 @@ public function processPayments( } $transactionTime = $bridgePayment->payment_time; - do { - try { - $paymentDetails = $demandClient->fetchPaymentDetails( - $networkHost->host, - $bridgePayment->payment_id, - $limit, - $offset, - ); - } catch (EmptyInventoryException | UnexpectedClientResponseException) { - $bridgePayment->save(); - break 2; - } - - $processPaymentDetails = $paymentDetailsProcessor->processEventsPaidByBridge( - $bridgePayment, - $transactionTime, - $paymentDetails, - $eventValueSum, - ); - $eventValueSum += $processPaymentDetails->eventValuePartialSum(); + DB::beginTransaction(); + try { + do { + try { + $paymentDetails = $demandClient->fetchPaymentDetails( + $networkHost->host, + $bridgePayment->payment_id, + $limit, + $offset, + ); + } catch (EmptyInventoryException | UnexpectedClientResponseException) { + $bridgePayment->save(); + DB::commit(); + continue 2; + } - $bridgePayment->last_offset = $offset += $limit; - } while (count($paymentDetails) === $limit); + $processPaymentDetails = $paymentDetailsProcessor->processEventsPaidByBridge( + $bridgePayment, + $transactionTime, + $paymentDetails, + $eventValueSum, + ); + $eventValueSum += $processPaymentDetails->eventValuePartialSum(); - $paymentDetailsProcessor->addBridgeAdIncomeToUserLedger($bridgePayment); + $bridgePayment->last_offset = $offset += $limit; + } while (count($paymentDetails) === $limit); - $bridgePayment->status = BridgePayment::STATUS_DONE; - $bridgePayment->save(); - $paymentIds[] = $bridgePayment->id; - ++$processedPaymentsCount; + $paymentDetailsProcessor->addBridgeAdIncomeToUserLedger($bridgePayment); + $bridgePayment->status = BridgePayment::STATUS_DONE; + $bridgePayment->save(); + DB::commit(); + $paymentIds[] = $bridgePayment->id; + ++$processedPaymentsCount; + } catch (Throwable $throwable) { + DB::rollBack(); + Log::error( + sprintf( + 'Error during handling paid events for bridge payment id=%d (%s)', + $bridgePayment->id, + $throwable->getMessage() + ) + ); + } } return new ProcessedPaymentsMetaData($paymentIds, $processedPaymentsCount, $processedPaymentsCount); } diff --git a/tests/app/Services/Supply/OpenRtbBridgeTest.php b/tests/app/Services/Supply/OpenRtbBridgeTest.php index fe438da20..fbe658a9e 100644 --- a/tests/app/Services/Supply/OpenRtbBridgeTest.php +++ b/tests/app/Services/Supply/OpenRtbBridgeTest.php @@ -58,6 +58,8 @@ use Illuminate\Support\Collection; use Illuminate\Support\Facades\Http; use Illuminate\Support\Facades\Log; +use Mockery; +use PDOException; use Ramsey\Uuid\Uuid; use Symfony\Component\HttpFoundation\Response; @@ -408,14 +410,8 @@ public function fetchAndStorePaymentsWhileInvalidResponseProvider(): array public function testProcessPayments(): void { - $networkHost = self::registerHost(new DummyDemandClient()); - /** @var BridgePayment $bridgePayment */ - $bridgePayment = BridgePayment::factory()->create(['address' => $networkHost->address]); - $exchangeRateReader = self::createMock(ExchangeRateReader::class); - $exchangeRateReader->method('fetchExchangeRate') - ->willReturn(new ExchangeRate(new DateTime(), 1, 'USD')); - $licenseReader = self::createMock(LicenseReader::class); - $paymentDetailsProcessor = new PaymentDetailsProcessor($exchangeRateReader, $licenseReader); + $bridgePayment = $this->initBridgePaymentWithHost(); + $paymentDetailsProcessor = $this->mockPaymentDetailsProcessor(); $networkImpression = NetworkImpression::factory()->create(); /** @var User $publisher */ $publisher = User::factory()->create(); @@ -425,17 +421,17 @@ public function testProcessPayments(): void 'network_impression_id' => $networkImpression, 'publisher_id' => $publisher->uuid, ]); - $expectedPaidAmount = 1e11; - $expectedLicenseFee = 0; $demandClient = self::createMock(DemandClient::class); $demandClient->expects(self::once())->method('fetchPaymentDetails')->willReturn( $networkCases->map( fn($case) => [ 'case_id' => $case->case_id, - 'event_value' => (int)($expectedPaidAmount / $caseCount), + 'event_value' => (int)(1e11 / $caseCount), ], )->toArray() ); + $expectedPaidAmount = 1e11; + $expectedLicenseFee = 0; (new OpenRtbBridge())->processPayments($demandClient, $paymentDetailsProcessor); @@ -456,14 +452,8 @@ public function testProcessPayments(): void public function testProcessPaymentsRetryWhileDemandClientFailed(): void { - $networkHost = self::registerHost(new DummyDemandClient()); - /** @var BridgePayment $bridgePayment */ - $bridgePayment = BridgePayment::factory()->create(['address' => $networkHost->address]); - $exchangeRateReader = self::createMock(ExchangeRateReader::class); - $exchangeRateReader->method('fetchExchangeRate') - ->willReturn(new ExchangeRate(new DateTime(), 1, 'USD')); - $licenseReader = self::createMock(LicenseReader::class); - $paymentDetailsProcessor = new PaymentDetailsProcessor($exchangeRateReader, $licenseReader); + $bridgePayment = $this->initBridgePaymentWithHost(); + $paymentDetailsProcessor = $this->mockPaymentDetailsProcessor(); $networkImpression = NetworkImpression::factory()->create(); /** @var User $publisher */ $publisher = User::factory()->create(); @@ -477,19 +467,26 @@ public function testProcessPaymentsRetryWhileDemandClientFailed(): void $expectedLicenseFee = 0; $demandClient = self::createMock(DemandClient::class); $demandClient->expects(self::exactly(4))->method('fetchPaymentDetails')->willReturnOnConsecutiveCalls( - [[ - 'case_id' => $networkCases->first()->case_id, - 'event_value' => (int)($expectedPaidAmount / $caseCount), - ]], + [ + [ + 'case_id' => $networkCases->first()->case_id, + 'event_value' => (int)($expectedPaidAmount / $caseCount), + ] + ], $this->throwException(new UnexpectedClientResponseException('test-exception')), - [[ - 'case_id' => $networkCases->last()->case_id, - 'event_value' => (int)($expectedPaidAmount / $caseCount), - ]], + [ + [ + 'case_id' => $networkCases->last()->case_id, + 'event_value' => (int)($expectedPaidAmount / $caseCount), + ] + ], [], ); (new OpenRtbBridge())->processPayments($demandClient, $paymentDetailsProcessor, 1); + + self::assertEquals(1, $bridgePayment->refresh()->last_offset); + (new OpenRtbBridge())->processPayments($demandClient, $paymentDetailsProcessor, 1); self::assertDatabaseCount(BridgePayment::class, 1); @@ -507,11 +504,48 @@ public function testProcessPaymentsRetryWhileDemandClientFailed(): void ]); } + public function testProcessPaymentsWhilePaymentCannotBeAddedToPublisherAccount(): void + { + $this->initBridgePaymentWithHost(); + $exchangeRateReader = self::createMock(ExchangeRateReader::class); + $exchangeRateReader->method('fetchExchangeRate') + ->willReturn(new ExchangeRate(new DateTime(), 1, 'USD')); + $licenseReader = self::createMock(LicenseReader::class); + $paymentDetailsProcessor = Mockery::mock(new PaymentDetailsProcessor($exchangeRateReader, $licenseReader)); + $paymentDetailsProcessor->shouldReceive('addBridgeAdIncomeToUserLedger') + ->andThrow(new PDOException('test-exception')); + $networkImpression = NetworkImpression::factory()->create(); + /** @var User $publisher */ + $publisher = User::factory()->create(); + $caseCount = 2; + /** @var Collection $networkCases */ + $networkCases = NetworkCase::factory()->times($caseCount)->create([ + 'network_impression_id' => $networkImpression, + 'publisher_id' => $publisher->uuid, + ]); + $demandClient = self::createMock(DemandClient::class); + $demandClient->expects(self::once())->method('fetchPaymentDetails')->willReturn( + $networkCases->map( + fn($case) => [ + 'case_id' => $case->case_id, + 'event_value' => (int)(1e11 / $caseCount), + ], + )->toArray() + ); + + (new OpenRtbBridge())->processPayments($demandClient, $paymentDetailsProcessor); + + self::assertDatabaseCount(BridgePayment::class, 1); + self::assertDatabaseHas(BridgePayment::class, ['status' => BridgePayment::STATUS_NEW]); + self::assertDatabaseEmpty(UserLedgerEntry::class); + self::assertDatabaseEmpty(NetworkCasePayment::class); + self::assertDatabaseEmpty(NetworkPayment::class); + } + public function testProcessPaymentsWhileHostDeleted(): void { - $networkHost = self::registerHost(new DummyDemandClient()); - BridgePayment::factory()->create(['address' => $networkHost->address]); - $networkHost->delete(); + $this->initBridgePaymentWithHost(); + NetworkHost::first()->delete(); $paymentDetailsProcessor = self::createMock(PaymentDetailsProcessor::class); $demandClient = self::createMock(DemandClient::class); @@ -622,4 +656,25 @@ private function registerHost(DummyDemandClient $demandClient): NetworkHost 'info_url' => $info->getServerUrl() . 'info.json', ]); } + + private function initBridgePaymentWithHost(): BridgePayment + { + $info = (new DummyDemandClient())->fetchInfo(new NullUrl()); + /** @var NetworkHost $networkHost */ + $networkHost = NetworkHost::factory()->create([ + 'address' => '0001-00000000-9B6F', + 'info' => $info, + 'info_url' => $info->getServerUrl() . 'info.json', + ]); + return BridgePayment::factory()->create(['address' => $networkHost->address]); + } + + private function mockPaymentDetailsProcessor(): PaymentDetailsProcessor + { + $exchangeRateReader = self::createMock(ExchangeRateReader::class); + $exchangeRateReader->method('fetchExchangeRate') + ->willReturn(new ExchangeRate(new DateTime(), 1, 'USD')); + $licenseReader = self::createMock(LicenseReader::class); + return new PaymentDetailsProcessor($exchangeRateReader, $licenseReader); + } } From 6e35b14c4761d42f72a74bcd71e84511f93fc1be Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82Podkalicki?= Date: Mon, 20 Mar 2023 17:15:59 +0100 Subject: [PATCH 32/55] fetch payments before processing --- .../Commands/SupplyProcessPayments.php | 60 +++++++++---------- .../Commands/SupplyProcessPaymentsTest.php | 34 +++++++---- .../app/Services/Supply/OpenRtbBridgeTest.php | 10 ---- 3 files changed, 52 insertions(+), 52 deletions(-) diff --git a/app/Console/Commands/SupplyProcessPayments.php b/app/Console/Commands/SupplyProcessPayments.php index 3d930cc1a..11dfce251 100644 --- a/app/Console/Commands/SupplyProcessPayments.php +++ b/app/Console/Commands/SupplyProcessPayments.php @@ -94,17 +94,38 @@ public function handle(): void $this->info('Start command ' . $this->getName()); $processedAdsPaymentMetaData = $this->processAdsPayments(); - $processedBridgePaymentMetaData = (new OpenRtbBridge())->processPayments( - $this->demandClient, - $this->paymentDetailsProcessor, - (int)$this->option('chunkSize'), + $processedPaymentsForAds = $processedAdsPaymentMetaData->getProcessedPaymentsForAds(); + $processedPaymentsTotal = $processedAdsPaymentMetaData->getProcessedPaymentsTotal(); + $timestamps = $this->fetchTimestampsToUpdate( + $processedAdsPaymentMetaData->getPaymentIds(), + self::COLUMN_ADS_PAYMENT_ID, ); - $processedPaymentsForAds = $processedAdsPaymentMetaData->getProcessedPaymentsForAds() - + $processedBridgePaymentMetaData->getProcessedPaymentsForAds(); - $processedPaymentsTotal = $processedAdsPaymentMetaData->getProcessedPaymentsTotal() - + $processedBridgePaymentMetaData->getProcessedPaymentsTotal(); - $this->invalidateUpdatedCasesStatistics($processedAdsPaymentMetaData, $processedBridgePaymentMetaData); + if (OpenRtbBridge::isActive()) { + $openRtbBridge = new OpenRtbBridge(); + $openRtbBridge->fetchAndStorePayments(); + + $processedBridgePaymentMetaData = $openRtbBridge->processPayments( + $this->demandClient, + $this->paymentDetailsProcessor, + (int)$this->option('chunkSize'), + ); + $processedPaymentsForAds += $processedBridgePaymentMetaData->getProcessedPaymentsForAds(); + $processedPaymentsTotal += $processedBridgePaymentMetaData->getProcessedPaymentsTotal(); + $timestamps = array_unique( + array_merge( + $timestamps, + $this->fetchTimestampsToUpdate( + $processedBridgePaymentMetaData->getPaymentIds(), + self::COLUMN_BRIDGE_PAYMENT_ID, + ), + ) + ); + } + + foreach ($timestamps as $timestamp) { + NetworkCaseLogsHourlyMeta::invalidate($timestamp); + } ServerEvent::dispatch(ServerEventType::IncomingAdPaymentProcessed, [ 'adsPaymentCount' => $processedPaymentsForAds, 'totalPaymentCount' => $processedPaymentsTotal, @@ -224,27 +245,6 @@ private function handleEventPaymentCandidate(AdsPayment $incomingPayment): void } } - private function invalidateUpdatedCasesStatistics( - ProcessedPaymentsMetaData $processedAdsPaymentMetaData, - ProcessedPaymentsMetaData $processedBridgePaymentMetaData, - ): void { - $timestamps = array_unique( - array_merge( - $this->fetchTimestampsToUpdate( - $processedAdsPaymentMetaData->getPaymentIds(), - self::COLUMN_ADS_PAYMENT_ID, - ), - $this->fetchTimestampsToUpdate( - $processedBridgePaymentMetaData->getPaymentIds(), - self::COLUMN_BRIDGE_PAYMENT_ID, - ), - ), - ); - foreach ($timestamps as $timestamp) { - NetworkCaseLogsHourlyMeta::invalidate($timestamp); - } - } - private function fetchTimestampsToUpdate(array $paymentIds, string $paymentIdColumn): array { if (empty($paymentIds)) { diff --git a/tests/app/Console/Commands/SupplyProcessPaymentsTest.php b/tests/app/Console/Commands/SupplyProcessPaymentsTest.php index 794a54f7f..27f961bd5 100644 --- a/tests/app/Console/Commands/SupplyProcessPaymentsTest.php +++ b/tests/app/Console/Commands/SupplyProcessPaymentsTest.php @@ -24,6 +24,7 @@ use Adshares\Adserver\Console\Locker; use Adshares\Adserver\Models\AdsPayment; use Adshares\Adserver\Models\BridgePayment; +use Adshares\Adserver\Models\Config; use Adshares\Adserver\Models\NetworkCase; use Adshares\Adserver\Models\NetworkCaseLogsHourlyMeta; use Adshares\Adserver\Models\NetworkCasePayment; @@ -34,6 +35,7 @@ use Adshares\Adserver\Models\UserLedgerEntry; use Adshares\Adserver\Services\PaymentDetailsProcessor; use Adshares\Adserver\Tests\Console\ConsoleTestCase; +use Adshares\Adserver\Utilities\DatabaseConfigReader; use Adshares\Adserver\ViewModel\ServerEventType; use Adshares\Common\Application\Service\LicenseVault; use Adshares\Common\Domain\ValueObject\NullUrl; @@ -54,8 +56,7 @@ class SupplyProcessPaymentsTest extends ConsoleTestCase public function testAdsProcessOutdated(): void { - $demandClient = new DummyDemandClient(); - $networkHost = self::registerHost($demandClient); + $networkHost = self::registerHost(); $createdAt = new DateTimeImmutable('-30 hours'); AdsPayment::factory()->create([ @@ -88,8 +89,7 @@ public function testAdsProcessMissingHost(): void public function testAdsProcessDepositWithoutUser(): void { - $demandClient = new DummyDemandClient(); - $networkHost = self::registerHost($demandClient); + $networkHost = self::registerHost(); AdsPayment::factory()->create([ 'txid' => self::TX_ID_SEND_ONE, @@ -119,7 +119,7 @@ function () { public function testAdsProcessEventPayment(): void { $demandClient = new DummyDemandClient(); - $networkHost = self::registerHost($demandClient); + $networkHost = self::registerHost(); $networkImpression = NetworkImpression::factory()->create(); $paymentDetails = $demandClient->fetchPaymentDetails('', '', 333, 0); @@ -170,7 +170,8 @@ public function testAdsProcessEventPayment(): void public function testHandleBridgePayment(): void { - $networkHost = self::registerHost(new DummyDemandClient()); + $this->initOpenRtbConfiguration(); + $networkHost = self::registerHost(); /** @var BridgePayment $bridgePayment */ $bridgePayment = BridgePayment::factory()->create(['address' => $networkHost->address]); $networkImpression = NetworkImpression::factory()->create(); @@ -221,13 +222,13 @@ public function testAdsProcessEventPaymentWithPaymentProcessorError(): void $this->app->bind(PaymentDetailsProcessor::class, fn() => $paymentDetailsProcessor); $demandClient = new DummyDemandClient(); - $networkHost = self::registerHost($demandClient); + $networkHost = self::registerHost(); $networkImpression = NetworkImpression::factory()->create(); $paymentDetail = $demandClient->fetchPaymentDetails('', '', 1, 0)[0]; NetworkCase::factory()->create([ 'case_id' => $paymentDetail['case_id'], - 'network_impression_id' => $networkImpression->id, + 'network_impression_id' => $networkImpression, 'publisher_id' => $paymentDetail['publisher_id'], ]); @@ -250,7 +251,7 @@ public function testAdsProcessEventPaymentWithPaymentProcessorError(): void public function testAdsProcessEventPaymentWithServerError(): void { $demandClient = new DummyDemandClient(); - $networkHost = self::registerHost($demandClient); + $networkHost = self::registerHost(); $networkImpression = NetworkImpression::factory()->create(); $paymentDetails = $demandClient->fetchPaymentDetails('', '', 333, 0); @@ -339,7 +340,7 @@ function ( public function testAdsProcessEventPaymentWithNoLicense(): void { $demandClient = new DummyDemandClient(); - $networkHost = self::registerHost($demandClient); + $networkHost = self::registerHost(); /** @var NetworkImpression $networkImpression */ $networkImpression = NetworkImpression::factory()->create(); @@ -386,9 +387,9 @@ public function testLock(): void $this->artisan(self::SIGNATURE)->assertExitCode(0); } - private function registerHost(DummyDemandClient $demandClient): NetworkHost + private function registerHost(): NetworkHost { - $info = $demandClient->fetchInfo(new NullUrl()); + $info = (new DummyDemandClient())->fetchInfo(new NullUrl()); return NetworkHost::factory()->create([ 'address' => '0001-00000000-9B6F', 'info' => $info, @@ -402,4 +403,13 @@ private static function assertAdPaymentProcessedEventDispatched(int $count = 0): 'adsPaymentCount' => $count, ]); } + + private static function initOpenRtbConfiguration(): void + { + Config::updateAdminSettings([ + Config::OPEN_RTB_BRIDGE_ACCOUNT_ADDRESS => '0001-00000001-8B4E', + Config::OPEN_RTB_BRIDGE_URL => 'https://example.com', + ]); + DatabaseConfigReader::overwriteAdministrationConfig(); + } } diff --git a/tests/app/Services/Supply/OpenRtbBridgeTest.php b/tests/app/Services/Supply/OpenRtbBridgeTest.php index fbe658a9e..6b82ca002 100644 --- a/tests/app/Services/Supply/OpenRtbBridgeTest.php +++ b/tests/app/Services/Supply/OpenRtbBridgeTest.php @@ -647,16 +647,6 @@ private static function validPaymentResponseEntry(array $merge = [], string $rem return $data; } - private function registerHost(DummyDemandClient $demandClient): NetworkHost - { - $info = $demandClient->fetchInfo(new NullUrl()); - return NetworkHost::factory()->create([ - 'address' => '0001-00000000-9B6F', - 'info' => $info, - 'info_url' => $info->getServerUrl() . 'info.json', - ]); - } - private function initBridgePaymentWithHost(): BridgePayment { $info = (new DummyDemandClient())->fetchInfo(new NullUrl()); From 4dfa914c5df211e02add8a0902d9963b9c21888f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82Podkalicki?= Date: Tue, 21 Mar 2023 10:32:07 +0100 Subject: [PATCH 33/55] add test case --- .../Services/PaymentDetailsProcessorTest.php | 65 +++++++++++++++---- 1 file changed, 54 insertions(+), 11 deletions(-) diff --git a/tests/app/Services/PaymentDetailsProcessorTest.php b/tests/app/Services/PaymentDetailsProcessorTest.php index e7c94fd93..bc5b89010 100644 --- a/tests/app/Services/PaymentDetailsProcessorTest.php +++ b/tests/app/Services/PaymentDetailsProcessorTest.php @@ -43,6 +43,8 @@ use DateTime; use DateTimeImmutable; use Illuminate\Support\Collection; +use Illuminate\Support\Facades\Log; +use Mockery; final class PaymentDetailsProcessorTest extends TestCase { @@ -207,6 +209,25 @@ public function currencyProvider(): array ]; } + public function testAddAdIncomeToUserLedgerWhenNoUser(): void + { + $adsPayment = $this->createAdsPayment(100_000_000_000); + /** @var NetworkCase $networkCase */ + $networkCase = NetworkCase::factory()->create([ + 'publisher_id' => '10000000000000000000000000000000', + ]); + /** @var NetworkCasePayment $networkCasePayment */ + NetworkCasePayment::factory()->create([ + 'network_case_id' => $networkCase->id, + 'ads_payment_id' => $adsPayment->id, + ]); + $paymentDetailsProcessor = $this->getPaymentDetailsProcessor(); + + $paymentDetailsProcessor->addAdIncomeToUserLedger($adsPayment); + + self::assertDatabaseEmpty(UserLedgerEntry::class); + } + public function testBridgeAdIncomeToUserLedger(): void { $payment = BridgePayment::factory()->create(); @@ -218,7 +239,7 @@ public function testBridgeAdIncomeToUserLedger(): void $rate = 5.0; $paidAmountCurrency = (int)floor($paidAmount * $rate); NetworkCasePayment::factory()->create([ - 'bridge_payment_id' => $payment->id, + 'bridge_payment_id' => $payment, 'exchange_rate' => $rate, 'license_fee' => 0, 'network_case_id' => $networkCase->id, @@ -237,24 +258,46 @@ public function testBridgeAdIncomeToUserLedger(): void self::assertEquals($expectedPaidAmount, $entries->first()->amount); } - public function testAddAdIncomeToUserLedgerWhenNoUser(): void + public function testBridgeAdIncomeToUserLedgerWhenNoUser(): void { - $adsPayment = $this->createAdsPayment(100_000_000_000); + Log::spy(); + $payment = BridgePayment::factory()->create(); /** @var NetworkCase $networkCase */ - $networkCase = NetworkCase::factory()->create([ - 'publisher_id' => '10000000000000000000000000000000', - ]); - /** @var NetworkCasePayment $networkCasePayment */ + $networkCase = NetworkCase::factory()->create(['publisher_id' => '10000000000000000000000000000000']); + $paidAmount = 100_000_000; + $rate = 5.0; + $paidAmountCurrency = (int)floor($paidAmount * $rate); NetworkCasePayment::factory()->create([ + 'bridge_payment_id' => $payment, + 'exchange_rate' => $rate, + 'license_fee' => 0, 'network_case_id' => $networkCase->id, - 'ads_payment_id' => $adsPayment->id, + 'operator_fee' => 0, + 'paid_amount' => $paidAmount, + 'paid_amount_currency' => $paidAmountCurrency, + 'total_amount' => $paidAmount, ]); $paymentDetailsProcessor = $this->getPaymentDetailsProcessor(); - $paymentDetailsProcessor->addAdIncomeToUserLedger($adsPayment); + $paymentDetailsProcessor->addBridgeAdIncomeToUserLedger($payment); - $entries = UserLedgerEntry::all(); - self::assertCount(0, $entries); + self::assertDatabaseEmpty(UserLedgerEntry::class); + Log::shouldHaveReceived('warning') + ->with( + Mockery::on(function ($argument) { + if ( + preg_match( + '/^\[PaymentDetailsProcessor] User id \(10000000000000000000000000000000\) does not exist. ' + . 'BridgePayment id \(\d+\). Amount \(\d+\)./', + $argument + ) + ) { + return true; + } + return false; + }) + ) + ->once(); } private function getExchangeRateReader(): ExchangeRateReader From ab5bb8523f4270c7d7a3897002dbfff512716a99 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82Podkalicki?= Date: Tue, 21 Mar 2023 12:09:00 +0100 Subject: [PATCH 34/55] restrict payment id length to 18 chars --- app/Services/PaymentDetailsProcessor.php | 2 +- app/Services/Supply/OpenRtbBridge.php | 18 ++++++------ ...17_144439_create_bridge_payments_table.php | 2 +- .../Services/PaymentDetailsProcessorTest.php | 29 ++++++++++--------- .../app/Services/Supply/OpenRtbBridgeTest.php | 10 ++++++- 5 files changed, 36 insertions(+), 25 deletions(-) diff --git a/app/Services/PaymentDetailsProcessor.php b/app/Services/PaymentDetailsProcessor.php index 3dff82571..b2e51c62b 100644 --- a/app/Services/PaymentDetailsProcessor.php +++ b/app/Services/PaymentDetailsProcessor.php @@ -197,7 +197,7 @@ public function addBridgeAdIncomeToUserLedger(BridgePayment $payment): void UserLedgerEntry::TYPE_AD_INCOME, $payment->address, $adServerAddress, - '', + $payment->payment_id, )->save(); } } diff --git a/app/Services/Supply/OpenRtbBridge.php b/app/Services/Supply/OpenRtbBridge.php index 585763e75..f413d0131 100644 --- a/app/Services/Supply/OpenRtbBridge.php +++ b/app/Services/Supply/OpenRtbBridge.php @@ -238,20 +238,20 @@ private function isPaymentResponseValid(mixed $content): bool Log::error('Invalid bridge payments response: entry is not an array'); return false; } - $fields = [ - 'id', - 'created_at', - 'status', - 'value', - ]; - foreach ($fields as $field) { + foreach (['id', 'created_at', 'status', 'value'] as $field) { if (!array_key_exists($field, $entry)) { Log::error(sprintf('Invalid bridge payments response: missing key %s', $field)); return false; } } - if (!is_string($entry['status'])) { - Log::error('Invalid bridge payments response: status is not a string'); + foreach (['id', 'created_at', 'status'] as $field) { + if (!is_string($entry[$field])) { + Log::error(sprintf('Invalid bridge payments response: %s is not a string', $field)); + return false; + } + } + if (18 < strlen($entry['id'])) { + Log::error('Invalid bridge payments response: id must have at most 18 characters'); return false; } if (false === DateTimeImmutable::createFromFormat(DateTimeInterface::ATOM, $entry['created_at'])) { diff --git a/database/migrations/2023_03_17_144439_create_bridge_payments_table.php b/database/migrations/2023_03_17_144439_create_bridge_payments_table.php index 04328557a..cbb5867ee 100644 --- a/database/migrations/2023_03_17_144439_create_bridge_payments_table.php +++ b/database/migrations/2023_03_17_144439_create_bridge_payments_table.php @@ -32,7 +32,7 @@ public function up(): void $table->id(); $table->timestamps(); $table->string('address', 18); - $table->string('payment_id'); + $table->string('payment_id', 18); $table->timestamp('payment_time'); $table->bigInteger('amount')->nullable(); $table->tinyInteger('status')->default(0); diff --git a/tests/app/Services/PaymentDetailsProcessorTest.php b/tests/app/Services/PaymentDetailsProcessorTest.php index bc5b89010..6c61a39ef 100644 --- a/tests/app/Services/PaymentDetailsProcessorTest.php +++ b/tests/app/Services/PaymentDetailsProcessorTest.php @@ -196,9 +196,11 @@ public function testAddAdIncomeToUserLedger(Currency $currency): void $paymentDetailsProcessor = $this->getPaymentDetailsProcessor(); $paymentDetailsProcessor->addAdIncomeToUserLedger($adsPayment); - $entries = UserLedgerEntry::all(); - self::assertCount(1, $entries); - self::assertEquals($expectedPaidAmount, $entries->first()->amount); + self::assertDatabaseCount(UserLedgerEntry::class, 1); + self::assertDatabaseHas(UserLedgerEntry::class, [ + 'amount' => $expectedPaidAmount, + 'txid' => $adsPayment->txid, + ]); } public function currencyProvider(): array @@ -230,6 +232,7 @@ public function testAddAdIncomeToUserLedgerWhenNoUser(): void public function testBridgeAdIncomeToUserLedger(): void { + /** @var BridgePayment $payment */ $payment = BridgePayment::factory()->create(); /** @var User $user */ $user = User::factory()->create(); @@ -253,9 +256,11 @@ public function testBridgeAdIncomeToUserLedger(): void $paymentDetailsProcessor->addBridgeAdIncomeToUserLedger($payment); - $entries = UserLedgerEntry::all(); - self::assertCount(1, $entries); - self::assertEquals($expectedPaidAmount, $entries->first()->amount); + self::assertDatabaseCount(UserLedgerEntry::class, 1); + self::assertDatabaseHas(UserLedgerEntry::class, [ + 'amount' => $expectedPaidAmount, + 'txid' => $payment->payment_id, + ]); } public function testBridgeAdIncomeToUserLedgerWhenNoUser(): void @@ -331,12 +336,10 @@ private function getPaymentDetailsProcessor(): PaymentDetailsProcessor private function createAdsPayment(int $amount): AdsPayment { - $adsPayment = new AdsPayment(); - $adsPayment->txid = '0002:000017C3:0001'; - $adsPayment->amount = $amount; - $adsPayment->address = '0002-00000007-055A'; - $adsPayment->save(); - - return $adsPayment; + return AdsPayment::factory()->create([ + 'address' => '0002-00000007-055A', + 'amount' => $amount, + 'txid' => '0002:000017C3:0001', + ]); } } diff --git a/tests/app/Services/Supply/OpenRtbBridgeTest.php b/tests/app/Services/Supply/OpenRtbBridgeTest.php index 6b82ca002..c6e175a59 100644 --- a/tests/app/Services/Supply/OpenRtbBridgeTest.php +++ b/tests/app/Services/Supply/OpenRtbBridgeTest.php @@ -395,6 +395,14 @@ public function fetchAndStorePaymentsWhileInvalidResponseProvider(): array [self::validPaymentResponseEntry(remove: 'value')], 'Invalid bridge payments response: missing key value', ], + 'invalid id type' => [ + [self::validPaymentResponseEntry(['id' => 1])], + 'Invalid bridge payments response: id is not a string', + ], + 'invalid id too long' => [ + [self::validPaymentResponseEntry(['id' => '12345678901234567890'])], + 'Invalid bridge payments response: id must have at most 18 characters', + ], 'invalid created_at format' => [ [self::validPaymentResponseEntry(['created_at' => '2023-01-01'])], 'Invalid bridge payments response: created_at is not in ISO8601 format', @@ -635,7 +643,7 @@ private function getFoundBanners(): FoundBanners private static function validPaymentResponseEntry(array $merge = [], string $remove = null): array { $data = array_merge([ - 'id' => 1678953600, + 'id' => '1678953600', 'created_at' => '2023-03-17T16:04:33+00:00', 'updated_at' => '2023-03-17T16:04:33+00:00', 'status' => 'done', From 5103882c64688fe3a5d6903bf4ef77b4e1797cbf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82Podkalicki?= Date: Tue, 21 Mar 2023 14:29:44 +0100 Subject: [PATCH 35/55] fix test --- .../Http/Controllers/SupplyControllerTest.php | 19 +++++++++++++++++++ .../app/Services/Supply/OpenRtbBridgeTest.php | 8 ++++---- 2 files changed, 23 insertions(+), 4 deletions(-) diff --git a/tests/app/Http/Controllers/SupplyControllerTest.php b/tests/app/Http/Controllers/SupplyControllerTest.php index 9ffb806d4..a1981680f 100644 --- a/tests/app/Http/Controllers/SupplyControllerTest.php +++ b/tests/app/Http/Controllers/SupplyControllerTest.php @@ -271,6 +271,25 @@ public function testFind(): void $response->assertJsonPath('data.0.id', '3'); } + public function testFindFailWhileInvalidPlacement(): void + { + $data = [ + 'context' => [ + 'iid' => '0123456789ABCDEF0123456789ABCDEF', + 'url' => 'https://example.com', + 'metamask' => true, + 'uid' => 'good-user', + ], + 'placements' => [ + 'id' => '3', + ], + ]; + + $response = $this->postJson(self::BANNER_FIND_URI, $data); + + $response->assertStatus(Response::HTTP_UNPROCESSABLE_ENTITY); + } + public function testFindOpenRtb(): void { Http::preventStrayRequests(); diff --git a/tests/app/Services/Supply/OpenRtbBridgeTest.php b/tests/app/Services/Supply/OpenRtbBridgeTest.php index c6e175a59..902c7adca 100644 --- a/tests/app/Services/Supply/OpenRtbBridgeTest.php +++ b/tests/app/Services/Supply/OpenRtbBridgeTest.php @@ -273,28 +273,28 @@ public function testFetchAndStorePayments(): void ]); $responseData = [ [ - 'id' => 1678953600, + 'id' => '1678953600', 'created_at' => '2023-03-17T16:04:33+00:00', 'updated_at' => '2023-03-17T16:04:33+00:00', 'status' => 'done', 'value' => 100_000_000_000, ], [ - 'id' => 1678957200, + 'id' => '1678957200', 'created_at' => '2023-03-17T16:04:33+00:00', 'updated_at' => '2023-03-17T16:04:33+00:00', 'status' => 'done', 'value' => 123_400_000_000, ], [ - 'id' => 1678960800, + 'id' => '1678960800', 'created_at' => '2023-03-17T16:04:33+00:00', 'updated_at' => '2023-03-17T16:04:33+00:00', 'status' => 'error', 'value' => null, ], [ - 'id' => 1678964400, + 'id' => '1678964400', 'created_at' => '2023-03-17T16:04:33+00:00', 'updated_at' => '2023-03-17T16:04:33+00:00', 'status' => 'processing', From 0e66c30499b0cc8b2fb16e3d217a7ff946f1b0fd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82Podkalicki?= Date: Wed, 15 Mar 2023 15:35:58 +0100 Subject: [PATCH 36/55] Increase coverage on new code --- app/Models/EventLogsHourlyMeta.php | 4 ++-- app/Models/NetworkCaseLogsHourlyMeta.php | 4 ++-- app/Models/NetworkPayment.php | 4 +++- app/Models/Site.php | 3 +-- .../Http/Controllers/WalletControllerTest.php | 18 +++++++++--------- tests/app/Models/SiteTest.php | 18 ++++++++++++++++++ tests/app/Models/SitesRejectedDomainTest.php | 5 +++++ 7 files changed, 40 insertions(+), 16 deletions(-) diff --git a/app/Models/EventLogsHourlyMeta.php b/app/Models/EventLogsHourlyMeta.php index c74fdbf5e..3312746fe 100644 --- a/app/Models/EventLogsHourlyMeta.php +++ b/app/Models/EventLogsHourlyMeta.php @@ -1,7 +1,7 @@ $id], ['status' => NetworkCaseLogsHourlyMeta::STATUS_INVALID]); + $meta = self::updateOrCreate(['id' => $id], ['status' => self::STATUS_INVALID]); $meta->touch(); return $meta; diff --git a/app/Models/NetworkCaseLogsHourlyMeta.php b/app/Models/NetworkCaseLogsHourlyMeta.php index abfa319e7..1e682dd03 100644 --- a/app/Models/NetworkCaseLogsHourlyMeta.php +++ b/app/Models/NetworkCaseLogsHourlyMeta.php @@ -1,7 +1,7 @@ $id], ['status' => NetworkCaseLogsHourlyMeta::STATUS_INVALID]); + $meta = self::updateOrCreate(['id' => $id], ['status' => self::STATUS_INVALID]); $meta->touch(); return $meta; diff --git a/app/Models/NetworkPayment.php b/app/Models/NetworkPayment.php index e965ecd7a..780051a7f 100644 --- a/app/Models/NetworkPayment.php +++ b/app/Models/NetworkPayment.php @@ -1,7 +1,7 @@ 'required|max:64', @@ -241,7 +240,7 @@ public static function fetchById(int $id): ?self public static function fetchByPublicId(string $publicId): ?self { - return self::where('uuid', hex2bin($publicId))->first(); + return (new self())->where('uuid', hex2bin($publicId))->first(); } public static function fetchSite(int $userId, string $domain): ?Site diff --git a/tests/app/Http/Controllers/WalletControllerTest.php b/tests/app/Http/Controllers/WalletControllerTest.php index ce496f309..5c884f9b4 100644 --- a/tests/app/Http/Controllers/WalletControllerTest.php +++ b/tests/app/Http/Controllers/WalletControllerTest.php @@ -371,6 +371,7 @@ public function withdrawAdsWalletProvider(): array public function testWithdrawBscWallet(): void { + /** @var User $user */ $user = User::factory()->create([ 'email_confirmed_at' => now(), 'admin_confirmed_at' => now(), @@ -380,22 +381,21 @@ public function testWithdrawBscWallet(): void '0xace8d624e8c12c0a16df4a61dee85b0fd3f94ceb' ) ]); - $this->actingAs($user, 'api'); + $this->login($user); $this->generateUserIncome($user->id, 200_000_000_000); $amount = 100_000_000_000; $response = $this->postJson('/api/wallet/withdraw', ['amount' => $amount]); $response->assertStatus(Response::HTTP_NO_CONTENT); - $tokens = Token::all(); - self::assertCount(0, $tokens); + self::assertDatabaseEmpty(Token::class); Mail::assertNothingQueued(); - - $userLedgerEntry = UserLedgerEntry::where(['type' => UserLedgerEntry::TYPE_WITHDRAWAL])->first(); - $this->assertNotNull($userLedgerEntry); - $this->assertEquals(UserLedgerEntry::STATUS_PENDING, $userLedgerEntry->status); - $this->assertEquals(-100_050_000_000, $userLedgerEntry->amount); - Queue::assertPushed(AdsSendOne::class, 1); + self::assertDatabaseHas(UserLedgerEntry::class, [ + 'amount' => -100_050_000_000, + 'status' => UserLedgerEntry::STATUS_PENDING, + 'type' => UserLedgerEntry::TYPE_WITHDRAWAL, + ]); + Queue::assertPushed(AdsSendOne::class, fn(AdsSendOne $job) => $amount === $job->getAmount()); } /** diff --git a/tests/app/Models/SiteTest.php b/tests/app/Models/SiteTest.php index 31d7b043c..ab4d581be 100644 --- a/tests/app/Models/SiteTest.php +++ b/tests/app/Models/SiteTest.php @@ -164,4 +164,22 @@ public function testApprovalProcedureWhileRejectedDomainHasReason(): void self::assertEquals(Site::STATUS_REJECTED, $site->status); self::assertEquals($siteRejectReason->id, $site->reject_reason_id); } + + public function testGetRejectReasonAttribute(): void + { + /** @var SiteRejectReason $siteRejectReason */ + $siteRejectReason = SiteRejectReason::factory()->create(); + /** @var Site $site */ + $site = Site::factory()->create(['reject_reason_id' => $siteRejectReason]); + + self::assertEquals($siteRejectReason->reject_reason, $site->reject_reason); + } + + public function testFetchByPublicId(): void + { + /** @var Site $site */ + $site = Site::factory()->create(); + + self::assertNotNull(Site::fetchByPublicId($site->uuid)); + } } diff --git a/tests/app/Models/SitesRejectedDomainTest.php b/tests/app/Models/SitesRejectedDomainTest.php index 4b60ffee5..d7f83df54 100644 --- a/tests/app/Models/SitesRejectedDomainTest.php +++ b/tests/app/Models/SitesRejectedDomainTest.php @@ -84,4 +84,9 @@ public function testDomainRejectedReasonIdWhileDomainIsNotRejected(): void SitesRejectedDomain::domainRejectedReasonId('example.com'); } + + public function testDomainRejectedReasonIdWhileDomainIsEmpty(): void + { + self::assertNull(SitesRejectedDomain::domainRejectedReasonId('')); + } } From 6a95f711f2ffc4a3645ed92e6bd2882d6098952b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82Podkalicki?= Date: Tue, 28 Mar 2023 11:05:50 +0200 Subject: [PATCH 37/55] Pass topframe in find --- app/Http/Controllers/SupplyController.php | 25 ++- resources/js/supply/find/find.js | 18 +- .../Http/Controllers/SupplyControllerTest.php | 197 +++++++++++++++--- tests/mock/Client/DummyAdSelectClient.php | 3 +- 4 files changed, 200 insertions(+), 43 deletions(-) diff --git a/app/Http/Controllers/SupplyController.php b/app/Http/Controllers/SupplyController.php index 7fc91e65b..3b694392f 100644 --- a/app/Http/Controllers/SupplyController.php +++ b/app/Http/Controllers/SupplyController.php @@ -420,7 +420,7 @@ private function checkDecodedQueryData(array $decodedQueryData): void } } - private function decodeZones(array $decodedQueryData): array + private function extractZones(array $decodedQueryData): array { $zones = $decodedQueryData['placements'] ?? $decodedQueryData['zones'] ?? [];// Key 'zones' is for legacy search if (!$zones) { @@ -472,7 +472,7 @@ private function findBanners( throw new AccessDeniedHttpException('Crawlers are not allowed'); } - $zones = $this->decodeZones($decodedQueryData); + $zones = $this->extractZones($decodedQueryData); if ($userContext->pageRank() <= self::UNACCEPTABLE_PAGE_RANK) { if ($userContext->pageRank() == Aduser::CPA_ONLY_PAGE_RANK) { foreach ($zones as &$zone) { @@ -1091,6 +1091,7 @@ private function getZoneType(array $placement): ?string private static function mapFindInput(array $input): array { + $isPageFrame = false; $context = $input['context']; $mapped = [ 'page' => [ @@ -1106,16 +1107,24 @@ private static function mapFindInput(array $input): array } foreach ($input['placements'] as $placement) { + $options = [ + 'banner_type' => $placement['types'] ?? null, + 'banner_mime' => $placement['mimes'] ?? null, + ]; + if (isset($placement['topframe'])) { + $options['topframe'] = $placement['topframe']; + $isPageFrame |= !$placement['topframe']; + } $mapped['placements'][] = [ 'id' => $placement['id'], 'placementId' => $placement['placementId'], - 'options' => [ - 'banner_type' => $placement['types'] ?? null, - 'banner_mime' => $placement['mimes'] ?? null, - ], + 'options' => $options, ]; } + if ($isPageFrame) { + $mapped['page']['frame'] = true; + } return $mapped; } @@ -1196,6 +1205,10 @@ private static function validatePlacementCommonFields(array $placement): void } } } + + if (array_key_exists('topframe', $placement) && !is_bool($placement['topframe'])) { + throw new UnprocessableEntityHttpException('Field `placements[].topframe` must be a boolean'); + } } private static function validatePlacementFields(array $placement): void diff --git a/resources/js/supply/find/find.js b/resources/js/supply/find/find.js index 44a66100a..595804c40 100644 --- a/resources/js/supply/find/find.js +++ b/resources/js/supply/find/find.js @@ -74,8 +74,8 @@ var encodeZones = function (zone_data) { var insertedElements = []; var logInsertedElement = function(el) { if (insertedElements.length === 0) { - addListener(window, 'beforeunload', function (event) { - var x; + addListener(window, 'beforeunload', function (_event) { + let x; while (x = insertedElements.pop()) { x.parentElement && x.parentElement.removeChild(x); } @@ -664,12 +664,13 @@ var isBannerPop = function (banner) { domReady(function () { aduserPixel(getImpressionId(), function () { getActiveZones(function (zones, params) { - var context = params.shift() - var placements = params.map((p, index) => ({ + const context = params.shift() + const placements = params.map((p, index) => ({ id: index.toString(), placementId: p.zone, + topframe: !context.frame, })); - var data = { + const data = { context: { iid: context.iid, metamask: !!(context.metamask || 0), @@ -677,8 +678,8 @@ domReady(function () { }, placements: placements, }; - var url = serverOrigin + '/supply/find'; - var options = { + const url = serverOrigin + '/supply/find'; + const options = { json: true, method: 'post', post: data, @@ -884,10 +885,9 @@ var fetchBanner = function (banner, context, zone_options) { fetchURL(banner.serveUrl, { binary: true, noCredentials: true - }).then(function (data, xhr) { + }).then(function (data, _xhr) { context.cid = getCid(); context.page.zone = banner.placementId; - const contextParam = encodeZones([context.page]); context.click_url = addUrlParam( banner.clickUrl, { diff --git a/tests/app/Http/Controllers/SupplyControllerTest.php b/tests/app/Http/Controllers/SupplyControllerTest.php index ecac19afd..db434b200 100644 --- a/tests/app/Http/Controllers/SupplyControllerTest.php +++ b/tests/app/Http/Controllers/SupplyControllerTest.php @@ -35,6 +35,7 @@ use Adshares\Adserver\Models\NetworkImpression; use Adshares\Adserver\Models\NetworkVectorsMeta; use Adshares\Adserver\Models\Site; +use Adshares\Adserver\Models\SitesRejectedDomain; use Adshares\Adserver\Models\User; use Adshares\Adserver\Models\Zone; use Adshares\Adserver\Tests\TestCase; @@ -251,10 +252,8 @@ public function testFind(): void $this->instance(AdUser::class, $adUser); /** @var User $user */ $user = User::factory()->create(['api_token' => '1234', 'auto_withdrawal' => 1e11]); - /** @var Site $site */ - $site = Site::factory()->create(['user_id' => $user->id, 'status' => Site::STATUS_ACTIVE]); /** @var Zone $zone */ - $zone = Zone::factory()->create(['site_id' => $site->id]); + $zone = Zone::factory()->create(['site_id' => Site::factory()->create(['user_id' => $user])]); $data = [ 'context' => [ 'iid' => '0123456789ABCDEF0123456789ABCDEF', @@ -293,10 +292,8 @@ function () { /** @var User $user */ $user = User::factory()->create(['api_token' => '1234', 'auto_withdrawal' => 1e11]); - /** @var Site $site */ - $site = Site::factory()->create(['user_id' => $user->id, 'status' => Site::STATUS_ACTIVE]); /** @var Zone $zone */ - $zone = Zone::factory()->create(['site_id' => $site->id]); + $zone = Zone::factory()->create(['site_id' => Site::factory()->create(['user_id' => $user])]); $data = [ 'context' => [ 'iid' => '0123456789ABCDEF0123456789ABCDEF', @@ -317,20 +314,155 @@ function () { $response->assertJsonCount(0, 'data'); } - public function testFindWithoutPlacements(): void + public function testFindFailWhileSiteRejected(): void + { + SitesRejectedDomain::factory()->create(['domain' => 'example.com']); + + /** @var User $user */ + $user = User::factory()->create(['api_token' => '1234', 'auto_withdrawal' => 1e11]); + /** @var Zone $zone */ + $zone = Zone::factory()->create(['site_id' => Site::factory()->create(['user_id' => $user])]); + $data = [ + 'context' => [ + 'iid' => '0123456789ABCDEF0123456789ABCDEF', + 'url' => 'https://example.com', + 'metamask' => true, + 'uid' => 'good-user', + ], + 'placements' => [ + [ + 'id' => '3', + 'placementId' => $zone->uuid, + ], + ], + ]; + + $response = $this->postJson(self::BANNER_FIND_URI, $data); + + $response->assertStatus(Response::HTTP_BAD_REQUEST); + } + + public function testFindFailWhilePlacementInFrame(): void { + Config::updateAdminSettings([Config::ALLOW_ZONE_IN_IFRAME => '0']); + /** @var User $user */ + $user = User::factory()->create(['api_token' => '1234', 'auto_withdrawal' => 1e11]); + /** @var Zone $zone */ + $zone = Zone::factory()->create(['site_id' => Site::factory()->create(['user_id' => $user])]); $data = [ 'context' => [ 'iid' => '0123456789ABCDEF0123456789ABCDEF', 'url' => 'https://example.com', + 'metamask' => true, + 'uid' => 'good-user', + ], + 'placements' => [ + [ + 'id' => '3', + 'placementId' => $zone->uuid, + 'topframe' => false, + ], ], ]; $response = $this->postJson(self::BANNER_FIND_URI, $data); + $response->assertStatus(Response::HTTP_BAD_REQUEST); + } + + /** + * @dataProvider findFailProvider + */ + public function testFindFail(array $data): void + { + $response = $this->postJson(self::BANNER_FIND_URI, $data); + $response->assertStatus(Response::HTTP_UNPROCESSABLE_ENTITY); } + public function findFailProvider(): array + { + return [ + 'without placements' => [ + [ + 'context' => [ + 'iid' => '0123456789ABCDEF0123456789ABCDEF', + 'url' => 'https://example.com', + ], + ] + ], + 'invalid placements[] type' => [ + [ + 'context' => [ + 'iid' => '0123456789ABCDEF0123456789ABCDEF', + 'url' => 'https://example.com', + ], + 'placements' => [ + 'id' => '1', + 'placementId' => '0123456789ABCDEF0123456789ABCDEF', + ], + ] + ], + 'missing placement id' => [ + [ + 'context' => [ + 'iid' => '0123456789ABCDEF0123456789ABCDEF', + 'url' => 'https://example.com', + ], + 'placements' => [ + [ + 'placementId' => '0123456789ABCDEF0123456789ABCDEF', + ] + ], + ] + ], + 'invalid placement id type' => [ + [ + 'context' => [ + 'iid' => '0123456789ABCDEF0123456789ABCDEF', + 'url' => 'https://example.com', + ], + 'placements' => [ + [ + 'id' => 1, + 'placementId' => '0123456789ABCDEF0123456789ABCDEF', + ] + ], + ] + ], + 'invalid placement mimes type' => [ + [ + 'context' => [ + 'iid' => '0123456789ABCDEF0123456789ABCDEF', + 'url' => 'https://example.com', + ], + 'placements' => [ + [ + 'id' => '1', + 'placementId' => '0123456789ABCDEF0123456789ABCDEF', + 'mimes' => 'image/png', + ] + ], + ] + ], + 'invalid placement mimes[] type' => [ + [ + 'context' => [ + 'iid' => '0123456789ABCDEF0123456789ABCDEF', + 'url' => 'https://example.com', + ], + 'placements' => [ + [ + 'id' => '1', + 'placementId' => '0123456789ABCDEF0123456789ABCDEF', + 'mimes' => [true], + ] + ], + ] + ], + ]; + } + public function testFindWithExistingUserWhoIsAdvertiserOnly(): void { $this->mockAdSelect(); @@ -408,9 +540,11 @@ public function findDynamicFailProvider(): array { return [ 'unsupported popup' => [ - self::getDynamicFindData(['placements' => [ - self::getPlacementData(['types' => [Banner::TEXT_TYPE_DIRECT_LINK]]) - ]]), + self::getDynamicFindData([ + 'placements' => [ + self::getPlacementData(['types' => [Banner::TEXT_TYPE_DIRECT_LINK]]) + ] + ]), ], 'missing context.medium' => [ self::getDynamicFindData(['context' => self::getContextData(remove: 'medium')]) @@ -459,6 +593,13 @@ public function findDynamicFailProvider(): array ], ]) ], + 'invalid placement topframe' => [ + self::getDynamicFindData([ + 'placements' => [ + self::getPlacementData(['topframe' => null]) + ], + ]) + ], 'no matching scopes' => [ self::getDynamicFindData([ 'context' => self::getContextData(['medium' => 'metaverse', 'vendor' => 'decentraland']), @@ -969,23 +1110,25 @@ private static function initBeforeLoggingView(): array 'view_url' => 'https://example.com/view', ]); $iid = Utils::base64UrlEncodeWithChecksumFromBinUuidString(hex2bin($impression->impression_id)); - $ctx = Utils::UrlSafeBase64Encode(json_encode( - [ - 'page' => [ - 'iid' => $iid, - 'frame' => 0, - 'width' => 1024, - 'height' => 768, - 'url' => 'https://adshares.net', - 'keywords' => '', - 'metamask' => 0, - 'ref' => '', - 'pop' => 0, - 'zone' => $zone->uuid, - 'options' => '[]', - ], - ] - )); + $ctx = Utils::UrlSafeBase64Encode( + json_encode( + [ + 'page' => [ + 'iid' => $iid, + 'frame' => 0, + 'width' => 1024, + 'height' => 768, + 'url' => 'https://adshares.net', + 'keywords' => '', + 'metamask' => 0, + 'ref' => '', + 'pop' => 0, + 'zone' => $zone->uuid, + 'options' => '[]', + ], + ] + ) + ); $redirectUrl = Utils::urlSafeBase64Encode($banner->view_url); $query = [ 'cid' => '13245679801324567980132456798012', diff --git a/tests/mock/Client/DummyAdSelectClient.php b/tests/mock/Client/DummyAdSelectClient.php index e6604c9ab..ab76c74b1 100644 --- a/tests/mock/Client/DummyAdSelectClient.php +++ b/tests/mock/Client/DummyAdSelectClient.php @@ -56,7 +56,8 @@ private function getBestBanners(array $zones, string $impressionId): array $bannerIds = []; $zoneData = []; foreach ($zones as $zoneInfo) { - $zone = Zone::where('uuid', hex2bin($zoneInfo['zone']))->first(); + $zoneId = $zoneInfo['placementId'] ?? (string)$zoneInfo['zone'];// Key 'zone' is for legacy search + $zone = Zone::where('uuid', hex2bin($zoneId))->first(); if (!$zone) { $bannerIds[] = ''; continue; From 18aaedd556d2ac515068b6b2d6a3031573719bfd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82Podkalicki?= Date: Tue, 28 Mar 2023 15:31:01 +0200 Subject: [PATCH 38/55] change key case --- app/Client/GuzzleAdSelectClient.php | 2 +- app/Services/Supply/OpenRtbBridge.php | 2 +- tests/app/Http/Controllers/SupplyControllerTest.php | 2 +- tests/app/Services/Supply/OpenRtbBridgeTest.php | 2 +- tests/mock/Client/DummyAdSelectClient.php | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/app/Client/GuzzleAdSelectClient.php b/app/Client/GuzzleAdSelectClient.php index 5485cc1d7..55dd854a8 100644 --- a/app/Client/GuzzleAdSelectClient.php +++ b/app/Client/GuzzleAdSelectClient.php @@ -297,7 +297,7 @@ private function fetchInOrderOfAppearance( $campaign = $banner->campaign; $data = [ 'id' => $bannerId, - 'demandId' => $banner->demand_banner_id, + 'demand_id' => $banner->demand_banner_id, 'publisher_id' => $zone->site->user->uuid, 'zone_id' => $zone->uuid, 'pay_from' => $campaign->source_address, diff --git a/app/Services/Supply/OpenRtbBridge.php b/app/Services/Supply/OpenRtbBridge.php index f413d0131..2c498982f 100644 --- a/app/Services/Supply/OpenRtbBridge.php +++ b/app/Services/Supply/OpenRtbBridge.php @@ -66,7 +66,7 @@ public function replaceOpenRtbBanners(FoundBanners $foundBanners, ImpressionCont if (null !== $foundBanner && $accountAddress === $foundBanner['pay_from']) { $openRtbBanners[(string)$index] = [ 'request_id' => (string)$index, - 'creative_id' => $foundBanner['demandId'], + 'creative_id' => $foundBanner['demand_id'], ]; } } diff --git a/tests/app/Http/Controllers/SupplyControllerTest.php b/tests/app/Http/Controllers/SupplyControllerTest.php index a1981680f..fb85672a9 100644 --- a/tests/app/Http/Controllers/SupplyControllerTest.php +++ b/tests/app/Http/Controllers/SupplyControllerTest.php @@ -1207,7 +1207,7 @@ private function initOpenRtb(): array new FoundBanners([ [ 'id' => $networkBanner->uuid, - 'demandId' => $networkBanner->demand_banner_id, + 'demand_id' => $networkBanner->demand_banner_id, 'publisher_id' => '0123456879ABCDEF0123456879ABCDEF', 'zone_id' => $zone->uuid, 'pay_from' => '0001-00000001-8B4E', diff --git a/tests/app/Services/Supply/OpenRtbBridgeTest.php b/tests/app/Services/Supply/OpenRtbBridgeTest.php index 902c7adca..e200ab879 100644 --- a/tests/app/Services/Supply/OpenRtbBridgeTest.php +++ b/tests/app/Services/Supply/OpenRtbBridgeTest.php @@ -602,7 +602,7 @@ private function getFoundBanners(): FoundBanners return new FoundBanners([ [ 'id' => $networkBanner->uuid, - 'demandId' => $networkBanner->demand_banner_id, + 'demand_id' => $networkBanner->demand_banner_id, 'publisher_id' => '0123456879ABCDEF0123456879ABCDEF', 'zone_id' => $zone->uuid, 'pay_from' => '0001-00000001-8B4E', diff --git a/tests/mock/Client/DummyAdSelectClient.php b/tests/mock/Client/DummyAdSelectClient.php index f265ef42a..11e4ce23a 100644 --- a/tests/mock/Client/DummyAdSelectClient.php +++ b/tests/mock/Client/DummyAdSelectClient.php @@ -86,7 +86,7 @@ private function getBestBanners(array $zones, string $impressionId): array $zoneId = $zoneData[$bannerId]['zone_id']; $banners[] = [ 'id' => $bannerId, - 'demandId' => $banner->demand_banner_id, + 'demand_id' => $banner->demand_banner_id, 'publisher_id' => $zoneData[$bannerId]['publisher_id'], 'zone_id' => $zoneId, 'pay_from' => $campaign->source_address, From e0fbdcc03bbdff8a72a1e9928895bb2a10ebbb94 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82Podkalicki?= Date: Tue, 28 Mar 2023 16:33:12 +0200 Subject: [PATCH 39/55] fix zone in frame checking --- app/Http/Controllers/SupplyController.php | 23 +++++++++++++++++------ 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/app/Http/Controllers/SupplyController.php b/app/Http/Controllers/SupplyController.php index 3b694392f..7677d0f25 100644 --- a/app/Http/Controllers/SupplyController.php +++ b/app/Http/Controllers/SupplyController.php @@ -408,7 +408,7 @@ private function checkDecodedQueryData(array $decodedQueryData): void if ($this->isPageBlacklisted($decodedQueryData['page']['url'] ?? '')) { throw new BadRequestHttpException('Site not accepted'); } - if (!config('app.allow_zone_in_iframe') && ($decodedQueryData['page']['frame'] ?? false)) { + if (!config('app.allow_zone_in_iframe') && $this->isAnyZoneInFrame($decodedQueryData)) { throw new BadRequestHttpException('Cannot run in iframe'); } if ( @@ -420,6 +420,22 @@ private function checkDecodedQueryData(array $decodedQueryData): void } } + private function isAnyZoneInFrame(array $decodedQueryData): bool + { + if ($decodedQueryData['page']['frame'] ?? false) { + // legacy code, should be deleted when legacyFind will be removed + return true; + } + if (isset($decodedQueryData['placements'])) { + foreach ($decodedQueryData['placements'] as $placement) { + if (!($placement['options']['topframe'] ?? true)) { + return true; + } + } + } + return false; + } + private function extractZones(array $decodedQueryData): array { $zones = $decodedQueryData['placements'] ?? $decodedQueryData['zones'] ?? [];// Key 'zones' is for legacy search @@ -1091,7 +1107,6 @@ private function getZoneType(array $placement): ?string private static function mapFindInput(array $input): array { - $isPageFrame = false; $context = $input['context']; $mapped = [ 'page' => [ @@ -1113,7 +1128,6 @@ private static function mapFindInput(array $input): array ]; if (isset($placement['topframe'])) { $options['topframe'] = $placement['topframe']; - $isPageFrame |= !$placement['topframe']; } $mapped['placements'][] = [ 'id' => $placement['id'], @@ -1122,9 +1136,6 @@ private static function mapFindInput(array $input): array ]; } - if ($isPageFrame) { - $mapped['page']['frame'] = true; - } return $mapped; } From 00c0825a724fdc6456cda966eac6f475a970a957 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82Podkalicki?= Date: Wed, 29 Mar 2023 11:45:15 +0200 Subject: [PATCH 40/55] Pass zone options to bridge --- app/Http/Controllers/SupplyController.php | 2 +- app/Services/Supply/OpenRtbBridge.php | 37 +++++++++++++--- database/factories/NetworkBannerFactory.php | 3 +- .../app/Services/Supply/OpenRtbBridgeTest.php | 44 +++++++++++++++++++ 4 files changed, 78 insertions(+), 8 deletions(-) diff --git a/app/Http/Controllers/SupplyController.php b/app/Http/Controllers/SupplyController.php index 721ac05ff..cf4c8357c 100644 --- a/app/Http/Controllers/SupplyController.php +++ b/app/Http/Controllers/SupplyController.php @@ -503,7 +503,7 @@ private function findBanners( $context = Utils::mergeImpressionContextAndUserContext($impressionContext, $userContext); $foundBanners = $bannerFinder->findBanners($zones, $context, $impressionId); if (OpenRtbBridge::isActive()) { - $foundBanners = (new OpenRtbBridge())->replaceOpenRtbBanners($foundBanners, $context); + $foundBanners = (new OpenRtbBridge())->replaceOpenRtbBanners($foundBanners, $context, $zones); } if ($foundBanners->exists(fn($key, $element) => null !== $element)) { diff --git a/app/Services/Supply/OpenRtbBridge.php b/app/Services/Supply/OpenRtbBridge.php index 2c498982f..b25a3c585 100644 --- a/app/Services/Supply/OpenRtbBridge.php +++ b/app/Services/Supply/OpenRtbBridge.php @@ -58,16 +58,22 @@ public static function isActive(): bool && null !== config('app.open_rtb_bridge_url'); } - public function replaceOpenRtbBanners(FoundBanners $foundBanners, ImpressionContext $context): FoundBanners - { + public function replaceOpenRtbBanners( + FoundBanners $foundBanners, + ImpressionContext $context, + array $zones = [], + ): FoundBanners { $accountAddress = config('app.open_rtb_bridge_account_address'); $openRtbBanners = []; foreach ($foundBanners as $index => $foundBanner) { if (null !== $foundBanner && $accountAddress === $foundBanner['pay_from']) { - $openRtbBanners[(string)$index] = [ - 'request_id' => (string)$index, - 'creative_id' => $foundBanner['demand_id'], - ]; + $openRtbBanners[(string)$index] = array_merge( + $this->extractZoneOptions($zones, $foundBanner['zone_id']), + [ + 'request_id' => (string)$index, + 'creative_id' => $foundBanner['demand_id'], + ] + ); } } if (empty($openRtbBanners)) { @@ -296,6 +302,25 @@ private function isOpenRtbAuctionResponseValid(mixed $content, array $openBtbBan return true; } + private function extractZoneOptions(array $zones, string $zoneId): array + { + foreach ($zones as $zone) { + $placementId = $zone['placementId'] ?? (string)$zone['zone'];// Key 'zone' is for legacy search + if ($placementId === $zoneId) { + $zoneOptions = $zone['options'] ?? []; + $options = []; + if (isset($zoneOptions['banner_mime'])) { + $options['mimes'] = $zoneOptions['banner_mime']; + } + if (isset($zoneOptions['topframe'])) { + $options['topframe'] = $zoneOptions['topframe']; + } + return $options; + } + } + return []; + } + public function getEventRedirectUrl(string $url): ?string { $redirectUrl = null; diff --git a/database/factories/NetworkBannerFactory.php b/database/factories/NetworkBannerFactory.php index 3ab5a6e25..953ec0596 100644 --- a/database/factories/NetworkBannerFactory.php +++ b/database/factories/NetworkBannerFactory.php @@ -1,7 +1,7 @@ $this->faker->uuid, + 'demand_banner_id' => $this->faker->uuid, 'network_campaign_id' => $this->faker->randomDigit(), 'serve_url' => $this->faker->url, 'view_url' => $this->faker->url, diff --git a/tests/app/Services/Supply/OpenRtbBridgeTest.php b/tests/app/Services/Supply/OpenRtbBridgeTest.php index e200ab879..ce9ebb5bd 100644 --- a/tests/app/Services/Supply/OpenRtbBridgeTest.php +++ b/tests/app/Services/Supply/OpenRtbBridgeTest.php @@ -55,6 +55,7 @@ use Closure; use DateTime; use Illuminate\Http\Client\ConnectionException; +use Illuminate\Http\Client\Request; use Illuminate\Support\Collection; use Illuminate\Support\Facades\Http; use Illuminate\Support\Facades\Log; @@ -102,6 +103,49 @@ public function testReplaceOpenRtbBanners(): void Http::assertSentCount(1); } + public function testReplaceOpenRtbBannersWithOptions(): void + { + Http::preventStrayRequests(); + Http::fake(function (Request $request) { + $request = json_decode($request->body(), true)['requests'][0]; + foreach (['mimes', 'topframe'] as $key) { + self::assertArrayHasKey($key, $request); + } + self::assertTrue($request['topframe']); + self::assertEquals(['image/jpeg', 'video/mp4'], $request['mimes']); + return Http::response([ + [ + 'ext_id' => '1', + 'request_id' => '0', + 'serve_url' => 'https://example.com/serve/1', + ] + ]); + }); + $initiallyFoundBanners = $this->getFoundBanners(); + $context = new ImpressionContext([], [], []); + $zones = [ + [ + 'id' => '0', + 'placementId' => $initiallyFoundBanners->first()['zone_id'], + 'options' => [ + 'banner_mime' => ['image/jpeg', 'video/mp4'], + 'banner_type' => ['image', 'video'], + 'cpa_only' => true, + 'topframe' => true, + ], + ] + ]; + + $foundBanners = (new OpenRtbBridge())->replaceOpenRtbBanners($initiallyFoundBanners, $context, $zones); + + self::assertCount(1, $foundBanners); + $foundBanner = $foundBanners->first(); + self::assertEquals('3', $foundBanner['request_id']); + self::assertStringContainsString('extid=1', $foundBanner['click_url']); + self::assertStringContainsString('extid=1', $foundBanner['view_url']); + Http::assertSentCount(1); + } + public function testReplaceOpenRtbBannersWhileEmptyResponse(): void { Http::preventStrayRequests(); From 0e84178c4a1083188b3294e7ab77d0c02b60e994 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82Podkalicki?= Date: Wed, 29 Mar 2023 15:29:16 +0200 Subject: [PATCH 41/55] rename bridge --- app/Client/GuzzleDemandClient.php | 4 +- app/Console/Commands/AdsFetchHosts.php | 6 +- .../Commands/SupplyProcessPayments.php | 10 +-- .../Manager/ServerConfigurationController.php | 4 +- app/Http/Controllers/SupplyController.php | 10 +-- app/Models/Config.php | 8 +- .../{OpenRtbBridge.php => DspBridge.php} | 46 +++++------ ...geRegistrar.php => DspBridgeRegistrar.php} | 20 ++--- .../Console/Commands/AdsFetchHostsTest.php | 12 +-- .../Commands/SupplyProcessPaymentsTest.php | 8 +- .../ServerConfigurationControllerTest.php | 4 +- .../Http/Controllers/SupplyControllerTest.php | 14 ++-- ...rarTest.php => DspBridgeRegistrarTest.php} | 62 +++++--------- ...penRtbBridgeTest.php => DspBridgeTest.php} | 80 +++++++++---------- 14 files changed, 135 insertions(+), 153 deletions(-) rename app/Services/Supply/{OpenRtbBridge.php => DspBridge.php} (88%) rename app/Services/Supply/{OpenRtbBridgeRegistrar.php => DspBridgeRegistrar.php} (73%) rename tests/app/Services/Supply/{OpenRtbBridgeRegistrarTest.php => DspBridgeRegistrarTest.php} (68%) rename tests/app/Services/Supply/{OpenRtbBridgeTest.php => DspBridgeTest.php} (89%) diff --git a/app/Client/GuzzleDemandClient.php b/app/Client/GuzzleDemandClient.php index 1cf332b6c..ac5e4a5a6 100644 --- a/app/Client/GuzzleDemandClient.php +++ b/app/Client/GuzzleDemandClient.php @@ -259,7 +259,7 @@ private function processData( 'updated_at' => DateTime::createFromFormat(DateTimeInterface::ATOM, $data['updated_at']), ]; - $isOpenRtbProvider = $sourceAddress === config('app.open_rtb_bridge_account_address'); + $isDspBridgeProvider = $sourceAddress === config('app.dsp_bridge_account_address'); $classifiersRequired = $this->classifierRepository->fetchRequiredClassifiersNames(); $banners = []; $bannersInput = $data['creatives'] ?? $data['banners'];// legacy fallback, field 'banners' is deprecated @@ -272,7 +272,7 @@ private function processData( unset($banner['id']); } - $banner['classification'] = $isOpenRtbProvider + $banner['classification'] = $isDspBridgeProvider ? self::flattenClassification($banner['classification'] ?? []) : $this->validateAndMapClassification($banner); if ($this->missingRequiredClassifier($classifiersRequired, $banner['classification'])) { diff --git a/app/Console/Commands/AdsFetchHosts.php b/app/Console/Commands/AdsFetchHosts.php index bfa153246..ab1b9c3fb 100644 --- a/app/Console/Commands/AdsFetchHosts.php +++ b/app/Console/Commands/AdsFetchHosts.php @@ -31,7 +31,7 @@ use Adshares\Adserver\Events\ServerEvent; use Adshares\Adserver\Http\Response\InfoResponse; use Adshares\Adserver\Models\NetworkHost; -use Adshares\Adserver\Services\Supply\OpenRtbBridgeRegistrar; +use Adshares\Adserver\Services\Supply\DspBridgeRegistrar; use Adshares\Adserver\ViewModel\ServerEventType; use Adshares\Common\Exception\RuntimeException; use Adshares\Config\AppMode; @@ -60,7 +60,7 @@ public function __construct( Locker $locker, private readonly AdsClient $adsClient, private readonly DemandClient $client, - private readonly OpenRtbBridgeRegistrar $openRtbProviderRegistrar, + private readonly DspBridgeRegistrar $dspBridgeRegistrar, ) { parent::__construct($locker); } @@ -89,7 +89,7 @@ public function handle(): int } $progressBar->finish(); $this->newLine(); - if ($this->openRtbProviderRegistrar->registerAsNetworkHost()) { + if ($this->dspBridgeRegistrar->registerAsNetworkHost()) { $found++; } diff --git a/app/Console/Commands/SupplyProcessPayments.php b/app/Console/Commands/SupplyProcessPayments.php index 11dfce251..3c3919e42 100644 --- a/app/Console/Commands/SupplyProcessPayments.php +++ b/app/Console/Commands/SupplyProcessPayments.php @@ -32,7 +32,7 @@ use Adshares\Adserver\Services\Dto\ProcessedPaymentsMetaData; use Adshares\Adserver\Services\LicenseFeeSender; use Adshares\Adserver\Services\PaymentDetailsProcessor; -use Adshares\Adserver\Services\Supply\OpenRtbBridge; +use Adshares\Adserver\Services\Supply\DspBridge; use Adshares\Adserver\ViewModel\ServerEventType; use Adshares\Common\Infrastructure\Service\LicenseReader; use Adshares\Supply\Application\Service\DemandClient; @@ -101,11 +101,11 @@ public function handle(): void self::COLUMN_ADS_PAYMENT_ID, ); - if (OpenRtbBridge::isActive()) { - $openRtbBridge = new OpenRtbBridge(); - $openRtbBridge->fetchAndStorePayments(); + if (DspBridge::isActive()) { + $bridge = new DspBridge(); + $bridge->fetchAndStorePayments(); - $processedBridgePaymentMetaData = $openRtbBridge->processPayments( + $processedBridgePaymentMetaData = $bridge->processPayments( $this->demandClient, $this->paymentDetailsProcessor, (int)$this->option('chunkSize'), diff --git a/app/Http/Controllers/Manager/ServerConfigurationController.php b/app/Http/Controllers/Manager/ServerConfigurationController.php index fc5d5864e..c6c8a811e 100644 --- a/app/Http/Controllers/Manager/ServerConfigurationController.php +++ b/app/Http/Controllers/Manager/ServerConfigurationController.php @@ -102,6 +102,8 @@ class ServerConfigurationController extends Controller Config::CRM_MAIL_ADDRESS_ON_USER_REGISTERED => 'nullable|email', Config::CURRENCY => 'nullable|appCurrency', Config::DEFAULT_USER_ROLES => 'nullable|notEmpty|list:userRole', + Config::DSP_BRIDGE_ACCOUNT_ADDRESS => 'nullable|accountId', + Config::DSP_BRIDGE_URL => 'nullable|url', Config::EMAIL_VERIFICATION_REQUIRED => 'nullable|boolean', Config::EXCHANGE_API_KEY => 'nullable', Config::EXCHANGE_API_SECRET => 'nullable', @@ -147,8 +149,6 @@ class ServerConfigurationController extends Controller Config::NOW_PAYMENTS_IPN_SECRET => 'nullable', Config::NOW_PAYMENTS_MAX_AMOUNT => 'nullable|integer|min:0', Config::NOW_PAYMENTS_MIN_AMOUNT => 'nullable|integer|min:0', - Config::OPEN_RTB_BRIDGE_ACCOUNT_ADDRESS => 'nullable|accountId', - Config::OPEN_RTB_BRIDGE_URL => 'nullable|url', Config::OPERATOR_RX_FEE => 'nullable|commission', Config::OPERATOR_TX_FEE => 'nullable|commission', Config::PUBLISHER_APPLY_FORM_URL => 'nullable|url', diff --git a/app/Http/Controllers/SupplyController.php b/app/Http/Controllers/SupplyController.php index cf4c8357c..890e600cb 100644 --- a/app/Http/Controllers/SupplyController.php +++ b/app/Http/Controllers/SupplyController.php @@ -35,7 +35,7 @@ use Adshares\Adserver\Models\User; use Adshares\Adserver\Models\Zone; use Adshares\Adserver\Rules\PayoutAddressRule; -use Adshares\Adserver\Services\Supply\OpenRtbBridge; +use Adshares\Adserver\Services\Supply\DspBridge; use Adshares\Adserver\Utilities\AdsAuthenticator; use Adshares\Adserver\Utilities\AdsUtils; use Adshares\Adserver\Utilities\CssUtils; @@ -502,8 +502,8 @@ private function findBanners( $context = Utils::mergeImpressionContextAndUserContext($impressionContext, $userContext); $foundBanners = $bannerFinder->findBanners($zones, $context, $impressionId); - if (OpenRtbBridge::isActive()) { - $foundBanners = (new OpenRtbBridge())->replaceOpenRtbBanners($foundBanners, $context, $zones); + if (DspBridge::isActive()) { + $foundBanners = (new DspBridge())->replaceBridgeBanners($foundBanners, $context, $zones); } if ($foundBanners->exists(fn($key, $element) => null !== $element)) { @@ -676,7 +676,7 @@ public function logNetworkClick(Request $request, string $bannerId): BaseRespons } if ($isDspBridge) { - $redirectUrl = (new OpenRtbBridge())->getEventRedirectUrl($url) + $redirectUrl = (new DspBridge())->getEventRedirectUrl($url) ?: route('why', ['bid' => $bannerId, 'cid' => $caseId]); $response = new RedirectResponse($redirectUrl); } else { @@ -812,7 +812,7 @@ public function logNetworkView(Request $request, string $bannerId): BaseResponse } if ($isDspBridge) { - $redirectUrl = (new OpenRtbBridge())->getEventRedirectUrl($url); + $redirectUrl = (new DspBridge())->getEventRedirectUrl($url); $response = null !== $redirectUrl ? new RedirectResponse($redirectUrl) : new BaseResponse(status: BaseResponse::HTTP_NO_CONTENT); diff --git a/app/Models/Config.php b/app/Models/Config.php index f7bb7b316..dcee55373 100644 --- a/app/Models/Config.php +++ b/app/Models/Config.php @@ -104,6 +104,8 @@ class Config extends Model public const CURRENCY = 'currency'; public const DEFAULT_USER_ROLES = 'default-user-roles'; public const DISPLAY_CURRENCY = 'display-currency'; + public const DSP_BRIDGE_ACCOUNT_ADDRESS = 'dsp-bridge-account-address'; + public const DSP_BRIDGE_URL = 'dsp-bridge-url'; public const EMAIL_VERIFICATION_REQUIRED = 'email-verification-required'; public const EXCHANGE_API_KEY = 'exchange-api-key'; public const EXCHANGE_API_SECRET = 'exchange-api-secret'; @@ -150,8 +152,6 @@ class Config extends Model public const NOW_PAYMENTS_IPN_SECRET = 'now-payments-ipn-secret'; public const NOW_PAYMENTS_MAX_AMOUNT = 'now-payments-max-amount'; public const NOW_PAYMENTS_MIN_AMOUNT = 'now-payments-min-amount'; - public const OPEN_RTB_BRIDGE_ACCOUNT_ADDRESS = 'open-rtb-bridge-account-address'; - public const OPEN_RTB_BRIDGE_URL = 'open-rtb-bridge-url'; public const OPERATOR_RX_FEE = 'payment-rx-fee'; public const OPERATOR_TX_FEE = 'payment-tx-fee'; public const OPERATOR_WALLET_EMAIL_LAST_TIME = 'operator-wallet-transfer-email-time'; @@ -531,8 +531,8 @@ private static function getDefaultAdminSettings(array $fetched = []): array self::NOW_PAYMENTS_IPN_SECRET => '', self::NOW_PAYMENTS_MAX_AMOUNT => 1000, self::NOW_PAYMENTS_MIN_AMOUNT => 25, - self::OPEN_RTB_BRIDGE_ACCOUNT_ADDRESS => null, - self::OPEN_RTB_BRIDGE_URL => null, + self::DSP_BRIDGE_ACCOUNT_ADDRESS => null, + self::DSP_BRIDGE_URL => null, self::OPERATOR_RX_FEE => 0.01, self::OPERATOR_TX_FEE => 0.01, self::PUBLISHER_APPLY_FORM_URL => null, diff --git a/app/Services/Supply/OpenRtbBridge.php b/app/Services/Supply/DspBridge.php similarity index 88% rename from app/Services/Supply/OpenRtbBridge.php rename to app/Services/Supply/DspBridge.php index b25a3c585..ce93f9538 100644 --- a/app/Services/Supply/OpenRtbBridge.php +++ b/app/Services/Supply/DspBridge.php @@ -41,7 +41,7 @@ use Symfony\Component\HttpFoundation\Response as BaseResponse; use Throwable; -class OpenRtbBridge +class DspBridge { private const PAYMENT_REPORT_READY_STATUS = 'done'; private const PAYMENTS_PATH = '/payment-reports'; @@ -54,20 +54,20 @@ class OpenRtbBridge public static function isActive(): bool { - return null !== config('app.open_rtb_bridge_account_address') - && null !== config('app.open_rtb_bridge_url'); + return null !== config('app.dsp_bridge_account_address') + && null !== config('app.dsp_bridge_url'); } - public function replaceOpenRtbBanners( + public function replaceBridgeBanners( FoundBanners $foundBanners, ImpressionContext $context, array $zones = [], ): FoundBanners { - $accountAddress = config('app.open_rtb_bridge_account_address'); - $openRtbBanners = []; + $accountAddress = config('app.dsp_bridge_account_address'); + $bridgeBanners = []; foreach ($foundBanners as $index => $foundBanner) { if (null !== $foundBanner && $accountAddress === $foundBanner['pay_from']) { - $openRtbBanners[(string)$index] = array_merge( + $bridgeBanners[(string)$index] = array_merge( $this->extractZoneOptions($zones, $foundBanner['zone_id']), [ 'request_id' => (string)$index, @@ -76,20 +76,20 @@ public function replaceOpenRtbBanners( ); } } - if (empty($openRtbBanners)) { + if (empty($bridgeBanners)) { return $foundBanners; } try { $response = Http::post( - config('app.open_rtb_bridge_url') . self::SERVE_PATH, + config('app.dsp_bridge_url') . self::SERVE_PATH, [ 'context' => $context->toArray(), - 'requests' => $openRtbBanners, + 'requests' => $bridgeBanners, ], ); if ( BaseResponse::HTTP_OK === $response->status() - && $this->isOpenRtbAuctionResponseValid($content = $response->json(), $openRtbBanners) + && $this->isAuctionResponseValid($content = $response->json(), $bridgeBanners) ) { foreach ($content as $entry) { $externalId = $entry['ext_id']; @@ -103,13 +103,13 @@ public function replaceOpenRtbBanners( } $foundBanner['serve_url'] = $entry['serve_url']; $foundBanners->set((int)$entry['request_id'], $foundBanner); - unset($openRtbBanners[$entry['request_id']]); + unset($bridgeBanners[$entry['request_id']]); } } } catch (HttpClientException $exception) { - Log::error(sprintf('Replacing OpenRtb banner failed: %s', $exception->getMessage())); + Log::error(sprintf('Replacing bridge banner failed: %s', $exception->getMessage())); } - foreach ($openRtbBanners as $index => $serveUrl) { + foreach ($bridgeBanners as $index => $serveUrl) { $foundBanners->set($index, null); } return $foundBanners; @@ -118,7 +118,7 @@ public function replaceOpenRtbBanners( public function fetchAndStorePayments(): void { try { - $response = Http::get(config('app.open_rtb_bridge_url') . self::PAYMENTS_PATH); + $response = Http::get(config('app.dsp_bridge_url') . self::PAYMENTS_PATH); if ( BaseResponse::HTTP_OK === $response->status() && $this->isPaymentResponseValid($content = $response->json()) @@ -127,7 +127,7 @@ public function fetchAndStorePayments(): void return; } $paymentIds = array_map(fn(array $entry) => $entry['id'], $content); - $accountAddress = config('app.open_rtb_bridge_account_address'); + $accountAddress = config('app.dsp_bridge_account_address'); $bridgePayments = BridgePayment::fetchByAddressAndPaymentIds($accountAddress, $paymentIds) ->keyBy('payment_id'); foreach ($content as $entry) { @@ -268,15 +268,15 @@ private function isPaymentResponseValid(mixed $content): bool return true; } - private function isOpenRtbAuctionResponseValid(mixed $content, array $openBtbBanners): bool + private function isAuctionResponseValid(mixed $content, array $bridgeBanners): bool { if (!is_array($content)) { - Log::error('Invalid OpenRTB response: body is not an array'); + Log::error('Invalid DSP bridge response: body is not an array'); return false; } foreach ($content as $entry) { if (!is_array($entry)) { - Log::error('Invalid OpenRTB response: entry is not an array'); + Log::error('Invalid DSP bridge response: entry is not an array'); return false; } $fields = [ @@ -286,16 +286,16 @@ private function isOpenRtbAuctionResponseValid(mixed $content, array $openBtbBan ]; foreach ($fields as $field) { if (!isset($entry[$field])) { - Log::error(sprintf('Invalid OpenRTB response: missing key %s', $field)); + Log::error(sprintf('Invalid DSP bridge response: missing key %s', $field)); return false; } if (!is_string($entry[$field])) { - Log::error(sprintf('Invalid OpenRTB response: %s is not a string', $field)); + Log::error(sprintf('Invalid DSP bridge response: %s is not a string', $field)); return false; } } - if (!array_key_exists($entry['request_id'], $openBtbBanners)) { - Log::error(sprintf('Invalid OpenRTB response: request %s is not known', $entry['request_id'])); + if (!array_key_exists($entry['request_id'], $bridgeBanners)) { + Log::error(sprintf('Invalid DSP bridge response: request %s is not known', $entry['request_id'])); return false; } } diff --git a/app/Services/Supply/OpenRtbBridgeRegistrar.php b/app/Services/Supply/DspBridgeRegistrar.php similarity index 73% rename from app/Services/Supply/OpenRtbBridgeRegistrar.php rename to app/Services/Supply/DspBridgeRegistrar.php index a5c67ff82..57a90b5de 100644 --- a/app/Services/Supply/OpenRtbBridgeRegistrar.php +++ b/app/Services/Supply/DspBridgeRegistrar.php @@ -32,10 +32,10 @@ use DateTimeImmutable; use Illuminate\Support\Facades\Log; -class OpenRtbBridgeRegistrar +class DspBridgeRegistrar { private const INFO_JSON_PATH = '/info.json'; - private const OPEN_RTB_MODULE_NAME = 'openrtb'; + private const DSP_BRIDGE_MODULE_NAME = 'dsp-bridge'; public function __construct(private readonly DemandClient $demandClient) { @@ -44,38 +44,38 @@ public function __construct(private readonly DemandClient $demandClient) public function registerAsNetworkHost(): bool { if ( - null === ($accountAddress = config('app.open_rtb_bridge_account_address')) - || null === ($url = config('app.open_rtb_bridge_url')) + null === ($accountAddress = config('app.dsp_bridge_account_address')) + || null === ($url = config('app.dsp_bridge_url')) ) { return false; } $url = $url . self::INFO_JSON_PATH; if (!AccountId::isValid($accountAddress, true)) { - Log::error('OpenRTB provider registration failed: configured account address is not valid'); + Log::error('DSP bridge provider registration failed: configured account address is not valid'); return false; } try { $infoUrl = new Url($url); } catch (RuntimeException $exception) { - Log::error(sprintf('OpenRTB provider registration failed: %s', $exception->getMessage())); + Log::error(sprintf('DSP bridge provider registration failed: %s', $exception->getMessage())); return false; } try { $info = $this->demandClient->fetchInfo($infoUrl); } catch (UnexpectedClientResponseException $exception) { - Log::error(sprintf('OpenRTB provider registration failed: %s', $exception->getMessage())); + Log::error(sprintf('DSP bridge provider registration failed: %s', $exception->getMessage())); return false; } - if ($info->getModule() !== self::OPEN_RTB_MODULE_NAME) { + if ($info->getModule() !== self::DSP_BRIDGE_MODULE_NAME) { Log::error( - sprintf('OpenRTB provider registration failed: Info for invalid module: %s', $info->getModule()) + sprintf('DSP bridge provider registration failed: Info for invalid module: %s', $info->getModule()) ); return false; } if ($info->getAdsAddress() !== $accountAddress) { - Log::error('OpenRTB provider registration failed: Info address does not match'); + Log::error('DSP bridge provider registration failed: Info address does not match'); return false; } $host = NetworkHost::registerHost($accountAddress, $url, $info, new DateTimeImmutable()); diff --git a/tests/app/Console/Commands/AdsFetchHostsTest.php b/tests/app/Console/Commands/AdsFetchHostsTest.php index 184c378c9..e978b9da0 100644 --- a/tests/app/Console/Commands/AdsFetchHostsTest.php +++ b/tests/app/Console/Commands/AdsFetchHostsTest.php @@ -29,7 +29,7 @@ use Adshares\Adserver\Console\Locker; use Adshares\Adserver\Models\Config; use Adshares\Adserver\Models\NetworkHost; -use Adshares\Adserver\Services\Supply\OpenRtbBridgeRegistrar; +use Adshares\Adserver\Services\Supply\DspBridgeRegistrar; use Adshares\Adserver\Tests\Console\ConsoleTestCase; use Adshares\Adserver\ViewModel\ServerEventType; use Adshares\Config\AppMode; @@ -242,7 +242,7 @@ public function testBroadcastError(): void self::assertServerEventDispatched(ServerEventType::HostBroadcastProcessed); } - public function testRegisterOpenRtbProvider(): void + public function testRegisterDspBridge(): void { $this->app->bind( AdsClient::class, @@ -254,9 +254,9 @@ function () { } ); $this->app->bind( - OpenRtbBridgeRegistrar::class, + DspBridgeRegistrar::class, function () { - $mock = self::createMock(OpenRtbBridgeRegistrar::class); + $mock = self::createMock(DspBridgeRegistrar::class); $mock->method('registerAsNetworkHost') ->will($this->returnCallback(function () { NetworkHost::factory()->create(); @@ -266,8 +266,8 @@ function () { } ); Config::updateAdminSettings([ - Config::OPEN_RTB_BRIDGE_ACCOUNT_ADDRESS => '0001-00000001-8B4E', - Config::OPEN_RTB_BRIDGE_URL => 'https://example.com', + Config::DSP_BRIDGE_ACCOUNT_ADDRESS => '0001-00000001-8B4E', + Config::DSP_BRIDGE_URL => 'https://example.com', ]); self::artisan(self::COMMAND_SIGNATURE)->assertExitCode(0); diff --git a/tests/app/Console/Commands/SupplyProcessPaymentsTest.php b/tests/app/Console/Commands/SupplyProcessPaymentsTest.php index 27f961bd5..c588e412e 100644 --- a/tests/app/Console/Commands/SupplyProcessPaymentsTest.php +++ b/tests/app/Console/Commands/SupplyProcessPaymentsTest.php @@ -170,7 +170,7 @@ public function testAdsProcessEventPayment(): void public function testHandleBridgePayment(): void { - $this->initOpenRtbConfiguration(); + $this->initDspBridgeConfiguration(); $networkHost = self::registerHost(); /** @var BridgePayment $bridgePayment */ $bridgePayment = BridgePayment::factory()->create(['address' => $networkHost->address]); @@ -404,11 +404,11 @@ private static function assertAdPaymentProcessedEventDispatched(int $count = 0): ]); } - private static function initOpenRtbConfiguration(): void + private static function initDspBridgeConfiguration(): void { Config::updateAdminSettings([ - Config::OPEN_RTB_BRIDGE_ACCOUNT_ADDRESS => '0001-00000001-8B4E', - Config::OPEN_RTB_BRIDGE_URL => 'https://example.com', + Config::DSP_BRIDGE_ACCOUNT_ADDRESS => '0001-00000001-8B4E', + Config::DSP_BRIDGE_URL => 'https://example.com', ]); DatabaseConfigReader::overwriteAdministrationConfig(); } diff --git a/tests/app/Http/Controllers/Manager/ServerConfigurationControllerTest.php b/tests/app/Http/Controllers/Manager/ServerConfigurationControllerTest.php index a74669eaa..cd462afd6 100644 --- a/tests/app/Http/Controllers/Manager/ServerConfigurationControllerTest.php +++ b/tests/app/Http/Controllers/Manager/ServerConfigurationControllerTest.php @@ -234,8 +234,8 @@ public function storeDataProvider(): array 'INVENTORY_FAILED_CONNECTION_LIMIT' => [Config::INVENTORY_FAILED_CONNECTION_LIMIT, '8'], 'LANDING_URL' => [Config::LANDING_URL, 'https://example.com'], 'MAX_INVALID_LOGIN_ATTEMPTS' => [Config::MAX_INVALID_LOGIN_ATTEMPTS, '8'], - 'OPEN_RTB_BRIDGE_ACCOUNT_ADDRESS' => [Config::OPEN_RTB_BRIDGE_ACCOUNT_ADDRESS, '0001-00000003-AB0C'], - 'OPEN_RTB_BRIDGE_URL' => [Config::OPEN_RTB_BRIDGE_URL, 'http://localhost:8015'], + 'DSP_BRIDGE_ACCOUNT_ADDRESS' => [Config::DSP_BRIDGE_ACCOUNT_ADDRESS, '0001-00000003-AB0C'], + 'DSP_BRIDGE_URL' => [Config::DSP_BRIDGE_URL, 'http://localhost:8015'], 'REFERRAL_REFUND_COMMISSION' => [Config::REFERRAL_REFUND_COMMISSION, '0'], 'REFERRAL_REFUND_ENABLED' => [Config::REFERRAL_REFUND_ENABLED, '1'], 'SUPPORT_EMAIL' => [Config::SUPPORT_EMAIL, 'sup@example.com'], diff --git a/tests/app/Http/Controllers/SupplyControllerTest.php b/tests/app/Http/Controllers/SupplyControllerTest.php index c0f543c56..9bbcd0881 100644 --- a/tests/app/Http/Controllers/SupplyControllerTest.php +++ b/tests/app/Http/Controllers/SupplyControllerTest.php @@ -289,7 +289,7 @@ public function testFindFailWhileInvalidPlacement(): void $response->assertStatus(Response::HTTP_UNPROCESSABLE_ENTITY); } - public function testFindOpenRtb(): void + public function testFindDspBridge(): void { Http::preventStrayRequests(); Http::fake([ @@ -299,7 +299,7 @@ public function testFindOpenRtb(): void 'serve_url' => 'https://example.com/serve/1', ]]), ]); - $data = $this->initOpenRtb(); + $data = $this->initDspBridge(); $response = $this->postJson(self::BANNER_FIND_URI, $data); @@ -310,11 +310,11 @@ public function testFindOpenRtb(): void Http::assertSentCount(1); } - public function testFindOpenRtbWhileEmptyResponse(): void + public function testFindDspBridgeWhileEmptyResponse(): void { Http::preventStrayRequests(); Http::fake(['example.com/serve' => Http::response([])]); - $data = $this->initOpenRtb(); + $data = $this->initDspBridge(); $response = $this->postJson(self::BANNER_FIND_URI, $data); @@ -1322,11 +1322,11 @@ private function initAdUser(): void $this->instance(AdUser::class, $adUser); } - private function initOpenRtb(): array + private function initDspBridge(): array { Config::updateAdminSettings([ - Config::OPEN_RTB_BRIDGE_ACCOUNT_ADDRESS => '0001-00000001-8B4E', - Config::OPEN_RTB_BRIDGE_URL => 'https://example.com', + Config::DSP_BRIDGE_ACCOUNT_ADDRESS => '0001-00000001-8B4E', + Config::DSP_BRIDGE_URL => 'https://example.com', ]); NetworkHost::factory()->create([ 'address' => '0001-00000001-8B4E', diff --git a/tests/app/Services/Supply/OpenRtbBridgeRegistrarTest.php b/tests/app/Services/Supply/DspBridgeRegistrarTest.php similarity index 68% rename from tests/app/Services/Supply/OpenRtbBridgeRegistrarTest.php rename to tests/app/Services/Supply/DspBridgeRegistrarTest.php index 403976d97..7b4f9a8bf 100644 --- a/tests/app/Services/Supply/OpenRtbBridgeRegistrarTest.php +++ b/tests/app/Services/Supply/DspBridgeRegistrarTest.php @@ -25,7 +25,7 @@ use Adshares\Adserver\Models\Config; use Adshares\Adserver\Models\NetworkHost; -use Adshares\Adserver\Services\Supply\OpenRtbBridgeRegistrar; +use Adshares\Adserver\Services\Supply\DspBridgeRegistrar; use Adshares\Adserver\Tests\TestCase; use Adshares\Adserver\Utilities\DatabaseConfigReader; use Adshares\Common\Domain\ValueObject\AccountId; @@ -38,12 +38,12 @@ use Adshares\Supply\Application\Service\Exception\UnexpectedClientResponseException; use PHPUnit\Framework\MockObject\MockObject; -class OpenRtbBridgeRegistrarTest extends TestCase +class DspBridgeRegistrarTest extends TestCase { public function testRegisterAsNetworkHost(): void { - $this->initOpenRtb(); - $registrar = new OpenRtbBridgeRegistrar($this->getDemandClient()); + $this->initDspBridge(); + $registrar = new DspBridgeRegistrar($this->getDemandClient()); $result = $registrar->registerAsNetworkHost(); @@ -56,10 +56,10 @@ public function testRegisterAsNetworkHost(): void public function testRegisterAsNetworkHostFailWhileInvalidResponse(): void { - $this->initOpenRtb(); + $this->initDspBridge(); $clientMock = self::createMock(DemandClient::class); $clientMock->method('fetchInfo')->willThrowException(new UnexpectedClientResponseException('test-exception')); - $registrar = new OpenRtbBridgeRegistrar($clientMock); + $registrar = new DspBridgeRegistrar($clientMock); $result = $registrar->registerAsNetworkHost(); @@ -68,7 +68,7 @@ public function testRegisterAsNetworkHostFailWhileInvalidResponse(): void public function testRegisterAsNetworkHostFailWhileNoConfiguration(): void { - $registrar = new OpenRtbBridgeRegistrar($this->getDemandClient()); + $registrar = new DspBridgeRegistrar($this->getDemandClient()); $result = $registrar->registerAsNetworkHost(); @@ -77,8 +77,8 @@ public function testRegisterAsNetworkHostFailWhileNoConfiguration(): void public function testRegisterAsNetworkHostFailWhileInvalidConfigurationAddress(): void { - $this->initOpenRtb([Config::OPEN_RTB_BRIDGE_ACCOUNT_ADDRESS => '0001-00000004']); - $registrar = new OpenRtbBridgeRegistrar($this->getDemandClient()); + $this->initDspBridge([Config::DSP_BRIDGE_ACCOUNT_ADDRESS => '0001-00000004']); + $registrar = new DspBridgeRegistrar($this->getDemandClient()); $result = $registrar->registerAsNetworkHost(); @@ -87,8 +87,8 @@ public function testRegisterAsNetworkHostFailWhileInvalidConfigurationAddress(): public function testRegisterAsNetworkHostFailWhileInvalidConfigurationUrl(): void { - $this->initOpenRtb([Config::OPEN_RTB_BRIDGE_URL => 'example.com']); - $registrar = new OpenRtbBridgeRegistrar($this->getDemandClient()); + $this->initDspBridge([Config::DSP_BRIDGE_URL => 'example.com']); + $registrar = new DspBridgeRegistrar($this->getDemandClient()); $result = $registrar->registerAsNetworkHost(); @@ -97,8 +97,8 @@ public function testRegisterAsNetworkHostFailWhileInvalidConfigurationUrl(): voi public function testRegisterAsNetworkHostFailWhileInfoForAdserver(): void { - $this->initOpenRtb(); - $registrar = new OpenRtbBridgeRegistrar(new DummyDemandClient()); + $this->initDspBridge(); + $registrar = new DspBridgeRegistrar(new DummyDemandClient()); $result = $registrar->registerAsNetworkHost(); @@ -111,26 +111,8 @@ public function testRegisterAsNetworkHostFailWhileInfoForAdserver(): void public function testRegisterAsNetworkHostFailWhileInfoForDifferentAddress(): void { - $this->initOpenRtb(); - $info = new Info( - 'openrtb', - 'OpenRTB Provider ', - '0.1.0', - new Url('https://app.example.com'), - new Url('https://panel.example.com'), - new Url('https://example.com'), - new Url('https://example.com/privacy'), - new Url('https://example.com/terms'), - new Url('https://example.com/inventory'), - new AccountId('0001-00000005-CBCA'), - null, - [Info::CAPABILITY_ADVERTISER], - RegistrationMode::PRIVATE, - AppMode::OPERATIONAL - ); - $clientMock = self::createMock(DemandClient::class); - $clientMock->method('fetchInfo')->willReturn($info); - $registrar = new OpenRtbBridgeRegistrar($clientMock); + $this->initDspBridge(); + $registrar = new DspBridgeRegistrar($this->getDemandClient('0001-00000005-CBCA')); $result = $registrar->registerAsNetworkHost(); @@ -141,11 +123,11 @@ public function testRegisterAsNetworkHostFailWhileInfoForDifferentAddress(): voi ]); } - private function getDemandClient(): MockObject|DemandClient + private function getDemandClient(string $address = '0001-00000004-DBEB'): MockObject|DemandClient { $info = new Info( - 'openrtb', - 'OpenRTB Provider ', + 'dsp-bridge', + 'DSP bridge', '0.1.0', new Url('https://app.example.com'), new Url('https://panel.example.com'), @@ -153,7 +135,7 @@ private function getDemandClient(): MockObject|DemandClient new Url('https://example.com/privacy'), new Url('https://example.com/terms'), new Url('https://example.com/inventory'), - new AccountId('0001-00000004-DBEB'), + new AccountId($address), null, [Info::CAPABILITY_ADVERTISER], RegistrationMode::PRIVATE, @@ -164,12 +146,12 @@ private function getDemandClient(): MockObject|DemandClient return $clientMock; } - private function initOpenRtb(array $settings = []): void + private function initDspBridge(array $settings = []): void { $mergedSettings = array_merge( [ - Config::OPEN_RTB_BRIDGE_ACCOUNT_ADDRESS => '0001-00000004-DBEB', - Config::OPEN_RTB_BRIDGE_URL => 'https://example.com', + Config::DSP_BRIDGE_ACCOUNT_ADDRESS => '0001-00000004-DBEB', + Config::DSP_BRIDGE_URL => 'https://example.com', ], $settings, ); diff --git a/tests/app/Services/Supply/OpenRtbBridgeTest.php b/tests/app/Services/Supply/DspBridgeTest.php similarity index 89% rename from tests/app/Services/Supply/OpenRtbBridgeTest.php rename to tests/app/Services/Supply/DspBridgeTest.php index ce9ebb5bd..fc203a9c0 100644 --- a/tests/app/Services/Supply/OpenRtbBridgeTest.php +++ b/tests/app/Services/Supply/DspBridgeTest.php @@ -38,7 +38,7 @@ use Adshares\Adserver\Models\UserLedgerEntry; use Adshares\Adserver\Models\Zone; use Adshares\Adserver\Services\PaymentDetailsProcessor; -use Adshares\Adserver\Services\Supply\OpenRtbBridge; +use Adshares\Adserver\Services\Supply\DspBridge; use Adshares\Adserver\Tests\TestCase; use Adshares\Adserver\Utilities\AdsUtils; use Adshares\Adserver\Utilities\DatabaseConfigReader; @@ -64,21 +64,21 @@ use Ramsey\Uuid\Uuid; use Symfony\Component\HttpFoundation\Response; -class OpenRtbBridgeTest extends TestCase +class DspBridgeTest extends TestCase { public function testIsActiveWhileNotConfigured(): void { - self::assertFalse(OpenRtbBridge::isActive()); + self::assertFalse(DspBridge::isActive()); } public function testIsActiveWhileConfigured(): void { - $this->initOpenRtbConfiguration(); + $this->initDspBridgeConfiguration(); - self::assertTrue(OpenRtbBridge::isActive()); + self::assertTrue(DspBridge::isActive()); } - public function testReplaceOpenRtbBanners(): void + public function testReplaceBridgeBanners(): void { Http::preventStrayRequests(); Http::fake([ @@ -93,7 +93,7 @@ public function testReplaceOpenRtbBanners(): void $initiallyFoundBanners = $this->getFoundBanners(); $context = new ImpressionContext([], [], []); - $foundBanners = (new OpenRtbBridge())->replaceOpenRtbBanners($initiallyFoundBanners, $context); + $foundBanners = (new DspBridge())->replaceBridgeBanners($initiallyFoundBanners, $context); self::assertCount(1, $foundBanners); $foundBanner = $foundBanners->first(); @@ -103,7 +103,7 @@ public function testReplaceOpenRtbBanners(): void Http::assertSentCount(1); } - public function testReplaceOpenRtbBannersWithOptions(): void + public function testReplaceBridgeBannersWithOptions(): void { Http::preventStrayRequests(); Http::fake(function (Request $request) { @@ -136,7 +136,7 @@ public function testReplaceOpenRtbBannersWithOptions(): void ] ]; - $foundBanners = (new OpenRtbBridge())->replaceOpenRtbBanners($initiallyFoundBanners, $context, $zones); + $foundBanners = (new DspBridge())->replaceBridgeBanners($initiallyFoundBanners, $context, $zones); self::assertCount(1, $foundBanners); $foundBanner = $foundBanners->first(); @@ -146,21 +146,21 @@ public function testReplaceOpenRtbBannersWithOptions(): void Http::assertSentCount(1); } - public function testReplaceOpenRtbBannersWhileEmptyResponse(): void + public function testReplaceBridgeBannersWhileEmptyResponse(): void { Http::preventStrayRequests(); Http::fake(['example.com/serve' => Http::response([])]); $initiallyFoundBanners = $this->getFoundBanners(); $context = new ImpressionContext([], [], []); - $foundBanners = (new OpenRtbBridge())->replaceOpenRtbBanners($initiallyFoundBanners, $context); + $foundBanners = (new DspBridge())->replaceBridgeBanners($initiallyFoundBanners, $context); self::assertCount(1, $foundBanners); self::assertNull($foundBanners->first()); Http::assertSentCount(1); } - public function testReplaceOpenRtbBannersWhileNoBannersFromBridge(): void + public function testReplaceBridgeBannersWhileNoBannersFromBridge(): void { Http::preventStrayRequests(); Http::fake(); @@ -170,43 +170,43 @@ public function testReplaceOpenRtbBannersWhileNoBannersFromBridge(): void $initiallyFoundBanners->set(0, $banner); $context = new ImpressionContext([], [], []); - $foundBanners = (new OpenRtbBridge())->replaceOpenRtbBanners($initiallyFoundBanners, $context); + $foundBanners = (new DspBridge())->replaceBridgeBanners($initiallyFoundBanners, $context); self::assertCount(1, $foundBanners); self::assertEquals('3', $foundBanners->first()['request_id']); Http::assertNothingSent(); } - public function testReplaceOpenRtbBannersWhileInvalidStatus(): void + public function testReplaceBridgeBannersWhileInvalidStatus(): void { Http::preventStrayRequests(); Http::fake(['example.com/serve' => Http::response(status: Response::HTTP_NOT_FOUND)]); $initiallyFoundBanners = $this->getFoundBanners(); $context = new ImpressionContext([], [], []); - $foundBanners = (new OpenRtbBridge())->replaceOpenRtbBanners($initiallyFoundBanners, $context); + $foundBanners = (new DspBridge())->replaceBridgeBanners($initiallyFoundBanners, $context); self::assertCount(1, $foundBanners); self::assertNull($foundBanners->first()); Http::assertSentCount(1); } - public function testReplaceOpenRtbBannersWhileConnectionException(): void + public function testReplaceBridgeBannersWhileConnectionException(): void { Http::fake(fn() => throw new ConnectionException('test-exception')); $initiallyFoundBanners = $this->getFoundBanners(); $context = new ImpressionContext([], [], []); - $foundBanners = (new OpenRtbBridge())->replaceOpenRtbBanners($initiallyFoundBanners, $context); + $foundBanners = (new DspBridge())->replaceBridgeBanners($initiallyFoundBanners, $context); self::assertCount(1, $foundBanners); self::assertNull($foundBanners->first()); } /** - * @dataProvider replaceOpenRtbBannersWhileInvalidResponseProvider + * @dataProvider replaceBridgeBannersWhileInvalidResponseProvider */ - public function testReplaceOpenRtbBannersWhileInvalidResponse(mixed $response): void + public function testReplaceBridgeBannersWhileInvalidResponse(mixed $response): void { Http::preventStrayRequests(); Http::fake([ @@ -215,14 +215,14 @@ public function testReplaceOpenRtbBannersWhileInvalidResponse(mixed $response): $initiallyFoundBanners = $this->getFoundBanners(); $context = new ImpressionContext([], [], []); - $foundBanners = (new OpenRtbBridge())->replaceOpenRtbBanners($initiallyFoundBanners, $context); + $foundBanners = (new DspBridge())->replaceBridgeBanners($initiallyFoundBanners, $context); self::assertCount(1, $foundBanners); self::assertNull($foundBanners->first()); Http::assertSentCount(1); } - public function replaceOpenRtbBannersWhileInvalidResponseProvider(): array + public function replaceBridgeBannersWhileInvalidResponseProvider(): array { return [ 'not existing request id' => [ @@ -280,7 +280,7 @@ public function testGetEventRedirectUrl(Closure $responseClosure, ?string $expec Http::preventStrayRequests(); Http::fake(['example.com' => $responseClosure()]); - $url = (new OpenRtbBridge())->getEventRedirectUrl('https://example.com'); + $url = (new DspBridge())->getEventRedirectUrl('https://example.com'); self::assertEquals($expectedUrl, $url); Http::assertSentCount(1); @@ -302,7 +302,7 @@ public function testGetEventRedirectUrlWhileConnectionException(): void { Http::fake(fn() => throw new ConnectionException('test-exception')); - $url = (new OpenRtbBridge())->getEventRedirectUrl('https://example.com'); + $url = (new DspBridge())->getEventRedirectUrl('https://example.com'); self::assertEquals(null, $url); } @@ -345,11 +345,11 @@ public function testFetchAndStorePayments(): void 'value' => null, ], ]; - $this->initOpenRtbConfiguration(); + $this->initDspBridgeConfiguration(); Http::preventStrayRequests(); Http::fake(['example.com/payment-reports' => Http::response($responseData)]); - (new OpenRtbBridge())->fetchAndStorePayments(); + (new DspBridge())->fetchAndStorePayments(); self::assertDatabaseHas( BridgePayment::class, @@ -375,11 +375,11 @@ public function testFetchAndStorePayments(): void public function testFetchAndStorePaymentsWhileResponseIsEmpty(): void { $responseData = []; - $this->initOpenRtbConfiguration(); + $this->initDspBridgeConfiguration(); Http::preventStrayRequests(); Http::fake(['example.com/payment-reports' => Http::response($responseData)]); - (new OpenRtbBridge())->fetchAndStorePayments(); + (new DspBridge())->fetchAndStorePayments(); self::assertDatabaseEmpty(BridgePayment::class); Http::assertSentCount(1); @@ -388,10 +388,10 @@ public function testFetchAndStorePaymentsWhileResponseIsEmpty(): void public function testFetchAndStorePaymentsWhileConnectionException(): void { Log::spy(); - $this->initOpenRtbConfiguration(); + $this->initDspBridgeConfiguration(); Http::fake(fn() => throw new ConnectionException('test-exception')); - (new OpenRtbBridge())->fetchAndStorePayments(); + (new DspBridge())->fetchAndStorePayments(); self::assertDatabaseEmpty(BridgePayment::class); Log::shouldHaveReceived('error') @@ -409,9 +409,9 @@ public function testFetchAndStorePaymentsWhileInvalidResponse(mixed $response, s Http::fake([ 'example.com/payment-reports' => Http::response($response), ]); - $this->initOpenRtbConfiguration(); + $this->initDspBridgeConfiguration(); - (new OpenRtbBridge())->fetchAndStorePayments(); + (new DspBridge())->fetchAndStorePayments(); self::assertDatabaseEmpty(BridgePayment::class); Http::assertSentCount(1); @@ -485,7 +485,7 @@ public function testProcessPayments(): void $expectedPaidAmount = 1e11; $expectedLicenseFee = 0; - (new OpenRtbBridge())->processPayments($demandClient, $paymentDetailsProcessor); + (new DspBridge())->processPayments($demandClient, $paymentDetailsProcessor); self::assertDatabaseCount(BridgePayment::class, 1); self::assertDatabaseHas(BridgePayment::class, ['status' => BridgePayment::STATUS_DONE]); @@ -535,11 +535,11 @@ public function testProcessPaymentsRetryWhileDemandClientFailed(): void [], ); - (new OpenRtbBridge())->processPayments($demandClient, $paymentDetailsProcessor, 1); + (new DspBridge())->processPayments($demandClient, $paymentDetailsProcessor, 1); self::assertEquals(1, $bridgePayment->refresh()->last_offset); - (new OpenRtbBridge())->processPayments($demandClient, $paymentDetailsProcessor, 1); + (new DspBridge())->processPayments($demandClient, $paymentDetailsProcessor, 1); self::assertDatabaseCount(BridgePayment::class, 1); self::assertDatabaseHas(BridgePayment::class, ['status' => BridgePayment::STATUS_DONE]); @@ -585,7 +585,7 @@ public function testProcessPaymentsWhilePaymentCannotBeAddedToPublisherAccount() )->toArray() ); - (new OpenRtbBridge())->processPayments($demandClient, $paymentDetailsProcessor); + (new DspBridge())->processPayments($demandClient, $paymentDetailsProcessor); self::assertDatabaseCount(BridgePayment::class, 1); self::assertDatabaseHas(BridgePayment::class, ['status' => BridgePayment::STATUS_NEW]); @@ -601,7 +601,7 @@ public function testProcessPaymentsWhileHostDeleted(): void $paymentDetailsProcessor = self::createMock(PaymentDetailsProcessor::class); $demandClient = self::createMock(DemandClient::class); - (new OpenRtbBridge())->processPayments($demandClient, $paymentDetailsProcessor); + (new DspBridge())->processPayments($demandClient, $paymentDetailsProcessor); self::assertDatabaseCount(BridgePayment::class, 1); self::assertDatabaseHas(BridgePayment::class, ['status' => BridgePayment::STATUS_INVALID]); @@ -609,12 +609,12 @@ public function testProcessPaymentsWhileHostDeleted(): void self::assertDatabaseEmpty(NetworkCasePayment::class); } - private function initOpenRtbConfiguration(array $settings = []): void + private function initDspBridgeConfiguration(array $settings = []): void { $mergedSettings = array_merge( [ - Config::OPEN_RTB_BRIDGE_ACCOUNT_ADDRESS => '0001-00000001-8B4E', - Config::OPEN_RTB_BRIDGE_URL => 'https://example.com', + Config::DSP_BRIDGE_ACCOUNT_ADDRESS => '0001-00000001-8B4E', + Config::DSP_BRIDGE_URL => 'https://example.com', ], $settings, ); @@ -624,7 +624,7 @@ private function initOpenRtbConfiguration(array $settings = []): void private function getFoundBanners(): FoundBanners { - $this->initOpenRtbConfiguration(); + $this->initDspBridgeConfiguration(); NetworkHost::factory()->create([ 'address' => '0001-00000001-8B4E', 'host' => 'https://example.com', From cea2d920b09184fecc0d17b8da26eda1ab9729d2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82Podkalicki?= Date: Wed, 29 Mar 2023 16:31:27 +0200 Subject: [PATCH 42/55] change dsp bridge bid endpoint path --- app/Models/Config.php | 4 ++-- app/Services/Supply/DspBridge.php | 4 ++-- tests/app/Http/Controllers/SupplyControllerTest.php | 4 ++-- tests/app/Services/Supply/DspBridgeTest.php | 10 ++++------ 4 files changed, 10 insertions(+), 12 deletions(-) diff --git a/app/Models/Config.php b/app/Models/Config.php index dcee55373..55e0714c6 100644 --- a/app/Models/Config.php +++ b/app/Models/Config.php @@ -486,6 +486,8 @@ private static function getDefaultAdminSettings(array $fetched = []): array self::CURRENCY => Currency::ADS->value, self::DEFAULT_USER_ROLES => [UserRole::ADVERTISER, UserRole::PUBLISHER], self::DISPLAY_CURRENCY => Currency::USD->value, + self::DSP_BRIDGE_ACCOUNT_ADDRESS => null, + self::DSP_BRIDGE_URL => null, self::EMAIL_VERIFICATION_REQUIRED => true, self::EXCHANGE_API_KEY => '', self::EXCHANGE_API_SECRET => '', @@ -531,8 +533,6 @@ private static function getDefaultAdminSettings(array $fetched = []): array self::NOW_PAYMENTS_IPN_SECRET => '', self::NOW_PAYMENTS_MAX_AMOUNT => 1000, self::NOW_PAYMENTS_MIN_AMOUNT => 25, - self::DSP_BRIDGE_ACCOUNT_ADDRESS => null, - self::DSP_BRIDGE_URL => null, self::OPERATOR_RX_FEE => 0.01, self::OPERATOR_TX_FEE => 0.01, self::PUBLISHER_APPLY_FORM_URL => null, diff --git a/app/Services/Supply/DspBridge.php b/app/Services/Supply/DspBridge.php index ce93f9538..417fe3fa0 100644 --- a/app/Services/Supply/DspBridge.php +++ b/app/Services/Supply/DspBridge.php @@ -45,7 +45,7 @@ class DspBridge { private const PAYMENT_REPORT_READY_STATUS = 'done'; private const PAYMENTS_PATH = '/payment-reports'; - private const SERVE_PATH = '/serve'; + private const BID_PATH = '/bid'; private const SQL_QUERY_GET_PROCESSED_PAYMENTS_AMOUNT = << $context->toArray(), 'requests' => $bridgeBanners, diff --git a/tests/app/Http/Controllers/SupplyControllerTest.php b/tests/app/Http/Controllers/SupplyControllerTest.php index 9bbcd0881..f5526614a 100644 --- a/tests/app/Http/Controllers/SupplyControllerTest.php +++ b/tests/app/Http/Controllers/SupplyControllerTest.php @@ -293,7 +293,7 @@ public function testFindDspBridge(): void { Http::preventStrayRequests(); Http::fake([ - 'example.com/serve' => Http::response([[ + 'example.com/bid' => Http::response([[ 'request_id' => '0', 'ext_id' => '1', 'serve_url' => 'https://example.com/serve/1', @@ -313,7 +313,7 @@ public function testFindDspBridge(): void public function testFindDspBridgeWhileEmptyResponse(): void { Http::preventStrayRequests(); - Http::fake(['example.com/serve' => Http::response([])]); + Http::fake(['example.com/bid' => Http::response([])]); $data = $this->initDspBridge(); $response = $this->postJson(self::BANNER_FIND_URI, $data); diff --git a/tests/app/Services/Supply/DspBridgeTest.php b/tests/app/Services/Supply/DspBridgeTest.php index fc203a9c0..47fd43567 100644 --- a/tests/app/Services/Supply/DspBridgeTest.php +++ b/tests/app/Services/Supply/DspBridgeTest.php @@ -82,7 +82,7 @@ public function testReplaceBridgeBanners(): void { Http::preventStrayRequests(); Http::fake([ - 'example.com/serve' => Http::response([ + 'example.com/bid' => Http::response([ [ 'ext_id' => '1', 'request_id' => '0', @@ -149,7 +149,7 @@ public function testReplaceBridgeBannersWithOptions(): void public function testReplaceBridgeBannersWhileEmptyResponse(): void { Http::preventStrayRequests(); - Http::fake(['example.com/serve' => Http::response([])]); + Http::fake(['example.com/bid' => Http::response([])]); $initiallyFoundBanners = $this->getFoundBanners(); $context = new ImpressionContext([], [], []); @@ -180,7 +180,7 @@ public function testReplaceBridgeBannersWhileNoBannersFromBridge(): void public function testReplaceBridgeBannersWhileInvalidStatus(): void { Http::preventStrayRequests(); - Http::fake(['example.com/serve' => Http::response(status: Response::HTTP_NOT_FOUND)]); + Http::fake(['example.com/bid' => Http::response(status: Response::HTTP_NOT_FOUND)]); $initiallyFoundBanners = $this->getFoundBanners(); $context = new ImpressionContext([], [], []); @@ -209,9 +209,7 @@ public function testReplaceBridgeBannersWhileConnectionException(): void public function testReplaceBridgeBannersWhileInvalidResponse(mixed $response): void { Http::preventStrayRequests(); - Http::fake([ - 'example.com/serve' => Http::response($response), - ]); + Http::fake(['example.com/bid' => Http::response($response)]); $initiallyFoundBanners = $this->getFoundBanners(); $context = new ImpressionContext([], [], []); From 269614657e2dc3ae48c90ae7cce1cc15f5265ebb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82Podkalicki?= Date: Wed, 29 Mar 2023 18:30:42 +0200 Subject: [PATCH 43/55] do not secure urls from bridge --- app/Client/GuzzleDemandClient.php | 5 +++-- .../Supply/NetworkCampaignRepository.php | 14 +++++++++----- app/Services/Supply/DspBridge.php | 5 +++++ src/Supply/Domain/ValueObject/BannerUrl.php | 11 ++++------- tests/app/Services/Supply/DspBridgeTest.php | 19 +++++++++++++++++++ 5 files changed, 40 insertions(+), 14 deletions(-) diff --git a/app/Client/GuzzleDemandClient.php b/app/Client/GuzzleDemandClient.php index ac5e4a5a6..47b5fc09f 100644 --- a/app/Client/GuzzleDemandClient.php +++ b/app/Client/GuzzleDemandClient.php @@ -28,6 +28,7 @@ use Adshares\Adserver\Models\NetworkCampaign; use Adshares\Adserver\Repository\Common\ClassifierExternalRepository; use Adshares\Adserver\Services\Common\ClassifierExternalSignatureVerifier; +use Adshares\Adserver\Services\Supply\DspBridge; use Adshares\Adserver\Services\Supply\SiteFilteringUpdater; use Adshares\Adserver\Utilities\AdsAuthenticator; use Adshares\Adserver\ViewModel\MediumName; @@ -259,7 +260,7 @@ private function processData( 'updated_at' => DateTime::createFromFormat(DateTimeInterface::ATOM, $data['updated_at']), ]; - $isDspBridgeProvider = $sourceAddress === config('app.dsp_bridge_account_address'); + $isFromDspBridge = DspBridge::isDspAddress($sourceAddress); $classifiersRequired = $this->classifierRepository->fetchRequiredClassifiersNames(); $banners = []; $bannersInput = $data['creatives'] ?? $data['banners'];// legacy fallback, field 'banners' is deprecated @@ -272,7 +273,7 @@ private function processData( unset($banner['id']); } - $banner['classification'] = $isDspBridgeProvider + $banner['classification'] = $isFromDspBridge ? self::flattenClassification($banner['classification'] ?? []) : $this->validateAndMapClassification($banner); if ($this->missingRequiredClassifier($classifiersRequired, $banner['classification'])) { diff --git a/app/Repository/Supply/NetworkCampaignRepository.php b/app/Repository/Supply/NetworkCampaignRepository.php index 794c56a25..8b8bed2b9 100644 --- a/app/Repository/Supply/NetworkCampaignRepository.php +++ b/app/Repository/Supply/NetworkCampaignRepository.php @@ -1,7 +1,7 @@ getBanners(); $networkBanners = []; + $isFromDspBridge = DspBridge::isDspAddress($networkCampaign->source_address); foreach ($banners as $domainBanner) { $banner = $domainBanner->toArray(); $banner[self::BANNER_UUID_FIELD] = $banner[self::BANNER_ID_FIELD]; - $banner[self::BANNER_SERVE_URL_FIELD] = SecureUrl::change($banner[self::BANNER_SERVE_URL_FIELD]); - $banner[self::BANNER_CLICK_URL_FIELD] = SecureUrl::change($banner[self::BANNER_CLICK_URL_FIELD]); - $banner[self::BANNER_VIEW_URL_FIELD] = SecureUrl::change($banner[self::BANNER_VIEW_URL_FIELD]); + if (!$isFromDspBridge) { + $banner[self::BANNER_SERVE_URL_FIELD] = SecureUrl::change($banner[self::BANNER_SERVE_URL_FIELD]); + $banner[self::BANNER_CLICK_URL_FIELD] = SecureUrl::change($banner[self::BANNER_CLICK_URL_FIELD]); + $banner[self::BANNER_VIEW_URL_FIELD] = SecureUrl::change($banner[self::BANNER_VIEW_URL_FIELD]); + } - unset($banner['id']); + unset($banner[self::BANNER_ID_FIELD]); $networkBanner = NetworkBanner::where('demand_banner_id', hex2bin($domainBanner->getDemandBannerId())) ->where('network_campaign_id', $networkCampaign->id)->first(); diff --git a/app/Services/Supply/DspBridge.php b/app/Services/Supply/DspBridge.php index 417fe3fa0..220a4e71a 100644 --- a/app/Services/Supply/DspBridge.php +++ b/app/Services/Supply/DspBridge.php @@ -58,6 +58,11 @@ public static function isActive(): bool && null !== config('app.dsp_bridge_url'); } + public static function isDspAddress(string $address): bool + { + return $address === config('app.dsp_bridge_account_address'); + } + public function replaceBridgeBanners( FoundBanners $foundBanners, ImpressionContext $context, diff --git a/src/Supply/Domain/ValueObject/BannerUrl.php b/src/Supply/Domain/ValueObject/BannerUrl.php index 15a996f6f..221b6c351 100644 --- a/src/Supply/Domain/ValueObject/BannerUrl.php +++ b/src/Supply/Domain/ValueObject/BannerUrl.php @@ -1,7 +1,7 @@ initDspBridgeConfiguration(); + + self::assertFalse(DspBridge::isDspAddress('0001-00000004-DBEB')); + } + + public function testIsDspAddressWhileConfigured(): void + { + $this->initDspBridgeConfiguration(); + + self::assertTrue(DspBridge::isDspAddress('0001-00000001-8B4E')); + } + public function testReplaceBridgeBanners(): void { Http::preventStrayRequests(); From cd166241860370e7b87ba55ada362a5d5d84ace2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82Podkalicki?= Date: Mon, 3 Apr 2023 15:31:58 +0200 Subject: [PATCH 44/55] import inventory from bridge even if not on whitelist --- app/Models/NetworkHost.php | 3 +++ tests/app/Models/NetworkHostTest.php | 15 +++++++++++++++ 2 files changed, 18 insertions(+) diff --git a/app/Models/NetworkHost.php b/app/Models/NetworkHost.php index 42d7b44e6..cde8d965d 100644 --- a/app/Models/NetworkHost.php +++ b/app/Models/NetworkHost.php @@ -180,6 +180,9 @@ public static function fetchHosts(array $whitelist = []): Collection [HostStatus::Initialization, HostStatus::Operational], ); if (!empty($whitelist)) { + if (null !== ($accountAddress = config('app.dsp_bridge_account_address'))) { + $whitelist[] = $accountAddress; + } $query->whereIn('address', $whitelist); } return $query->get(); diff --git a/tests/app/Models/NetworkHostTest.php b/tests/app/Models/NetworkHostTest.php index bdea3d136..d8b95c597 100644 --- a/tests/app/Models/NetworkHostTest.php +++ b/tests/app/Models/NetworkHostTest.php @@ -135,6 +135,21 @@ public function testHandleWhitelist(): void self::assertEquals(HostStatus::Failure, $host5->refresh()->status); } + public function testFetchHostsWhileDspBridgeNotOnWhitelist(): void + { + $whitelist = ['0001-00000002-BB2D', '0001-00000003-AB0C']; + Config::updateAdminSettings([ + Config::DSP_BRIDGE_ACCOUNT_ADDRESS => '0001-00000001-8B4E', + Config::DSP_BRIDGE_URL => 'https://example.com', + ]); + DatabaseConfigReader::overwriteAdministrationConfig(); + NetworkHost::factory()->create(['address' => '0001-00000001-8B4E']); + + $hosts = NetworkHost::fetchHosts($whitelist); + + self::assertCount(1, $hosts); + } + public function testFetchUnreachableHostsForImportingInventory(): void { NetworkHost::factory()->count(3) From af39f4d5f0e4aa707dd3657820b6fb1a70a03080 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82Podkalicki?= Date: Tue, 4 Apr 2023 10:23:19 +0200 Subject: [PATCH 45/55] repeat adselect query discarding dsp bridge --- app/Client/GuzzleAdSelectClient.php | 6 + app/Http/Controllers/SupplyController.php | 25 ++++ .../Http/Controllers/SupplyControllerTest.php | 122 ++++++++++-------- 3 files changed, 97 insertions(+), 56 deletions(-) diff --git a/app/Client/GuzzleAdSelectClient.php b/app/Client/GuzzleAdSelectClient.php index 55dd854a8..ea5f08aba 100644 --- a/app/Client/GuzzleAdSelectClient.php +++ b/app/Client/GuzzleAdSelectClient.php @@ -177,6 +177,12 @@ public function findBanners(array $zones, ImpressionContext $context, string $im if (isset($zones[$i]['options']['banner_mime'])) { $sitesMap[$siteId]['filters']['require']['mime'] = (array)$zones[$i]['options']['banner_mime']; } + if (isset($zones[$i]['options']['exclude'])) { + $sitesMap[$siteId]['filters']['exclude'] = array_merge_recursive( + $sitesMap[$siteId]['filters']['exclude'], + $zones[$i]['options']['exclude'], + ); + } // always include active pop zones foreach ($site->zones as $popupZone) { if (!in_array($popupZone->uuid, $zoneIds)) { diff --git a/app/Http/Controllers/SupplyController.php b/app/Http/Controllers/SupplyController.php index 890e600cb..8a56c40cf 100644 --- a/app/Http/Controllers/SupplyController.php +++ b/app/Http/Controllers/SupplyController.php @@ -503,7 +503,32 @@ private function findBanners( $context = Utils::mergeImpressionContextAndUserContext($impressionContext, $userContext); $foundBanners = $bannerFinder->findBanners($zones, $context, $impressionId); if (DspBridge::isActive()) { + $indices = []; + foreach ($foundBanners as $index => $banner) { + if (null !== $banner) { + $indices[] = $index; + } + } $foundBanners = (new DspBridge())->replaceBridgeBanners($foundBanners, $context, $zones); + $indicesToReplace = []; + foreach ($indices as $index) { + if (null === $foundBanners[$index]) { + $indicesToReplace[$index] = $index; + } + } + if (!empty($indicesToReplace)) { + $exclude = ['source_host' => config('app.dsp_bridge_url')]; + $zonesToReplace = array_map( + function ($zone) use ($exclude) { + $zone['options']['exclude'] = $exclude; + return $zone; + }, + array_intersect_key($zones, $indicesToReplace), + ); + foreach ($bannerFinder->findBanners($zonesToReplace, $context, $impressionId) as $banner) { + $foundBanners[array_shift($indicesToReplace)] = $banner; + } + } } if ($foundBanners->exists(fn($key, $element) => null !== $element)) { diff --git a/tests/app/Http/Controllers/SupplyControllerTest.php b/tests/app/Http/Controllers/SupplyControllerTest.php index f5526614a..f18439650 100644 --- a/tests/app/Http/Controllers/SupplyControllerTest.php +++ b/tests/app/Http/Controllers/SupplyControllerTest.php @@ -56,6 +56,7 @@ use Illuminate\Support\Facades\Storage; use Psr\Http\Message\ResponseInterface; use Ramsey\Uuid\Uuid; +use Ramsey\Uuid\UuidInterface; use Symfony\Component\HttpFoundation\Response; // phpcs:ignoreFile PHPCompatibility.Numbers.RemovedHexadecimalNumericStrings.Found @@ -315,12 +316,28 @@ public function testFindDspBridgeWhileEmptyResponse(): void Http::preventStrayRequests(); Http::fake(['example.com/bid' => Http::response([])]); $data = $this->initDspBridge(); + $dspNetworkBanner = (new NetworkBanner())->first(); + /** @var NetworkBanner $adserverNetworkBanner */ + $adserverNetworkBanner = NetworkBanner::factory()->create([ + 'network_campaign_id' => NetworkCampaign::factory()->create(), + 'serve_url' => 'https://adshares.net/serve/' . Uuid::uuid4()->toString(), + ]); + $zone = Zone::fetchByPublicId($data['placements'][0]['placementId']); + $impressionId = $data['context']['iid']; + $adSelect = self::createMock(AdSelect::class); + $adSelect->expects(self::exactly(2))->method('findBanners')->willReturnOnConsecutiveCalls( + new FoundBanners([$this->createFoundBanner($dspNetworkBanner, $zone, $impressionId)]), + new FoundBanners([$this->createFoundBanner($adserverNetworkBanner, $zone, $impressionId)]), + ); + $this->app->bind(AdSelect::class, fn () => $adSelect); $response = $this->postJson(self::BANNER_FIND_URI, $data); $response->assertStatus(Response::HTTP_OK); $response->assertJsonStructure(self::FIND_BANNER_STRUCTURE); - $response->assertJsonCount(0, 'data'); + $response->assertJsonCount(1, 'data'); + $response->assertJsonPath('data.0.id', '3'); + $response->assertJsonPath('data.0.serveUrl', $adserverNetworkBanner->serve_url); Http::assertSentCount(1); } @@ -1183,14 +1200,7 @@ private function mockAdSelect(): void $response->method('getBody')->willReturn($content); return $response; }); - - - $this->app->bind( - AdSelect::class, - static function () use ($client) { - return new GuzzleAdSelectClient($client); - } - ); + $this->app->bind(AdSelect::class, fn () => new GuzzleAdSelectClient($client)); } private static function getDynamicFindData(array $merge = []): array @@ -1347,54 +1357,11 @@ private function initDspBridge(): array $impressionId = Uuid::uuid4(); $adSelect = self::createMock(AdSelect::class); $adSelect->method('findBanners')->willReturn( - new FoundBanners([ - [ - 'id' => $networkBanner->uuid, - 'demand_id' => $networkBanner->demand_banner_id, - 'publisher_id' => '0123456879ABCDEF0123456879ABCDEF', - 'zone_id' => $zone->uuid, - 'pay_from' => '0001-00000001-8B4E', - 'pay_to' => AdsUtils::normalizeAddress(config('app.adshares_address')), - 'type' => $networkBanner->type, - 'size' => $networkBanner->size, - 'serve_url' => $networkBanner->serve_url, - 'creative_sha1' => '', - 'click_url' => SecureUrl::change( - route( - 'log-network-click', - [ - 'id' => $networkBanner->uuid, - 'iid' => $impressionId, - 'r' => Utils::urlSafeBase64Encode($networkBanner->click_url), - 'zid' => $zone->uuid, - ] - ) - ), - 'view_url' => SecureUrl::change( - route( - 'log-network-view', - [ - 'id' => $networkBanner->uuid, - 'iid' => $impressionId, - 'r' => Utils::urlSafeBase64Encode($networkBanner->view_url), - 'zid' => $zone->uuid, - ] - ) - ), - 'info_box' => true, - 'rpm' => 0.5, - 'request_id' => '3', - ] - ]) - ); - $this->app->bind( - AdSelect::class, - static function () use ($adSelect) { - return $adSelect; - } + new FoundBanners([$this->createFoundBanner($networkBanner, $zone, $impressionId)]) ); + $this->app->bind(AdSelect::class, fn () => $adSelect); $this->initAdUser(); - $data = [ + return [ 'context' => [ 'iid' => $impressionId, 'url' => 'https://example.com', @@ -1408,6 +1375,49 @@ static function () use ($adSelect) { ], ], ]; - return $data; + } + + private function createFoundBanner( + NetworkBanner $networkBanner, + Zone $zone, + UuidInterface $impressionId + ): array { + return [ + 'id' => $networkBanner->uuid, + 'demand_id' => $networkBanner->demand_banner_id, + 'publisher_id' => '0123456879ABCDEF0123456879ABCDEF', + 'zone_id' => $zone->uuid, + 'pay_from' => '0001-00000001-8B4E', + 'pay_to' => AdsUtils::normalizeAddress(config('app.adshares_address')), + 'type' => $networkBanner->type, + 'size' => $networkBanner->size, + 'serve_url' => $networkBanner->serve_url, + 'creative_sha1' => '', + 'click_url' => SecureUrl::change( + route( + 'log-network-click', + [ + 'id' => $networkBanner->uuid, + 'iid' => $impressionId, + 'r' => Utils::urlSafeBase64Encode($networkBanner->click_url), + 'zid' => $zone->uuid, + ] + ) + ), + 'view_url' => SecureUrl::change( + route( + 'log-network-view', + [ + 'id' => $networkBanner->uuid, + 'iid' => $impressionId, + 'r' => Utils::urlSafeBase64Encode($networkBanner->view_url), + 'zid' => $zone->uuid, + ] + ) + ), + 'info_box' => true, + 'rpm' => 0.5, + 'request_id' => '3', + ]; } } From 91954bba2bc7711433f0d7c141ad22cb54d49e15 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82Podkalicki?= Date: Tue, 4 Apr 2023 16:06:25 +0200 Subject: [PATCH 46/55] add test --- tests/app/Client/GuzzleAdSelectClientTest.php | 191 ++++++++++++++++++ .../Mapper/AdSelect/CampaignMapperTest.php | 8 +- 2 files changed, 195 insertions(+), 4 deletions(-) create mode 100644 tests/app/Client/GuzzleAdSelectClientTest.php diff --git a/tests/app/Client/GuzzleAdSelectClientTest.php b/tests/app/Client/GuzzleAdSelectClientTest.php new file mode 100644 index 000000000..d5ed42936 --- /dev/null +++ b/tests/app/Client/GuzzleAdSelectClientTest.php @@ -0,0 +1,191 @@ + + */ + +declare(strict_types=1); + +namespace Adshares\Adserver\Tests\Client; + +use Adshares\Adserver\Client\GuzzleAdSelectClient; +use Adshares\Adserver\Models\NetworkBanner; +use Adshares\Adserver\Models\NetworkCampaign; +use Adshares\Adserver\Models\Site; +use Adshares\Adserver\Models\User; +use Adshares\Adserver\Models\Zone; +use Adshares\Adserver\Tests\TestCase; +use Adshares\Common\Domain\ValueObject\Uuid; +use Adshares\Supply\Application\Dto\ImpressionContext; +use Adshares\Supply\Application\Service\Exception\UnexpectedClientResponseException; +use Adshares\Supply\Domain\Factory\CampaignFactory; +use Adshares\Supply\Domain\Model\CampaignCollection; +use DateTime; +use GuzzleHttp\Client; +use GuzzleHttp\Handler\MockHandler; +use GuzzleHttp\HandlerStack; +use GuzzleHttp\Psr7\Response; + +class GuzzleAdSelectClientTest extends TestCase +{ + private const URI_FIND_BANNERS = '/api/v1/find'; + private const URI_INVENTORY = '/api/v1/campaigns'; + + public function testDeleteFromInventory(): void + { + $ids = ['10000000000000000000000000000000', '20000000000000000000000000000000']; + $campaigns = new CampaignCollection(); + foreach ($ids as $id) { + $campaigns->add(CampaignFactory::createFromArray($this->getCampaignData(['id' => new Uuid($id)]))); + } + $client = self::createMock(Client::class); + $client->expects(self::once())->method('delete')->with( + self::URI_INVENTORY, + self::callback(function ($options) use ($ids): bool { + self::assertIsArray($options); + self::assertArrayHasKey('json', $options); + self::assertIsArray($options['json']); + self::assertArrayHasKey('campaigns', $options['json']); + self::assertIsArray($options['json']['campaigns']); + self::assertEquals($ids, $options['json']['campaigns']); + return true; + }) + ); + $guzzleAdSelectClient = new GuzzleAdSelectClient($client); + + $guzzleAdSelectClient->deleteFromInventory($campaigns); + } + + public function testDeleteFromInventoryFail(): void + { + $ids = ['10000000000000000000000000000000', '20000000000000000000000000000000']; + $campaigns = new CampaignCollection(); + foreach ($ids as $id) { + $campaigns->add(CampaignFactory::createFromArray($this->getCampaignData(['id' => new Uuid($id)]))); + } + $mock = new MockHandler([ + new Response(500), + ]); + $handlerStack = HandlerStack::create($mock); + $client = new Client(['base_uri' => 'https://example.com', 'handler' => $handlerStack]); + $guzzleAdSelectClient = new GuzzleAdSelectClient($client); + + self::expectException(UnexpectedClientResponseException::class); + + $guzzleAdSelectClient->deleteFromInventory($campaigns); + } + + public function testFindBanners(): void + { + /** @var NetworkBanner $networkBanner */ + $networkBanner = NetworkBanner::factory()->create([ + 'network_campaign_id' => NetworkCampaign::factory()->create(), + 'serve_url' => 'https://adshares.net/serve/' . Uuid::v4()->toString(), + ]); + $responseBody = [ + '1' => [ + [ + 'banner_id' => $networkBanner->uuid, + 'rpm' => 0.5, + ], + ], + ]; + $client = self::createMock(Client::class); + $client->expects(self::once()) + ->method('post') + ->with( + self::URI_FIND_BANNERS, + self::callback(function ($options): bool { + self::assertIsArray($options); + self::assertArrayHasKey('json', $options); + self::assertIsArray($options['json']); + self::assertCount(1, $options['json']); + self::assertArrayHasKey('banner_filters', $options['json'][0]); + $filters = $options['json'][0]['banner_filters']; + self::assertEquals(['image'], $filters['require']['type']); + self::assertEquals(['image/png'], $filters['require']['mime']); + self::assertEquals(['source_host' => 'http://localhost:8000'], $filters['exclude']); + return true; + }) + ) + ->willReturn(new Response(body: json_encode($responseBody))); + $guzzleAdSelectClient = new GuzzleAdSelectClient($client); + + /** @var Zone $zone */ + $zone = Zone::factory() + ->create(['site_id' => Site::factory()->create(['user_id' => User::factory()->create()])]); + $zones = [ + [ + 'id' => '1', + 'placementId' => $zone->uuid, + 'options' => [ + 'banner_mime' => ['image/png'], + 'banner_type' => ['image'], + 'exclude' => ['source_host' => 'http://localhost:8000'], + ] + ], + ]; + $context = new ImpressionContext( + [], + [], + [ + 'keywords' => [], + 'uid' => Uuid::v4()->toString(), + ] + ); + $impressionId = Uuid::v4()->toString(); + + $foundBanners = $guzzleAdSelectClient->findBanners($zones, $context, $impressionId); + + self::assertCount(1, $foundBanners); + self::assertNotNull($foundBanners[0]); + self::assertEquals($networkBanner->uuid, $foundBanners[0]['id']); + } + + private function getCampaignData(array $merge = []): array + { + return array_merge( + [ + 'id' => Uuid::v4(), + 'demand_id' => Uuid::v4(), + 'landing_url' => 'https://exmaple.com', + 'date_start' => new DateTime('-1 day'), + 'date_end' => new DateTime('+2 days'), + 'created_at' => new DateTime('-1 days'), + 'updated_at' => new DateTime('-1 days'), + 'source_campaign' => [ + 'host' => 'localhost:8101', + 'address' => '0001-00000001-8B4E', + 'version' => '0.1', + 'created_at' => new DateTime(), + 'updated_at' => new DateTime(), + ], + 'banners' => [], + 'max_cpc' => null, + 'max_cpm' => null, + 'budget' => 1_000_000_000_000, + 'demand_host' => 'localhost:8101', + 'medium' => 'web', + 'vendor' => null, + 'targeting_excludes' => [], + 'targeting_requires' => [], + ], + $merge, + ); + } +} diff --git a/tests/app/Client/Mapper/AdSelect/CampaignMapperTest.php b/tests/app/Client/Mapper/AdSelect/CampaignMapperTest.php index d2a66e682..872109d17 100644 --- a/tests/app/Client/Mapper/AdSelect/CampaignMapperTest.php +++ b/tests/app/Client/Mapper/AdSelect/CampaignMapperTest.php @@ -182,10 +182,10 @@ private function getCampaignData(): array 'id' => Uuid::v4(), 'demand_id' => Uuid::v4(), 'landing_url' => 'http://adshares.pl', - 'date_start' => (new DateTime())->modify('-1 day'), - 'date_end' => (new DateTime())->modify('+2 days'), - 'created_at' => (new DateTime())->modify('-1 days'), - 'updated_at' => (new DateTime())->modify('-1 days'), + 'date_start' => new DateTime('-1 day'), + 'date_end' => new DateTime('+2 days'), + 'created_at' => new DateTime('-1 days'), + 'updated_at' => new DateTime('-1 days'), 'source_campaign' => [ 'host' => 'localhost:8101', 'address' => '0001-00000001-8B4E', From 0dc6a4d4028fda1b273ff4a33b4de35cbef15ec5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82Podkalicki?= Date: Wed, 5 Apr 2023 09:39:17 +0200 Subject: [PATCH 47/55] exclusion value as array --- CHANGELOG.md | 2 ++ app/Http/Controllers/SupplyController.php | 2 +- tests/app/Client/GuzzleAdSelectClientTest.php | 4 +-- .../Http/Controllers/SupplyControllerTest.php | 27 +++++++++++++++++++ 4 files changed, 32 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8bc9189eb..97f630383 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [Unreleased] +### Added +- DSP Bridge integration ## [2.4.12] - 2023-03-31 ### Added diff --git a/app/Http/Controllers/SupplyController.php b/app/Http/Controllers/SupplyController.php index 8a56c40cf..5ea4ea1f0 100644 --- a/app/Http/Controllers/SupplyController.php +++ b/app/Http/Controllers/SupplyController.php @@ -517,7 +517,7 @@ private function findBanners( } } if (!empty($indicesToReplace)) { - $exclude = ['source_host' => config('app.dsp_bridge_url')]; + $exclude = ['source_host' => [config('app.dsp_bridge_url')]]; $zonesToReplace = array_map( function ($zone) use ($exclude) { $zone['options']['exclude'] = $exclude; diff --git a/tests/app/Client/GuzzleAdSelectClientTest.php b/tests/app/Client/GuzzleAdSelectClientTest.php index d5ed42936..f2f67ec0b 100644 --- a/tests/app/Client/GuzzleAdSelectClientTest.php +++ b/tests/app/Client/GuzzleAdSelectClientTest.php @@ -119,7 +119,7 @@ public function testFindBanners(): void $filters = $options['json'][0]['banner_filters']; self::assertEquals(['image'], $filters['require']['type']); self::assertEquals(['image/png'], $filters['require']['mime']); - self::assertEquals(['source_host' => 'http://localhost:8000'], $filters['exclude']); + self::assertEquals(['source_host' => ['http://localhost:8000']], $filters['exclude']); return true; }) ) @@ -136,7 +136,7 @@ public function testFindBanners(): void 'options' => [ 'banner_mime' => ['image/png'], 'banner_type' => ['image'], - 'exclude' => ['source_host' => 'http://localhost:8000'], + 'exclude' => ['source_host' => ['http://localhost:8000']], ] ], ]; diff --git a/tests/app/Http/Controllers/SupplyControllerTest.php b/tests/app/Http/Controllers/SupplyControllerTest.php index f18439650..3ecd5eff1 100644 --- a/tests/app/Http/Controllers/SupplyControllerTest.php +++ b/tests/app/Http/Controllers/SupplyControllerTest.php @@ -480,6 +480,19 @@ public function findFailProvider(): array ], ] ], + 'missing placement placementId' => [ + [ + 'context' => [ + 'iid' => '0123456789ABCDEF0123456789ABCDEF', + 'url' => 'https://example.com', + ], + 'placements' => [ + [ + 'id' => '1', + ] + ], + ] + ], 'invalid placement id type' => [ [ 'context' => [ @@ -494,6 +507,20 @@ public function findFailProvider(): array ], ] ], + 'invalid placement placementId format' => [ + [ + 'context' => [ + 'iid' => '0123456789ABCDEF0123456789ABCDEF', + 'url' => 'https://example.com', + ], + 'placements' => [ + [ + 'id' => '1', + 'placementId' => '0123456789ABCDEF', + ] + ], + ] + ], 'invalid placement mimes type' => [ [ 'context' => [ From 6316ece377c42c88369d4ae0e867d1c06e0b9ac4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82Podkalicki?= Date: Wed, 5 Apr 2023 10:06:13 +0200 Subject: [PATCH 48/55] fix test --- tests/app/Client/GuzzleAdSelectClientTest.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/app/Client/GuzzleAdSelectClientTest.php b/tests/app/Client/GuzzleAdSelectClientTest.php index f2f67ec0b..6b48bfe85 100644 --- a/tests/app/Client/GuzzleAdSelectClientTest.php +++ b/tests/app/Client/GuzzleAdSelectClientTest.php @@ -141,7 +141,7 @@ public function testFindBanners(): void ], ]; $context = new ImpressionContext( - [], + ['page' => 'https://example.com'], [], [ 'keywords' => [], From 19013e6f58ef33198fa582091477b58d4175254b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82Podkalicki?= Date: Wed, 5 Apr 2023 13:26:32 +0200 Subject: [PATCH 49/55] Why page --- app/Http/Controllers/SupplyController.php | 37 +++++++++----- app/Models/NetworkCampaign.php | 3 +- resources/views/supply/why.blade.php | 2 +- .../Http/Controllers/SupplyControllerTest.php | 48 +++++++++++++++---- 4 files changed, 65 insertions(+), 25 deletions(-) diff --git a/app/Http/Controllers/SupplyController.php b/app/Http/Controllers/SupplyController.php index 5ea4ea1f0..3c82ed9e5 100644 --- a/app/Http/Controllers/SupplyController.php +++ b/app/Http/Controllers/SupplyController.php @@ -24,6 +24,7 @@ use Adshares\Adserver\Http\Controller; use Adshares\Adserver\Http\Utils; use Adshares\Adserver\Models\NetworkBanner; +use Adshares\Adserver\Models\NetworkCampaign; use Adshares\Adserver\Models\NetworkCase; use Adshares\Adserver\Models\NetworkCaseClick; use Adshares\Adserver\Models\NetworkHost; @@ -933,13 +934,12 @@ public function why(Request $request): View if (null === ($banner = NetworkBanner::fetchByPublicId($bannerId))) { throw new NotFoundHttpException('No matching banner'); } + /** @var NetworkCampaign $campaign */ $campaign = $banner->campaign()->first(); - $networkHost = NetworkHost::fetchByHost($campaign->source_host); + $isDsp = DspBridge::isDspAddress($campaign->source_address); - $info = $networkHost->info ?? null; - - $data = [ - 'url' => $banner->serve_url, + $supplyData = [ + 'url' => $isDsp ? '' : $banner->serve_url, 'source' => strtolower(preg_replace('/\s/', '-', config('app.adserver_name'))), 'supplyName' => config('app.adserver_name'), 'supplyTermsUrl' => route('terms-url'), @@ -955,27 +955,38 @@ public function why(Request $request): View ) ), 'supplyBannerRejectUrl' => config('app.adpanel_url') . '/publisher/classifier/' . $bannerId, - 'demand' => false, 'bannerSize' => $banner->size, 'bannerType' => $banner->type, ]; - if ($info) { - $data = array_merge( - $data, - [ + $demandData = [ + 'demand' => false, + ]; + if ($isDsp) { + $demandData = [ + 'demand' => true, + 'demandName' => $supplyData['supplyName'], + 'demandTermsUrl' => $supplyData['supplyTermsUrl'], + 'demandPrivacyUrl' => $supplyData['supplyPrivacyUrl'], + 'demandLandingUrl' => $supplyData['supplyLandingUrl'], + ]; + } else { + $networkHost = NetworkHost::fetchByAddress($campaign->source_address); + $info = $networkHost->info ?? null; + if ($info) { + $demandData = [ 'demand' => true, 'demandName' => $info->getName(), 'demandTermsUrl' => $info->getTermsUrl(), 'demandPrivacyUrl' => $info->getPrivacyUrl(), 'demandLandingUrl' => $info->getLandingUrl(), - ] - ); + ]; + } } return view( 'supply/why', - $data + array_merge($supplyData, $demandData), ); } diff --git a/app/Models/NetworkCampaign.php b/app/Models/NetworkCampaign.php index c0ce87d2d..fe3dff6c8 100644 --- a/app/Models/NetworkCampaign.php +++ b/app/Models/NetworkCampaign.php @@ -1,7 +1,7 @@
- @if(in_array($bannerType, ['image', 'html', 'video'])) + @if($url && in_array($bannerType, ['image', 'html', 'video'])) diff --git a/tests/app/Http/Controllers/SupplyControllerTest.php b/tests/app/Http/Controllers/SupplyControllerTest.php index 3ecd5eff1..8bb54f24b 100644 --- a/tests/app/Http/Controllers/SupplyControllerTest.php +++ b/tests/app/Http/Controllers/SupplyControllerTest.php @@ -143,11 +143,11 @@ public function testPageWhyNonExistentBannerId(): void public function testPageWhy(): void { - $host = 'https://example.com'; - $campaignId = 1; - NetworkHost::factory()->create(['host' => $host]); - NetworkCampaign::factory()->create(['id' => $campaignId, 'source_host' => $host]); - $banner = NetworkBanner::factory()->create(['id' => 1, 'network_campaign_id' => $campaignId]); + $address = '0001-00000003-AB0C'; + NetworkHost::factory()->create(['address' => $address]); + $campaign = NetworkCampaign::factory()->create(['source_address' => $address]); + /** @var NetworkBanner $banner */ + $banner = NetworkBanner::factory()->create(['id' => 1, 'network_campaign_id' => $campaign]); $query = [ 'bid' => $banner->uuid, 'cid' => '0123456789abcdef0123456789abcdef', @@ -156,16 +156,16 @@ public function testPageWhy(): void $response = $this->get(self::PAGE_WHY_URI . '?' . http_build_query($query)); $response->assertStatus(Response::HTTP_OK); + $response->assertViewHas('demand', true); } public function testPageWhyWhileCaseIdAndBannerIdAreUuid(): void { - $host = 'https://example.com'; - $campaignId = 1; - NetworkHost::factory()->create(['host' => $host]); - NetworkCampaign::factory()->create(['id' => $campaignId, 'source_host' => $host]); + $address = '0001-00000003-AB0C'; + NetworkHost::factory()->create(['address' => $address]); + $campaign = NetworkCampaign::factory()->create(['source_address' => $address]); /** @var NetworkBanner $banner */ - $banner = NetworkBanner::factory()->create(['id' => 1, 'network_campaign_id' => $campaignId]); + $banner = NetworkBanner::factory()->create(['id' => 1, 'network_campaign_id' => $campaign]); $query = [ 'bid' => Uuid::fromString($banner->uuid)->toString(), 'cid' => Uuid::uuid4()->toString(), @@ -174,6 +174,34 @@ public function testPageWhyWhileCaseIdAndBannerIdAreUuid(): void $response = $this->get(self::PAGE_WHY_URI . '?' . http_build_query($query)); $response->assertStatus(Response::HTTP_OK); + $response->assertViewHas('demand', true); + } + + public function testPageWhileBannerFromDsp(): void + { + $address = '0001-00000001-8B4E'; + $host = 'https://example.com'; + Config::updateAdminSettings([ + Config::DSP_BRIDGE_ACCOUNT_ADDRESS => $address, + Config::DSP_BRIDGE_URL => $host, + ]); + NetworkHost::factory()->create(['address' => $address]); + $campaign = NetworkCampaign::factory()->create([ + 'source_address' => $address, + 'source_host' => $host, + ]); + /** @var NetworkBanner $banner */ + $banner = NetworkBanner::factory()->create(['id' => 1, 'network_campaign_id' => $campaign]); + $query = [ + 'bid' => $banner->uuid, + 'cid' => '0123456789abcdef0123456789abcdef', + ]; + + $response = $this->get(self::PAGE_WHY_URI . '?' . http_build_query($query)); + + $response->assertStatus(Response::HTTP_OK); + $response->assertViewHas('demand', true); + $response->assertViewHas('demandName', 'AdServer'); } public function testReportAd(): void From 17641f03af86ca0f43e528160be26982b7460218 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82Podkalicki?= Date: Wed, 26 Apr 2023 13:46:58 +0200 Subject: [PATCH 50/55] hide logs --- tests/app/Http/Controllers/SupplyControllerTest.php | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/app/Http/Controllers/SupplyControllerTest.php b/tests/app/Http/Controllers/SupplyControllerTest.php index f7a72c940..acb4f18a0 100644 --- a/tests/app/Http/Controllers/SupplyControllerTest.php +++ b/tests/app/Http/Controllers/SupplyControllerTest.php @@ -1028,7 +1028,9 @@ public function testLogNetworkClickWhileBannerFromBridgeRedirection(): void Http::preventStrayRequests(); Http::fake(['example.com/click?*' => Http::response(['redirect_url' => 'https://adshares.net/click'])]); + ob_start(); $response = $this->get(self::buildLogClickUri($banner->uuid, $query)); + ob_get_clean(); $response->assertStatus(Response::HTTP_FOUND); $response->assertHeader('Location'); @@ -1046,7 +1048,9 @@ public function testLogNetworkClickWhileBannerFromBridgeNoRedirection(): void Http::preventStrayRequests(); Http::fake(['example.com/click?*' => Http::response(status: Response::HTTP_NO_CONTENT)]); + ob_start(); $response = $this->get(self::buildLogClickUri($banner->uuid, $query)); + ob_get_clean(); $response->assertStatus(Response::HTTP_FOUND); $response->assertHeader('Location'); @@ -1210,7 +1214,9 @@ public function testLogNetworkViewWhileBannerFromBridgeRedirection(): void Http::preventStrayRequests(); Http::fake(['example.com/view?*' => Http::response(['redirect_url' => 'https://adshares.net/view'])]); + ob_start(); $response = $this->get(self::buildLogViewUri($banner->uuid, $query)); + ob_get_clean(); $response->assertStatus(Response::HTTP_FOUND); $response->assertHeader('Location'); From 4e3abd5ec6dfcee0006d6fcb1857f4a49cf5018c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82Podkalicki?= Date: Wed, 26 Apr 2023 13:56:17 +0200 Subject: [PATCH 51/55] fix missing parameter after merge --- app/Services/PaymentDetailsProcessor.php | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/Services/PaymentDetailsProcessor.php b/app/Services/PaymentDetailsProcessor.php index 5c0e9fc4e..603fb18cf 100644 --- a/app/Services/PaymentDetailsProcessor.php +++ b/app/Services/PaymentDetailsProcessor.php @@ -130,7 +130,7 @@ public function processEventsPaidByBridge( $totalEventValue += $eventValue; } - return new PaymentProcessingResult($totalEventValue, 0); + return new PaymentProcessingResult($totalEventValue, 0, 0); } public function addAdIncomeToUserLedger(AdsPayment $adsPayment): void From 631e063201077125bd18d5d03222fb259f5410ec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82Podkalicki?= Date: Wed, 26 Apr 2023 15:04:34 +0200 Subject: [PATCH 52/55] fix tests --- .../Supply/DspBridgeRegistrarTest.php | 32 ++++++++----------- 1 file changed, 14 insertions(+), 18 deletions(-) diff --git a/tests/app/Services/Supply/DspBridgeRegistrarTest.php b/tests/app/Services/Supply/DspBridgeRegistrarTest.php index 7b4f9a8bf..2a7f5b9a1 100644 --- a/tests/app/Services/Supply/DspBridgeRegistrarTest.php +++ b/tests/app/Services/Supply/DspBridgeRegistrarTest.php @@ -28,9 +28,6 @@ use Adshares\Adserver\Services\Supply\DspBridgeRegistrar; use Adshares\Adserver\Tests\TestCase; use Adshares\Adserver\Utilities\DatabaseConfigReader; -use Adshares\Common\Domain\ValueObject\AccountId; -use Adshares\Common\Domain\ValueObject\Url; -use Adshares\Config\AppMode; use Adshares\Config\RegistrationMode; use Adshares\Mock\Client\DummyDemandClient; use Adshares\Supply\Application\Dto\Info; @@ -125,21 +122,20 @@ public function testRegisterAsNetworkHostFailWhileInfoForDifferentAddress(): voi private function getDemandClient(string $address = '0001-00000004-DBEB'): MockObject|DemandClient { - $info = new Info( - 'dsp-bridge', - 'DSP bridge', - '0.1.0', - new Url('https://app.example.com'), - new Url('https://panel.example.com'), - new Url('https://example.com'), - new Url('https://example.com/privacy'), - new Url('https://example.com/terms'), - new Url('https://example.com/inventory'), - new AccountId($address), - null, - [Info::CAPABILITY_ADVERTISER], - RegistrationMode::PRIVATE, - AppMode::OPERATIONAL + $info = Info::fromArray( + [ + 'module' => 'dsp-bridge', + 'name' => 'DSP bridge', + 'version' => '0.1.0', + 'serverUrl' => 'https://app.example.com', + 'panelUrl' => 'https://panel.example.com', + 'privacyUrl' => 'https://example.com/privacy', + 'termsUrl' => 'https://example.com/terms', + 'inventoryUrl' => 'https://example.com/inventory', + 'adsAddress' => $address, + 'capabilities' => [Info::CAPABILITY_ADVERTISER], + 'registrationMode' => RegistrationMode::PRIVATE, + ], ); $clientMock = self::createMock(DemandClient::class); $clientMock->method('fetchInfo')->willReturn($info); From fc42b30c598e1fb65360142d6e44cbbd0429ed41 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82Podkalicki?= Date: Fri, 5 May 2023 09:54:56 +0200 Subject: [PATCH 53/55] fix test --- tests/app/Http/Controllers/Manager/AdminControllerTest.php | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/app/Http/Controllers/Manager/AdminControllerTest.php b/tests/app/Http/Controllers/Manager/AdminControllerTest.php index 6b63356fc..c44f67c3b 100644 --- a/tests/app/Http/Controllers/Manager/AdminControllerTest.php +++ b/tests/app/Http/Controllers/Manager/AdminControllerTest.php @@ -27,6 +27,7 @@ use Adshares\Common\Domain\ValueObject\AccountId; use Adshares\Common\Domain\ValueObject\Commission; use Adshares\Common\Domain\ValueObject\License; +use Adshares\Common\Exception\RuntimeException; use DateTimeImmutable; use Symfony\Component\HttpFoundation\Response; @@ -85,6 +86,11 @@ public function testGetLicenseSuccess(): void public function testGetLicenseFail(): void { $this->actingAs(User::factory()->admin()->create(), 'api'); + $licenseVault = self::createMock(LicenseVault::class); + $licenseVault->expects(self::once()) + ->method('read') + ->willThrowException(new RuntimeException('test-exception')); + $this->app->bind(LicenseVault::class, fn() => $licenseVault); $response = $this->get(self::URI_LICENSE); From 859c86cf10c33f7126e377833501552693ece442 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82Podkalicki?= Date: Fri, 18 Aug 2023 13:12:58 +0200 Subject: [PATCH 54/55] Fix tests --- tests/app/Http/Controllers/SupplyControllerTest.php | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/tests/app/Http/Controllers/SupplyControllerTest.php b/tests/app/Http/Controllers/SupplyControllerTest.php index 522c3694b..ebdf3991d 100644 --- a/tests/app/Http/Controllers/SupplyControllerTest.php +++ b/tests/app/Http/Controllers/SupplyControllerTest.php @@ -148,8 +148,9 @@ public function testPageWhyNonExistentBannerId(): void public function testPageWhy(): void { $address = '0001-00000003-AB0C'; - NetworkHost::factory()->create(['address' => $address]); - $campaign = NetworkCampaign::factory()->create(['source_address' => $address]); + $host = 'https://example.com'; + NetworkHost::factory()->create(['address' => $address, 'host' => $host]); + $campaign = NetworkCampaign::factory()->create(['source_address' => $address, 'source_host' => $host]); /** @var NetworkBanner $banner */ $banner = NetworkBanner::factory()->create(['id' => 1, 'network_campaign_id' => $campaign]); $query = [ @@ -166,8 +167,9 @@ public function testPageWhy(): void public function testPageWhyWhileCaseIdAndBannerIdAreUuid(): void { $address = '0001-00000003-AB0C'; - NetworkHost::factory()->create(['address' => $address]); - $campaign = NetworkCampaign::factory()->create(['source_address' => $address]); + $host = 'https://example.com'; + NetworkHost::factory()->create(['address' => $address, 'host' => $host]); + $campaign = NetworkCampaign::factory()->create(['source_address' => $address, 'source_host' => $host]); /** @var NetworkBanner $banner */ $banner = NetworkBanner::factory()->create(['id' => 1, 'network_campaign_id' => $campaign]); $query = [ From 28f3b0b5433e0128cf25ab124cc9bb8218f9f654 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82Podkalicki?= Date: Fri, 18 Aug 2023 14:54:38 +0200 Subject: [PATCH 55/55] Increase coverage --- tests/app/Client/GuzzleDemandClientTest.php | 87 +++++++++++++++++++++ 1 file changed, 87 insertions(+) diff --git a/tests/app/Client/GuzzleDemandClientTest.php b/tests/app/Client/GuzzleDemandClientTest.php index 21a07897d..a9a2534ab 100644 --- a/tests/app/Client/GuzzleDemandClientTest.php +++ b/tests/app/Client/GuzzleDemandClientTest.php @@ -25,6 +25,8 @@ use Adshares\Adserver\Client\GuzzleDemandClient; use Adshares\Adserver\Models\Config; +use Adshares\Adserver\Models\NetworkBanner; +use Adshares\Adserver\Models\NetworkCampaign; use Adshares\Adserver\Repository\Common\ClassifierExternalRepository; use Adshares\Adserver\Services\Common\ClassifierExternalSignatureVerifier; use Adshares\Adserver\Tests\TestCase; @@ -239,6 +241,91 @@ public function testFetchAllInventoryWithSignatures(): void self::assertEquals('2022-02-10T14:08:00+00:00', $banner->getSignedAt()?->format(DateTimeInterface::ATOM)); } + public function testFetchAllInventoryWhileBannerExist(): void + { + $networkCampaign = NetworkCampaign::factory()->create( + [ + 'demand_campaign_id' => '12345678901234567890123456789012', + 'source_address' => '0001-00000004-DBEB', + 'uuid' => '10000000000000000000000000000000', + ] + ); + NetworkBanner::factory()->create( + [ + 'demand_banner_id' => '0123456789abcdef0123456789abcdef', + 'network_campaign_id' => $networkCampaign, + 'uuid' => '20000000000000000000000000000000', + ] + ); + $client = self::createMock(Client::class); + $inventoryResponse = json_decode($this->getInventoryResponse(), true); + unset($inventoryResponse[0]['medium']); + unset($inventoryResponse[0]['vendor']); + $client->expects(self::once()) + ->method('get') + ->willReturn(new GuzzleResponse(body: json_encode($inventoryResponse))); + /** @var Client $client */ + $demandClient = $this->createGuzzleDemandClient($client); + + $campaigns = $demandClient->fetchAllInventory( + new AccountId('0001-00000004-DBEB'), + 'https://example.com', + 'https://app.example.com/inventory', + false, + ); + + self::assertCount(1, $campaigns); + $campaign = $campaigns->get(0); + self::assertEquals('10000000000000000000000000000000', $campaign->getId()); + self::assertEquals('12345678901234567890123456789012', $campaign->getDemandCampaignId()); + self::assertCount(1, $campaign->getBanners()); + self::assertEquals('20000000000000000000000000000000', $campaign->getBanners()->get(0)->getId()); + self::assertEquals('0123456789abcdef0123456789abcdef', $campaign->getBanners()->get(0)->getDemandBannerId()); + } + + public function testFetchAllInventoryFromDspBridge(): void + { + Config::updateAdminSettings( + [ + Config::DSP_BRIDGE_ACCOUNT_ADDRESS => '0001-00000004-DBEB', + Config::DSP_BRIDGE_URL => 'https://example.com', + ] + ); + DatabaseConfigReader::overwriteAdministrationConfig(); + $client = self::createMock(Client::class); + $inventoryResponse = json_decode($this->getInventoryResponse(), true); + unset($inventoryResponse[0]['medium']); + unset($inventoryResponse[0]['vendor']); + $inventoryResponse[0]['targeting_requires'] = [ + 'site' => [ + 'domain' => ['https://scene-2-n5.decentraland.org'], + ], + ]; + $client->expects(self::once()) + ->method('get') + ->willReturn(new GuzzleResponse(body: json_encode($inventoryResponse))); + /** @var Client $client */ + $demandClient = $this->createGuzzleDemandClient($client); + + $campaigns = $demandClient->fetchAllInventory( + new AccountId('0001-00000004-DBEB'), + 'https://example.com', + 'https://app.example.com/inventory', + false, + ); + + self::assertCount(1, $campaigns); + /** @var Campaign $campaign */ + $campaign = $campaigns->get(0); + self::assertEquals('12345678901234567890123456789012', $campaign->getDemandCampaignId()); + self::assertEquals('metaverse', $campaign->getMedium()); + self::assertEquals('decentraland', $campaign->getVendor()); + self::assertCount(1, $campaign->getBanners()); + /** @var Banner $banner */ + $banner = $campaign->getBanners()->get(0); + self::assertEquals('0123456789abcdef0123456789abcdef', $banner->getDemandBannerId()); + } + public function testFetchAllInventoryWhileMissingRequiredClassification(): void { /** @var Client $client */