From eb17260b74da928d80e3df3e4284892dfa58a489 Mon Sep 17 00:00:00 2001 From: Tim MacDonald Date: Wed, 11 Oct 2023 15:17:42 +1100 Subject: [PATCH] Outgoing request improvements (#26) --- config/pulse.php | 4 + .../2023_06_07_000001_create_pulse_tables.php | 2 +- .../livewire/slow-outgoing-requests.blade.php | 8 +- src/Queries/SlowOutgoingRequests.php | 1 + src/Recorders/OutgoingRequests.php | 29 ++++- tests/Feature/OutgoingRequestsTest.php | 104 ++++++++++++++---- 6 files changed, 118 insertions(+), 30 deletions(-) diff --git a/config/pulse.php b/config/pulse.php index 3b07ae4d..876efe23 100644 --- a/config/pulse.php +++ b/config/pulse.php @@ -214,6 +214,10 @@ '/(.*)/' => '\1', ], + 'outgoing_request_uri_map' => [ + '/(.*)/' => '\1', + ], + /* |-------------------------------------------------------------------------- | Pulse Sample Rate diff --git a/database/migrations/2023_06_07_000001_create_pulse_tables.php b/database/migrations/2023_06_07_000001_create_pulse_tables.php index 0b49c541..df6a984c 100644 --- a/database/migrations/2023_06_07_000001_create_pulse_tables.php +++ b/database/migrations/2023_06_07_000001_create_pulse_tables.php @@ -90,7 +90,7 @@ public function up(): void $table->string('uri'); $table->unsignedInteger('duration'); $table->string('user_id')->nullable(); - // TODO: indexes? + $table->index(['uri', 'date', 'duration']); }); Schema::create('pulse_queue_sizes', function (Blueprint $table) { diff --git a/resources/views/livewire/slow-outgoing-requests.blade.php b/resources/views/livewire/slow-outgoing-requests.blade.php index 33cb367e..27b0479e 100644 --- a/resources/views/livewire/slow-outgoing-requests.blade.php +++ b/resources/views/livewire/slow-outgoing-requests.blade.php @@ -33,7 +33,7 @@ class="min-h-full flex flex-col" @else - @if (count($slowOutgoingRequests) === 0) + @if ($slowOutgoingRequests->isEmpty()) @else @@ -52,7 +52,7 @@ class="min-h-full flex flex-col" - @foreach ($slowOutgoingRequests as $request) + @foreach ($slowOutgoingRequests->take(100) as $request) @php [$method, $uri] = explode(' ', $request->uri, 2); @endphp @@ -63,7 +63,9 @@ class="min-h-full flex flex-col"
- + @if ($host = parse_url($uri, PHP_URL_HOST)) + + @endif {{ $uri }} diff --git a/src/Queries/SlowOutgoingRequests.php b/src/Queries/SlowOutgoingRequests.php index 4a8a5c55..93905afc 100644 --- a/src/Queries/SlowOutgoingRequests.php +++ b/src/Queries/SlowOutgoingRequests.php @@ -40,6 +40,7 @@ public function __invoke(Interval $interval): Collection ->where('duration', '>=', $this->config->get('pulse.slow_outgoing_request_threshold')) ->groupBy('uri') ->orderByDesc('slowest') + ->limit(101) ->get(); } } diff --git a/src/Recorders/OutgoingRequests.php b/src/Recorders/OutgoingRequests.php index fc981972..539ecb62 100644 --- a/src/Recorders/OutgoingRequests.php +++ b/src/Recorders/OutgoingRequests.php @@ -3,10 +3,12 @@ namespace Laravel\Pulse\Recorders; use Carbon\CarbonImmutable; +use Closure; use GuzzleHttp\Promise\RejectedPromise; +use Illuminate\Config\Repository; use Illuminate\Foundation\Application; use Illuminate\Http\Client\Factory as HttpFactory; -use Illuminate\Support\Str; +use Illuminate\Http\Request; use Laravel\Pulse\Concerns\ConfiguresAfterResolving; use Laravel\Pulse\Entry; use Laravel\Pulse\Pulse; @@ -31,6 +33,7 @@ class OutgoingRequests */ public function __construct( protected Pulse $pulse, + protected Repository $config, ) { // } @@ -53,13 +56,35 @@ public function record(RequestInterface $request, CarbonImmutable $startedAt): E $endedAt = new CarbonImmutable; return new Entry($this->table, [ - 'uri' => $request->getMethod().' '.Str::before($request->getUri(), '?'), + 'uri' => $this->normalizeUri($request), 'date' => $startedAt->toDateTimeString(), 'duration' => $startedAt->diffInMilliseconds($endedAt), 'user_id' => $this->pulse->authenticatedUserIdResolver(), ]); } + /** + * Normalize the request URI. + */ + protected function normalizeUri(RequestInterface $request): Closure + { + $method = $request->getMethod(); + + $uri = $request->getUri(); + + return function () use ($method, $uri) { + foreach ($this->config->get('pulse.outgoing_request_uri_map') as $pattern => $replacement) { + $normalized = preg_replace($pattern, $replacement, $uri, count: $count); + + if ($count > 0 && $normalized !== null) { + return "{$method} {$normalized}"; + } + } + + return "{$method} {$uri}"; + }; + } + /** * The recorder's middleware. */ diff --git a/tests/Feature/OutgoingRequestsTest.php b/tests/Feature/OutgoingRequestsTest.php index 86d0abf9..d755e9bc 100644 --- a/tests/Feature/OutgoingRequestsTest.php +++ b/tests/Feature/OutgoingRequestsTest.php @@ -21,16 +21,11 @@ Http::fake(['https://laravel.com' => Http::response('ok')]); Http::get('https://laravel.com'); - - expect(Pulse::entries())->toHaveCount(1); - Pulse::ignore(fn () => expect(DB::table('pulse_outgoing_requests')->count())->toBe(0)); - Pulse::store(app(Ingest::class)); - expect(Pulse::entries())->toHaveCount(0); $requests = Pulse::ignore(fn () => DB::table('pulse_outgoing_requests')->get()); expect($requests)->toHaveCount(1); - expect((array) $requests[0])->toEqual([ + expect($requests[0])->toHaveProperties([ 'date' => '2000-01-02 03:04:05', 'user_id' => null, 'uri' => 'GET https://laravel.com', @@ -47,24 +42,7 @@ $requests = Pulse::ignore(fn () => DB::table('pulse_outgoing_requests')->get()); expect($requests)->toHaveCount(1); - expect((array) $requests[0])->toEqual([ - 'date' => '2000-01-02 03:04:05', - 'user_id' => null, - 'uri' => 'GET https://laravel.com', - 'duration' => 0, - ]); -}); - -it('doesnt include query parameters', function () { - Carbon::setTestNow('2000-01-02 03:04:05'); - Http::fake(['https://laravel.com*' => Http::response('ok')]); - - Http::get('https://laravel.com?v=123'); - Pulse::store(app(Ingest::class)); - - $requests = Pulse::ignore(fn () => DB::table('pulse_outgoing_requests')->get()); - expect($requests)->toHaveCount(1); - expect((array) $requests[0])->toEqual([ + expect($requests[0])->toHaveProperties([ 'date' => '2000-01-02 03:04:05', 'user_id' => null, 'uri' => 'GET https://laravel.com', @@ -177,3 +155,81 @@ public function hasUser() expect($requests[1]->user_id)->toBe('567'); expect($requests[2]->user_id)->toBe('789'); }); + +it('stores the original URI by default', function () { + Carbon::setTestNow('2000-01-02 03:04:05'); + Http::fake(['https://laravel.com*' => Http::response('ok')]); + + Http::get('https://laravel.com?foo=123'); + Pulse::store(app(Ingest::class)); + + $requests = Pulse::ignore(fn () => DB::table('pulse_outgoing_requests')->get()); + expect($requests)->toHaveCount(1); + expect($requests[0])->toHaveProperties([ + 'date' => '2000-01-02 03:04:05', + 'user_id' => null, + 'uri' => 'GET https://laravel.com?foo=123', + 'duration' => 0, + ]); +}); + +it('can normalize URI', function () { + Carbon::setTestNow('2000-01-02 03:04:05'); + Http::fake(fn () => Http::response('ok')); + + Config::set('pulse.outgoing_request_uri_map', [ + '#^https://github\.com/([^/]+)/([^/]+)/commits/([^/]+)$#' => 'github.com/{user}/{repo}/commits/{branch}', + ]); + Http::get('https://github.com/laravel/pulse/commits/1.x'); + Pulse::store(app(Ingest::class)); + + $requests = Pulse::ignore(fn () => DB::table('pulse_outgoing_requests')->get()); + expect($requests)->toHaveCount(1); + expect($requests[0])->toHaveProperties([ + 'date' => '2000-01-02 03:04:05', + 'user_id' => null, + 'uri' => 'GET github.com/{user}/{repo}/commits/{branch}', + 'duration' => 0, + ]); +}); + +it('can use back references in normalized URI', function () { + Carbon::setTestNow('2000-01-02 03:04:05'); + Http::fake(fn () => Http::response('ok')); + + Config::set('pulse.outgoing_request_uri_map', [ + '#^https?://([^/]+).*$#' => '\1/*', + ]); + Http::get('https://github.com/laravel/pulse/commits/1.x'); + Pulse::store(app(Ingest::class)); + + $requests = Pulse::ignore(fn () => DB::table('pulse_outgoing_requests')->get()); + expect($requests)->toHaveCount(1); + expect($requests[0])->toHaveProperties([ + 'date' => '2000-01-02 03:04:05', + 'user_id' => null, + 'uri' => 'GET github.com/*', + 'duration' => 0, + ]); +}); + +it('can provide regex flags in normalization key', function () { + Carbon::setTestNow('2000-01-02 03:04:05'); + Http::fake(fn () => Http::response('ok')); + + Config::set('pulse.outgoing_request_uri_map', [ + '/parameter/i' => 'lowercase-parameter', + '/PARAMETER/i' => 'uppercase-parameter', + ]); + Http::get('https://github.com?PARAMETER=123'); + Pulse::store(app(Ingest::class)); + + $requests = Pulse::ignore(fn () => DB::table('pulse_outgoing_requests')->get()); + expect($requests)->toHaveCount(1); + expect($requests[0])->toHaveProperties([ + 'date' => '2000-01-02 03:04:05', + 'user_id' => null, + 'uri' => 'GET https://github.com?lowercase-parameter=123', + 'duration' => 0, + ]); +});