Outgoing request improvements (#26)

config/pulse.php

@@ -214,6 +214,10 @@
         '/(.*)/' => '\1',
     ],
 
+    'outgoing_request_uri_map' => [
+        '/(.*)/' => '\1',
+    ],
+
     /*
     |--------------------------------------------------------------------------
     | Pulse Sample Rate

database/migrations/2023_06_07_000001_create_pulse_tables.php

@@ -90,7 +90,7 @@ public function up(): void
             $table->string('uri');
             $table->unsignedInteger('duration');
             $table->string('user_id')->nullable();
-            // TODO: indexes?
+            $table->index(['uri', 'date', 'duration']);
         });
 
         Schema::create('pulse_queue_sizes', function (Blueprint $table) {

resources/views/livewire/slow-outgoing-requests.blade.php

@@ -33,7 +33,7 @@ class="min-h-full flex flex-col"
                     </div>
                 </div>
             @else
-                @if (count($slowOutgoingRequests) === 0)
+                @if ($slowOutgoingRequests->isEmpty())
                     <x-pulse::no-results class="flex-1" />
                 @else
                     <x-pulse::table>
@@ -52,7 +52,7 @@ class="min-h-full flex flex-col"
                             </tr>
                         </x-pulse::thead>
                         <tbody>
-                            @foreach ($slowOutgoingRequests as $request)
+                            @foreach ($slowOutgoingRequests->take(100) as $request)
                                 @php
                                     [$method, $uri] = explode(' ', $request->uri, 2);
                                 @endphp
@@ -63,7 +63,9 @@ class="min-h-full flex flex-col"
                                     </x-pulse::td>
                                     <x-pulse::td class="max-w-[1px]">
                                         <div class="flex items-center" title="{{ $uri }}">
-                                            <img wire:ignore src="https://unavatar.io/{{ parse_url($uri, PHP_URL_HOST) }}?fallback=false" class="w-4 h-4 mr-2" onerror="this.style.display='none'" />
+                                            @if ($host = parse_url($uri, PHP_URL_HOST))
+                                                <img src="https://unavatar.io/{{ $host }}?fallback=false" loading="lazy" class="w-4 h-4 mr-2" onerror="this.style.display='none'" />
+                                            @endif
                                             <code class="block text-xs text-gray-900 dark:text-gray-100 truncate">
                                                 {{ $uri }}
                                             </code>

src/Queries/SlowOutgoingRequests.php

@@ -40,6 +40,7 @@ public function __invoke(Interval $interval): Collection
             ->where('duration', '>=', $this->config->get('pulse.slow_outgoing_request_threshold'))
             ->groupBy('uri')
             ->orderByDesc('slowest')
+            ->limit(101)
             ->get();
     }
 }

src/Recorders/OutgoingRequests.php

@@ -3,10 +3,12 @@
 namespace Laravel\Pulse\Recorders;
 
 use Carbon\CarbonImmutable;
+use Closure;
 use GuzzleHttp\Promise\RejectedPromise;
+use Illuminate\Config\Repository;
 use Illuminate\Foundation\Application;
 use Illuminate\Http\Client\Factory as HttpFactory;
-use Illuminate\Support\Str;
+use Illuminate\Http\Request;
 use Laravel\Pulse\Concerns\ConfiguresAfterResolving;
 use Laravel\Pulse\Entry;
 use Laravel\Pulse\Pulse;
@@ -31,6 +33,7 @@ class OutgoingRequests
      */
     public function __construct(
         protected Pulse $pulse,
+        protected Repository $config,
     ) {
         //
     }
@@ -53,13 +56,35 @@ public function record(RequestInterface $request, CarbonImmutable $startedAt): E
         $endedAt = new CarbonImmutable;
 
         return new Entry($this->table, [
-            'uri' => $request->getMethod().' '.Str::before($request->getUri(), '?'),
+            'uri' => $this->normalizeUri($request),
             'date' => $startedAt->toDateTimeString(),
             'duration' => $startedAt->diffInMilliseconds($endedAt),
             'user_id' => $this->pulse->authenticatedUserIdResolver(),
         ]);
     }
 
+    /**
+     * Normalize the request URI.
+     */
+    protected function normalizeUri(RequestInterface $request): Closure
+    {
+        $method = $request->getMethod();
+
+        $uri = $request->getUri();
+
+        return function () use ($method, $uri) {
+            foreach ($this->config->get('pulse.outgoing_request_uri_map') as $pattern => $replacement) {
+                $normalized = preg_replace($pattern, $replacement, $uri, count: $count);
+
+                if ($count > 0 && $normalized !== null) {
+                    return "{$method} {$normalized}";
+                }
+            }
+
+            return "{$method} {$uri}";
+        };
+    }
+
     /**
      * The recorder's middleware.
      */

tests/Feature/OutgoingRequestsTest.php

@@ -21,16 +21,11 @@
     Http::fake(['https://laravel.com' => Http::response('ok')]);
 
     Http::get('https://laravel.com');
-
-    expect(Pulse::entries())->toHaveCount(1);
-    Pulse::ignore(fn () => expect(DB::table('pulse_outgoing_requests')->count())->toBe(0));
-
     Pulse::store(app(Ingest::class));
 
-    expect(Pulse::entries())->toHaveCount(0);
     $requests = Pulse::ignore(fn () => DB::table('pulse_outgoing_requests')->get());
     expect($requests)->toHaveCount(1);
-    expect((array) $requests[0])->toEqual([
+    expect($requests[0])->toHaveProperties([
         'date' => '2000-01-02 03:04:05',
         'user_id' => null,
         'uri' => 'GET https://laravel.com',
@@ -47,24 +42,7 @@
 
     $requests = Pulse::ignore(fn () => DB::table('pulse_outgoing_requests')->get());
     expect($requests)->toHaveCount(1);
-    expect((array) $requests[0])->toEqual([
-        'date' => '2000-01-02 03:04:05',
-        'user_id' => null,
-        'uri' => 'GET https://laravel.com',
-        'duration' => 0,
-    ]);
-});
-
-it('doesnt include query parameters', function () {
-    Carbon::setTestNow('2000-01-02 03:04:05');
-    Http::fake(['https://laravel.com*' => Http::response('ok')]);
-
-    Http::get('https://laravel.com?v=123');
-    Pulse::store(app(Ingest::class));
-
-    $requests = Pulse::ignore(fn () => DB::table('pulse_outgoing_requests')->get());
-    expect($requests)->toHaveCount(1);
-    expect((array) $requests[0])->toEqual([
+    expect($requests[0])->toHaveProperties([
         'date' => '2000-01-02 03:04:05',
         'user_id' => null,
         'uri' => 'GET https://laravel.com',
@@ -177,3 +155,81 @@ public function hasUser()
     expect($requests[1]->user_id)->toBe('567');
     expect($requests[2]->user_id)->toBe('789');
 });
+
+it('stores the original URI by default', function () {
+    Carbon::setTestNow('2000-01-02 03:04:05');
+    Http::fake(['https://laravel.com*' => Http::response('ok')]);
+
+    Http::get('https://laravel.com?foo=123');
+    Pulse::store(app(Ingest::class));
+
+    $requests = Pulse::ignore(fn () => DB::table('pulse_outgoing_requests')->get());
+    expect($requests)->toHaveCount(1);
+    expect($requests[0])->toHaveProperties([
+        'date' => '2000-01-02 03:04:05',
+        'user_id' => null,
+        'uri' => 'GET https://laravel.com?foo=123',
+        'duration' => 0,
+    ]);
+});
+
+it('can normalize URI', function () {
+    Carbon::setTestNow('2000-01-02 03:04:05');
+    Http::fake(fn () => Http::response('ok'));
+
+    Config::set('pulse.outgoing_request_uri_map', [
+        '#^https://github\.com/([^/]+)/([^/]+)/commits/([^/]+)$#' => 'github.com/{user}/{repo}/commits/{branch}',
+    ]);
+    Http::get('https://github.com/laravel/pulse/commits/1.x');
+    Pulse::store(app(Ingest::class));
+
+    $requests = Pulse::ignore(fn () => DB::table('pulse_outgoing_requests')->get());
+    expect($requests)->toHaveCount(1);
+    expect($requests[0])->toHaveProperties([
+        'date' => '2000-01-02 03:04:05',
+        'user_id' => null,
+        'uri' => 'GET github.com/{user}/{repo}/commits/{branch}',
+        'duration' => 0,
+    ]);
+});
+
+it('can use back references in normalized URI', function () {
+    Carbon::setTestNow('2000-01-02 03:04:05');
+    Http::fake(fn () => Http::response('ok'));
+
+    Config::set('pulse.outgoing_request_uri_map', [
+        '#^https?://([^/]+).*$#' => '\1/*',
+    ]);
+    Http::get('https://github.com/laravel/pulse/commits/1.x');
+    Pulse::store(app(Ingest::class));
+
+    $requests = Pulse::ignore(fn () => DB::table('pulse_outgoing_requests')->get());
+    expect($requests)->toHaveCount(1);
+    expect($requests[0])->toHaveProperties([
+        'date' => '2000-01-02 03:04:05',
+        'user_id' => null,
+        'uri' => 'GET github.com/*',
+        'duration' => 0,
+    ]);
+});
+
+it('can provide regex flags in normalization key', function () {
+    Carbon::setTestNow('2000-01-02 03:04:05');
+    Http::fake(fn () => Http::response('ok'));
+
+    Config::set('pulse.outgoing_request_uri_map', [
+        '/parameter/i' => 'lowercase-parameter',
+        '/PARAMETER/i' => 'uppercase-parameter',
+    ]);
+    Http::get('https://github.com?PARAMETER=123');
+    Pulse::store(app(Ingest::class));
+
+    $requests = Pulse::ignore(fn () => DB::table('pulse_outgoing_requests')->get());
+    expect($requests)->toHaveCount(1);
+    expect($requests[0])->toHaveProperties([
+        'date' => '2000-01-02 03:04:05',
+        'user_id' => null,
+        'uri' => 'GET https://github.com?lowercase-parameter=123',
+        'duration' => 0,
+    ]);
+});