Use dedicated config value for storage (#427)

Author: timacdonald

config/pulse.php

@@ -59,6 +59,10 @@
     'storage' => [
         'driver' => env('PULSE_STORAGE_DRIVER', 'database'),
 
+        'trim' => [
+            'keep' => env('PULSE_STORAGE_KEEP', '7 days'),
+        ],
+
         'database' => [
             'connection' => env('PULSE_DB_CONNECTION'),
             'chunk' => 1000,

src/Storage/DatabaseStorage.php

@@ -125,16 +125,24 @@ public function trim(): void
     {
         $now = CarbonImmutable::now();
 
-        $keep = $this->config->get('pulse.ingest.trim.keep');
+        $keep = $this->config->get('pulse.storage.trim.keep') ?? '7 days';
+
+        $before = $now->subMilliseconds(
+            (int) CarbonInterval::fromString($keep)->totalMilliseconds
+        );
+
+        if ($now->subDays(7)->isAfter($before)) {
+            $before = $now->subDays(7);
+        }
 
         $this->connection()
             ->table('pulse_values')
-            ->where('timestamp', '<=', $now->sub($keep)->getTimestamp())
+            ->where('timestamp', '<=', $before->getTimestamp())
             ->delete();
 
         $this->connection()
             ->table('pulse_entries')
-            ->where('timestamp', '<=', $now->sub($keep)->getTimestamp())
+            ->where('timestamp', '<=', $before->getTimestamp())
             ->delete();
 
         $this->connection()
@@ -144,7 +152,7 @@ public function trim(): void
             ->each(fn (int $period) => $this->connection()
                 ->table('pulse_aggregates')
                 ->where('period', $period)
-                ->where('bucket', '<=', $now->subMinutes($period)->getTimestamp())
+                ->where('bucket', '<=', max($now->subMinutes($period)->getTimestamp(), $before->getTimestamp()))
                 ->delete());
     }
 

tests/Feature/Ingests/DatabaseTest.php

@@ -102,6 +102,23 @@
     expect(DB::table('pulse_aggregates')->where('period', 1440)->count())->toBe(1);
 });
 
+it('trims aggregates to the configured storage duration when configured trim is shorter than the bucket period duration', function () {
+    Config::set('pulse.storage.trim.keep', '23 minutes');
+    Date::setTestNow('2000-01-01 00:00:00'); // Bucket: 2000-01-01 00:00:00
+    Pulse::record('foo', 'xxxx', 1)->count();
+    Pulse::ingest();
+    Pulse::stopRecording();
+    expect(DB::table('pulse_aggregates')->where('period', 1440)->count())->toBe(1);
+
+    Date::setTestNow('2000-01-01 00:22:59');
+    App::make(DatabaseStorage::class)->trim();
+    expect(DB::table('pulse_aggregates')->where('period', 1440)->count())->toBe(1);
+
+    Date::setTestNow('2000-01-01 00:23:00');
+    Pulse::ignore(fn () => App::make(DatabaseStorage::class)->trim());
+    expect(DB::table('pulse_aggregates')->where('period', 1440)->count())->toBe(0);
+});
+
 it('trims aggregates once the 7 day bucket is no longer relevant', function () {
     Date::setTestNow('2000-01-01 02:23:59'); // Bucket: 1999-12-31 23:36:00
     Pulse::record('foo', 'xxxx', 1)->count();
@@ -124,19 +141,45 @@
 });
 
 it('can configure days of data to keep when trimming', function () {
-    Config::set('pulse.ingest.trim.keep', '14 days');
+    Config::set('pulse.storage.trim.keep', '2 days');
 
     Date::setTestNow('2000-01-01 00:00:04');
-    Pulse::record('foo', 'xxxx', 1);
+    Pulse::record('foo', 'xxxx', 1)->count();
+    Pulse::set('type', 'foo', 'value');
     Date::setTestNow('2000-01-01 00:00:05');
-    Pulse::record('bar', 'xxxx', 1);
+    Pulse::record('bar', 'xxxx', 1)->count();
+    Pulse::set('type', 'bar', 'value');
     Date::setTestNow('2000-01-01 00:00:06');
-    Pulse::record('baz', 'xxxx', 1);
+    Pulse::record('baz', 'xxxx', 1)->count();
+    Pulse::set('type', 'baz', 'value');
+    Pulse::ingest();
+
+    Pulse::stopRecording();
+    Date::setTestNow('2000-01-03 00:00:05');
+    App::make(DatabaseStorage::class)->trim();
+
+    expect(DB::table('pulse_entries')->pluck('type')->all())->toBe(['baz']);
+    expect(DB::table('pulse_values')->pluck('key')->all())->toBe(['baz']);
+});
+
+it('restricts trim duration to 7 days', function () {
+    Config::set('pulse.storage.trim.keep', '7 days');
+
+    Date::setTestNow('2000-01-01 00:00:04');
+    Pulse::record('foo', 'xxxx', 1)->count();
+    Pulse::set('type', 'foo', 'value');
+    Date::setTestNow('2000-01-01 00:00:05');
+    Pulse::record('bar', 'xxxx', 1)->count();
+    Pulse::set('type', 'bar', 'value');
+    Date::setTestNow('2000-01-01 00:00:06');
+    Pulse::record('baz', 'xxxx', 1)->count();
+    Pulse::set('type', 'baz', 'value');
     Pulse::ingest();
 
     Pulse::stopRecording();
-    Date::setTestNow('2000-01-15 00:00:05');
+    Date::setTestNow('2000-01-08 00:00:05');
     App::make(DatabaseStorage::class)->trim();
 
     expect(DB::table('pulse_entries')->pluck('type')->all())->toBe(['baz']);
+    expect(DB::table('pulse_values')->pluck('key')->all())->toBe(['baz']);
 });

tests/Feature/PulseTest.php

@@ -49,7 +49,7 @@
 });
 
 it('can configure days of data to keep when trimming', function () {
-    Config::set('pulse.ingest.trim.keep', '30 days');
+    Config::set('pulse.storage.trim.keep', '30 days');
     App::instance(Storage::class, $storage = new StorageFake);
 
     Pulse::record('foo', 'delete', 0, now()->subMonth());

tests/StorageFake.php

@@ -2,6 +2,7 @@
 
 namespace Tests;
 
+use Carbon\CarbonImmutable;
 use Carbon\CarbonInterval;
 use Illuminate\Support\Collection;
 use Laravel\Pulse\Contracts\Storage;
@@ -31,9 +32,15 @@ public function store(Collection $items): void
      */
     public function trim(): void
     {
-        $keep = config('pulse.ingest.trim.keep');
+        $now = CarbonImmutable::now();
 
-        $this->stored = $this->stored->reject(fn ($record) => $record->timestamp <= now()->sub($keep)->timestamp);
+        $keep = config('pulse.storage.trim.keep') ?? '7 days';
+
+        $before = $now->subMilliseconds(
+            (int) CarbonInterval::fromString($keep)->totalMilliseconds
+        );
+
+        $this->stored = $this->stored->reject(fn ($record) => $record->timestamp <= $before->timestamp);
     }
 
     /**