From 56208e7762ef59aeb9c110b1c63fa95b71003346 Mon Sep 17 00:00:00 2001 From: Tim MacDonald Date: Tue, 3 Dec 2024 02:17:11 +1100 Subject: [PATCH] Use dedicated config value for storage (#427) --- config/pulse.php | 4 ++ src/Storage/DatabaseStorage.php | 16 ++++++-- tests/Feature/Ingests/DatabaseTest.php | 53 +++++++++++++++++++++++--- tests/Feature/PulseTest.php | 2 +- tests/StorageFake.php | 11 +++++- 5 files changed, 74 insertions(+), 12 deletions(-) diff --git a/config/pulse.php b/config/pulse.php index d78c73b4..1edfa4ad 100644 --- a/config/pulse.php +++ b/config/pulse.php @@ -59,6 +59,10 @@ 'storage' => [ 'driver' => env('PULSE_STORAGE_DRIVER', 'database'), + 'trim' => [ + 'keep' => env('PULSE_STORAGE_KEEP', '7 days'), + ], + 'database' => [ 'connection' => env('PULSE_DB_CONNECTION'), 'chunk' => 1000, diff --git a/src/Storage/DatabaseStorage.php b/src/Storage/DatabaseStorage.php index a3213320..02c1af56 100644 --- a/src/Storage/DatabaseStorage.php +++ b/src/Storage/DatabaseStorage.php @@ -125,16 +125,24 @@ public function trim(): void { $now = CarbonImmutable::now(); - $keep = $this->config->get('pulse.ingest.trim.keep'); + $keep = $this->config->get('pulse.storage.trim.keep') ?? '7 days'; + + $before = $now->subMilliseconds( + (int) CarbonInterval::fromString($keep)->totalMilliseconds + ); + + if ($now->subDays(7)->isAfter($before)) { + $before = $now->subDays(7); + } $this->connection() ->table('pulse_values') - ->where('timestamp', '<=', $now->sub($keep)->getTimestamp()) + ->where('timestamp', '<=', $before->getTimestamp()) ->delete(); $this->connection() ->table('pulse_entries') - ->where('timestamp', '<=', $now->sub($keep)->getTimestamp()) + ->where('timestamp', '<=', $before->getTimestamp()) ->delete(); $this->connection() @@ -144,7 +152,7 @@ public function trim(): void ->each(fn (int $period) => $this->connection() ->table('pulse_aggregates') ->where('period', $period) - ->where('bucket', '<=', $now->subMinutes($period)->getTimestamp()) + ->where('bucket', '<=', max($now->subMinutes($period)->getTimestamp(), $before->getTimestamp())) ->delete()); } diff --git a/tests/Feature/Ingests/DatabaseTest.php b/tests/Feature/Ingests/DatabaseTest.php index 3ad6ed1e..c27bae9b 100644 --- a/tests/Feature/Ingests/DatabaseTest.php +++ b/tests/Feature/Ingests/DatabaseTest.php @@ -102,6 +102,23 @@ expect(DB::table('pulse_aggregates')->where('period', 1440)->count())->toBe(1); }); +it('trims aggregates to the configured storage duration when configured trim is shorter than the bucket period duration', function () { + Config::set('pulse.storage.trim.keep', '23 minutes'); + Date::setTestNow('2000-01-01 00:00:00'); // Bucket: 2000-01-01 00:00:00 + Pulse::record('foo', 'xxxx', 1)->count(); + Pulse::ingest(); + Pulse::stopRecording(); + expect(DB::table('pulse_aggregates')->where('period', 1440)->count())->toBe(1); + + Date::setTestNow('2000-01-01 00:22:59'); + App::make(DatabaseStorage::class)->trim(); + expect(DB::table('pulse_aggregates')->where('period', 1440)->count())->toBe(1); + + Date::setTestNow('2000-01-01 00:23:00'); + Pulse::ignore(fn () => App::make(DatabaseStorage::class)->trim()); + expect(DB::table('pulse_aggregates')->where('period', 1440)->count())->toBe(0); +}); + it('trims aggregates once the 7 day bucket is no longer relevant', function () { Date::setTestNow('2000-01-01 02:23:59'); // Bucket: 1999-12-31 23:36:00 Pulse::record('foo', 'xxxx', 1)->count(); @@ -124,19 +141,45 @@ }); it('can configure days of data to keep when trimming', function () { - Config::set('pulse.ingest.trim.keep', '14 days'); + Config::set('pulse.storage.trim.keep', '2 days'); Date::setTestNow('2000-01-01 00:00:04'); - Pulse::record('foo', 'xxxx', 1); + Pulse::record('foo', 'xxxx', 1)->count(); + Pulse::set('type', 'foo', 'value'); Date::setTestNow('2000-01-01 00:00:05'); - Pulse::record('bar', 'xxxx', 1); + Pulse::record('bar', 'xxxx', 1)->count(); + Pulse::set('type', 'bar', 'value'); Date::setTestNow('2000-01-01 00:00:06'); - Pulse::record('baz', 'xxxx', 1); + Pulse::record('baz', 'xxxx', 1)->count(); + Pulse::set('type', 'baz', 'value'); + Pulse::ingest(); + + Pulse::stopRecording(); + Date::setTestNow('2000-01-03 00:00:05'); + App::make(DatabaseStorage::class)->trim(); + + expect(DB::table('pulse_entries')->pluck('type')->all())->toBe(['baz']); + expect(DB::table('pulse_values')->pluck('key')->all())->toBe(['baz']); +}); + +it('restricts trim duration to 7 days', function () { + Config::set('pulse.storage.trim.keep', '7 days'); + + Date::setTestNow('2000-01-01 00:00:04'); + Pulse::record('foo', 'xxxx', 1)->count(); + Pulse::set('type', 'foo', 'value'); + Date::setTestNow('2000-01-01 00:00:05'); + Pulse::record('bar', 'xxxx', 1)->count(); + Pulse::set('type', 'bar', 'value'); + Date::setTestNow('2000-01-01 00:00:06'); + Pulse::record('baz', 'xxxx', 1)->count(); + Pulse::set('type', 'baz', 'value'); Pulse::ingest(); Pulse::stopRecording(); - Date::setTestNow('2000-01-15 00:00:05'); + Date::setTestNow('2000-01-08 00:00:05'); App::make(DatabaseStorage::class)->trim(); expect(DB::table('pulse_entries')->pluck('type')->all())->toBe(['baz']); + expect(DB::table('pulse_values')->pluck('key')->all())->toBe(['baz']); }); diff --git a/tests/Feature/PulseTest.php b/tests/Feature/PulseTest.php index c8245238..05cfa1d0 100644 --- a/tests/Feature/PulseTest.php +++ b/tests/Feature/PulseTest.php @@ -49,7 +49,7 @@ }); it('can configure days of data to keep when trimming', function () { - Config::set('pulse.ingest.trim.keep', '30 days'); + Config::set('pulse.storage.trim.keep', '30 days'); App::instance(Storage::class, $storage = new StorageFake); Pulse::record('foo', 'delete', 0, now()->subMonth()); diff --git a/tests/StorageFake.php b/tests/StorageFake.php index b52e5806..7e8fb8e5 100644 --- a/tests/StorageFake.php +++ b/tests/StorageFake.php @@ -2,6 +2,7 @@ namespace Tests; +use Carbon\CarbonImmutable; use Carbon\CarbonInterval; use Illuminate\Support\Collection; use Laravel\Pulse\Contracts\Storage; @@ -31,9 +32,15 @@ public function store(Collection $items): void */ public function trim(): void { - $keep = config('pulse.ingest.trim.keep'); + $now = CarbonImmutable::now(); - $this->stored = $this->stored->reject(fn ($record) => $record->timestamp <= now()->sub($keep)->timestamp); + $keep = config('pulse.storage.trim.keep') ?? '7 days'; + + $before = $now->subMilliseconds( + (int) CarbonInterval::fromString($keep)->totalMilliseconds + ); + + $this->stored = $this->stored->reject(fn ($record) => $record->timestamp <= $before->timestamp); } /**