Skip to content

Commit

Permalink
Merge pull request #133 from fleetbase/dev-v1.5.27
Browse files Browse the repository at this point in the history
v1.5.27 ~ Hotfix patch DataPurger support class
  • Loading branch information
roncodes authored Feb 5, 2025
2 parents 58da31c + d9d8058 commit 3b0dbaa
Show file tree
Hide file tree
Showing 4 changed files with 260 additions and 41 deletions.
2 changes: 1 addition & 1 deletion composer.json
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
{
"name": "fleetbase/core-api",
"version": "1.5.26",
"version": "1.5.27",
"description": "Core Framework and Resources for Fleetbase API",
"keywords": [
"fleetbase",
Expand Down
144 changes: 144 additions & 0 deletions src/Console/Commands/PurgeOrphanedModelRecords.php
Original file line number Diff line number Diff line change
@@ -0,0 +1,144 @@
<?php

namespace Fleetbase\Console\Commands;

use Illuminate\Console\Command;
use Illuminate\Database\Eloquent\SoftDeletes;
use Illuminate\Support\Facades\DB;
use Illuminate\Support\Facades\Schema;

class PurgeOrphanedModelRecords extends Command
{
/**
* The name and signature of the console command.
*
* @var string
*/
protected $signature = 'purge:orphaned-model-records';

/**
* The console command description.
*
* @var string
*/
protected $description = 'Deletes orphaned records from model_has_roles, model_has_policies, and model_has_permissions where the referenced model no longer exists or is soft deleted.';

/**
* The tables to clean.
*
* @var array
*/
protected $tables = [
'model_has_roles',
'model_has_policies',
'model_has_permissions',
];

/**
* Execute the console command.
*/
public function handle()
{
$this->info('Starting orphaned model record purge...');

foreach ($this->tables as $table) {
if (!Schema::hasTable($table)) {
$this->warn("Table {$table} does not exist, skipping...");
continue;
}

$this->purgeTable($table);
}

$this->info('Purge process completed.');
}

/**
* Purges orphaned records from a given table.
*/
protected function purgeTable(string $table)
{
$this->info("Checking table: {$table}...");

$query = DB::table($table)
->select('model_type', 'model_uuid')
->distinct()
->get();

$deletedCount = 0;

foreach ($query as $record) {
$modelClass = $record->model_type;
$modelIdentifier = $record->model_uuid; // Can be either `uuid` or `id`

// Skip if model class does not exist
if (!class_exists($modelClass)) {
$this->warn("Model class {$modelClass} does not exist, skipping...");
continue;
}

// Get the primary key for the model
$primaryKey = $this->getModelPrimaryKey($modelClass);
if (!$primaryKey) {
$this->warn("Could not determine primary key for {$modelClass}, skipping...");
continue;
}

// Check if model uses SoftDeletes
if ($this->usesSoftDeletes($modelClass)) {
// Ensure soft-deleted records are ignored
$modelExists = $modelClass::where($primaryKey, $modelIdentifier)->whereNull('deleted_at')->exists();
} else {
$modelExists = $modelClass::where($primaryKey, $modelIdentifier)->exists();
}

// Delete orphaned record if model does not exist
if (!$modelExists) {
DB::table($table)
->where('model_type', $modelClass)
->where('model_uuid', $modelIdentifier)
->delete();

$deletedCount++;
$this->line("Deleted orphaned record from {$table} where model_type = {$modelClass} and model_uuid = {$modelIdentifier}");
}
}

$this->info("Finished checking {$table}. {$deletedCount} orphaned records deleted.");
}

/**
* Determines the primary key for a given model.
*/
protected function getModelPrimaryKey(string $modelClass): ?string
{
try {
$table = (new $modelClass())->getTable();

// Check if the table has a `uuid` column; if not, fallback to `id`
if (Schema::hasColumn($table, 'uuid')) {
return 'uuid';
} elseif (Schema::hasColumn($table, 'id')) {
return 'id';
}
} catch (\Exception $e) {
return null;
}

return null;
}

/**
* Checks if a model uses SoftDeletes.
*/
protected function usesSoftDeletes(string $modelClass): bool
{
try {
$reflection = new \ReflectionClass($modelClass);

return $reflection->hasMethod('bootSoftDeletes') || in_array(SoftDeletes::class, class_uses($modelClass));
} catch (\ReflectionException $e) {
return false;
}
}
}
1 change: 1 addition & 0 deletions src/Providers/CoreServiceProvider.php
Original file line number Diff line number Diff line change
Expand Up @@ -77,6 +77,7 @@ class CoreServiceProvider extends ServiceProvider
\Fleetbase\Console\Commands\PurgeWebhookLogs::class,
\Fleetbase\Console\Commands\PurgeActivityLogs::class,
\Fleetbase\Console\Commands\PurgeScheduledTaskLogs::class,
\Fleetbase\Console\Commands\PurgeOrphanedModelRecords::class,
\Fleetbase\Console\Commands\BackupDatabase\MysqlS3Backup::class,
];

Expand Down
154 changes: 114 additions & 40 deletions src/Support/DataPurger.php
Original file line number Diff line number Diff line change
Expand Up @@ -8,14 +8,17 @@

class DataPurger
{
protected static array $skipColumns = ['companies', 'registry_', 'billing_', 'model_has', 'role_has', 'monitored_scheduled_task_log_items'];

/**
* Delete all data related to a company, including foreign key relationships.
*
* @param \Fleetbase\Models\Company $company
* @param bool $deletePermanently whether to permanently delete the company or soft delete
* @param bool $verbose whether to output detailed logs during deletion
* @param callable|null $progressCallback Optional callback to track progress
*/
public static function deleteCompanyData($company, bool $deletePermanently = true, bool $verbose = false)
public static function deleteCompanyData($company, bool $deletePermanently = true, bool $verbose = false, ?callable $progressCallback = null)
{
$companyUuid = $company->uuid;
$defaultConnection = 'mysql';
Expand All @@ -33,7 +36,9 @@ public static function deleteCompanyData($company, bool $deletePermanently = tru

try {
// Fetch all table names
$tables = DB::select('SHOW TABLES');
$tables = DB::select('SHOW TABLES');
$totalTables = count($tables);
$tablesProcessed = 0;

// Track related records for deletion
$relatedRecords = [];
Expand All @@ -43,7 +48,7 @@ public static function deleteCompanyData($company, bool $deletePermanently = tru
$columns = Schema::getColumnListing($tableName);

// Skip system tables
if (Str::startsWith($tableName, ['registry_', 'billing_'])) {
if (Str::startsWith($tableName, static::$skipColumns)) {
continue;
}

Expand Down Expand Up @@ -71,10 +76,17 @@ public static function deleteCompanyData($company, bool $deletePermanently = tru
echo "No rows found in {$tableName} for company_uuid {$companyUuid}.\n";
}
}

// Update progress
$tablesProcessed++;
$progress = round(($tablesProcessed / $totalTables) * 100, 2);
if ($progressCallback) {
$progressCallback($progress, $tableName, $rowCount);
}
}

// Handle dependent records by foreign keys
self::deleteRelatedRecords($relatedRecords, $verbose);
self::deleteRelatedRecords($relatedRecords, $verbose, $progressCallback);
} catch (\Exception $e) {
DB::statement('SET FOREIGN_KEY_CHECKS=1;');
throw $e;
Expand All @@ -85,69 +97,125 @@ public static function deleteCompanyData($company, bool $deletePermanently = tru

// Reset to the original default connection
DB::setDefaultConnection($defaultConnection);
DB::statement('SET FOREIGN_KEY_CHECKS=0;');

// Delete the company record itself
if ($deletePermanently) {
$deletedRows = DB::delete('DELETE FROM companies WHERE uuid = ?', [$companyUuid]);

if ($progressCallback) {
$progressCallback(100, 'companies', $deletedRows);
}

if ($verbose) {
if ($deletedRows) {
echo "Permanently deleted company record for UUID {$companyUuid}.\n";
} else {
echo "Failed to delete company record for UUID {$companyUuid}. It may not exist or could not be found.\n";
}
echo $deletedRows
? "Permanently deleted company record for UUID {$companyUuid}.\n"
: "Failed to delete company record for UUID {$companyUuid}.\n";
}
} else {
$company->delete(); // Soft delete
try {
$company->delete(); // Soft delete
} catch (\Exception $e) {
echo $e->getMessage();
DB::statement('SET FOREIGN_KEY_CHECKS=1;');

return;
}

if ($verbose) {
echo "Soft deleted company record for UUID {$companyUuid}.\n";
}
}

DB::statement('SET FOREIGN_KEY_CHECKS=1;');
}

/**
* Deletes records from tables that reference previously deleted records.
*
* @param array $relatedRecords an associative array of table names and their primary keys to delete
* @param bool $verbose whether to output logs
* @param array $relatedRecords an associative array of table names and their primary keys to delete
* @param bool $verbose whether to output logs
* @param callable|null $progressCallback Optional callback for progress tracking
*/
protected static function deleteRelatedRecords(array $relatedRecords, bool $verbose = false)
protected static function deleteRelatedRecords(array $relatedRecords, bool $verbose = false, ?callable $progressCallback = null)
{
$processedTables = [];
$totalTables = count($relatedRecords);
$tablesProcessed = 0;

// Cache table schemas to avoid redundant queries
$allTables = collect(Schema::getAllTables())->mapWithKeys(fn ($table) => [array_values((array) $table)[0] => true]);
$columnCache = [];

foreach ($relatedRecords as $table => $primaryKeys) {
$columns = Schema::getColumnListing($table);
if ($verbose) {
echo "Checking related records in table: {$table}\n";
}

foreach ($columns as $column) {
foreach (Schema::getAllTables() as $relatedTable) {
$relatedTableName = array_values((array) $relatedTable)[0];
// Cache columns for this table
if (!isset($columnCache[$table])) {
$columnCache[$table] = Schema::getColumnListing($table);
}
$columns = $columnCache[$table];

// Skip system tables
if (Str::startsWith($relatedTableName, ['registry_', 'billing_'])) {
continue;
}
if (empty($columns)) {
continue;
}

if (in_array($relatedTableName, $processedTables)) {
continue; // Skip already processed tables
foreach ($allTables as $relatedTableName => $_) {
// Skip system tables
if (Str::startsWith($relatedTableName, static::$skipColumns)) {
continue;
}

if (in_array($relatedTableName, $processedTables)) {
continue;
}

// Cache columns for related table
if (!isset($columnCache[$relatedTableName])) {
$columnCache[$relatedTableName] = Schema::getColumnListing($relatedTableName);
}
$relatedColumns = $columnCache[$relatedTableName];

if (empty($relatedColumns)) {
continue;
}

$primaryKey = self::getPrimaryKey($relatedColumns);
if (!$primaryKey) {
continue;
}

// Collect all related foreign keys for batch deletion
$foreignKeyMatches = [];
foreach ($relatedColumns as $relatedColumn) {
if (self::isForeignKey($relatedColumn, $table)) {
$foreignKeyMatches[] = $relatedColumn;
}
}

if (!empty($foreignKeyMatches)) {
foreach ($foreignKeyMatches as $foreignKey) {
$dependentRecords = DB::table($relatedTableName)
->whereIn($foreignKey, $primaryKeys)
->pluck($primaryKey)
->toArray();

if (!empty($dependentRecords)) {
// Batch delete instead of multiple queries
DB::table($relatedTableName)->whereIn($foreignKey, $primaryKeys)->delete();
$processedTables[] = $relatedTableName;

$relatedColumns = Schema::getColumnListing($relatedTableName);
foreach ($relatedColumns as $relatedColumn) {
if (self::isForeignKey($relatedColumn, $table)) {
// Find dependent records
$dependentRecords = DB::table($relatedTableName)
->whereIn($relatedColumn, $primaryKeys)
->pluck(self::getPrimaryKey($relatedColumns))
->toArray();

if (!empty($dependentRecords)) {
DB::table($relatedTableName)->whereIn($relatedColumn, $primaryKeys)->delete();
$processedTables[] = $relatedTableName;

if ($verbose) {
echo 'Deleted ' . count($dependentRecords) . " dependent records from {$relatedTableName} where {$relatedColumn} matched deleted primary keys.\n";
}
if ($verbose) {
echo 'Deleted ' . count($dependentRecords) . " records from {$relatedTableName} where {$foreignKey} matched deleted primary keys.\n";
}

// Update progress
$tablesProcessed++;
$progress = round(($tablesProcessed / $totalTables) * 100, 2);
if ($progressCallback) {
$progressCallback($progress, $relatedTableName, count($dependentRecords));
}
}
}
Expand All @@ -163,7 +231,13 @@ protected static function deleteRelatedRecords(array $relatedRecords, bool $verb
*/
protected static function getPrimaryKey(array $columns)
{
return in_array('uuid', $columns) ? 'uuid' : (in_array('id', $columns) ? 'id' : null);
$primaryKey = in_array('uuid', $columns) ? 'uuid' : (in_array('id', $columns) ? 'id' : null);

if (!$primaryKey) {
echo 'Warning: Could not determine primary key from columns: ' . implode(',', $columns) . "\n";
}

return $primaryKey;
}

/**
Expand Down

0 comments on commit 3b0dbaa

Please sign in to comment.