Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Write dump information and dump progress bar to stderr when verbose m… #113

Merged
merged 1 commit into from
Mar 4, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 6 additions & 0 deletions app/config/services.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,11 @@ services:
arguments:
- '@faker.service'

console.dump_info:
class: 'Smile\GdprDump\Console\Helper\DumpInfo'
arguments:
- '@event_dispatcher'

command.compile:
class: 'Smile\GdprDump\Console\Command\CompileCommand'
arguments:
Expand All @@ -40,6 +45,7 @@ services:
- '@config.loader'
- '@config.schema_validator'
- '@config.compiler'
- '@console.dump_info'

config.loader:
class: 'Smile\GdprDump\Config\Loader\ConfigLoader'
Expand Down
8 changes: 7 additions & 1 deletion src/Console/Command/DumpCommand.php
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@
use Smile\GdprDump\Config\Loader\ConfigLoaderInterface;
use Smile\GdprDump\Config\Validator\ValidationResultInterface;
use Smile\GdprDump\Config\Validator\ValidatorInterface;
use Smile\GdprDump\Console\Helper\DumpInfo;
use Smile\GdprDump\Dumper\DumperInterface;
use Symfony\Component\Console\Command\Command;
use Symfony\Component\Console\Helper\QuestionHelper;
Expand All @@ -27,7 +28,8 @@ public function __construct(
private DumperInterface $dumper,
private ConfigLoaderInterface $configLoader,
private ValidatorInterface $validator,
private CompilerInterface $compiler
private CompilerInterface $compiler,
private DumpInfo $dumpInfo
) {
parent::__construct();
}
Expand Down Expand Up @@ -69,6 +71,10 @@ public function execute(InputInterface $input, OutputInterface $output): int
$config->set('database', $database);
}

if ($output->isVerbose()) {
$this->dumpInfo->setOutput($output);
}

$this->dumper->dump($config);
} catch (Exception $e) {
if ($output->isVerbose()) {
Expand Down
181 changes: 181 additions & 0 deletions src/Console/Helper/DumpInfo.php
Original file line number Diff line number Diff line change
@@ -0,0 +1,181 @@
<?php

declare(strict_types=1);

namespace Smile\GdprDump\Console\Helper;

use Smile\GdprDump\Database\Metadata\MetadataInterface;
use Smile\GdprDump\Dumper\Config\DumperConfig;
use Smile\GdprDump\Dumper\Event\DumpEvent;
use Smile\GdprDump\Dumper\Event\DumpFinishedEvent;
use Symfony\Component\Console\Helper\ProgressBar;
use Symfony\Component\Console\Output\ConsoleOutputInterface;
use Symfony\Component\Console\Output\OutputInterface;
use Symfony\Component\EventDispatcher\EventDispatcherInterface;

class DumpInfo
{
private OutputInterface $output;
private ProgressBar $progressBar;
private ?array $lastTableInfo = null;

public function __construct(private EventDispatcherInterface $eventDispatcher)
{
}

/**
* Set the output that will display the dump progress.
*/
public function setOutput(OutputInterface $output): void
{
if ($output instanceof ConsoleOutputInterface) {
$output = $output->getErrorOutput();
}

$this->output = $output;
$this->progressBar = new ProgressBar($output);
$this->progressBar->minSecondsBetweenRedraws(0.1); // default: 0.04
$this->eventDispatcher->addListener(DumpEvent::class, $this->getDumpStartedListener());
$this->eventDispatcher->addListener(DumpFinishedEvent::class, $this->getDumpFinishedListener());
}

/**
* Get the listener that will be triggered when a dump is started.
*/
private function getDumpStartedListener(): callable
{
return function (DumpEvent $event): void {
$config = $event->getConfig();
$database = $event->getDatabase();

// DSN
$this->displaySection('Database settings');
$this->displaySectionItem('Dsn', $database->getDriver()->getDsn());
$this->displaySectionItem(
'Using password',
$database->getConnectionParams()->get('password') ? 'yes' : 'no'
);

// Dump settings
$this->output->writeln('');
$this->displaySection('Dump settings');
foreach ($config->getDumpSettings() as $name => $value) {
$this->displaySectionItem($name, $value);
}

$this->output->writeln('');
$this->displaySection('Dump progress');

// Configure and start the progress bar
$this->progressBar->setFormat(
' %current%/%max% [%bar%] %percent:3s%% - %title% - %elapsed:6s% - %memory:6s%'
);
$this->progressBar->setMaxSteps($this->getMaxSteps($config, $database->getMetadata()));
$this->progressBar->setMessage('<info>Starting dump</info>', 'title');
$this->progressBar->start();

// Set the hook that will update the bar during the dump creation
$event->getDumper()->setInfoHook($this->getDumpInfoHook());
};
}

/**
* Get the listener that will be triggered when a dump is finished.
*/
private function getDumpFinishedListener(): callable
{
return function (): void {
if ($this->lastTableInfo) {
// Display information of the last table that was dumped
$this->updateProgressBarMessage($this->lastTableInfo);
}

$this->progressBar->finish();
$this->output->writeln('');
};
}

/**
* Get the hook that will be triggered when an object is being dumped.
*/
private function getDumpInfoHook(): callable
{
return function (string $object, array $info): void {
if ($object !== 'table') {
// The max steps of the progress bar only include tables
return;
}

if ($info['completed']) {
$this->lastTableInfo = $info;
return;
}

$this->updateProgressBarMessage($info);

$info['rowCount'] === 0
? $this->progressBar->advance() // new table
: $this->progressBar->setProgress($this->progressBar->getProgress()); // refresh current table progress
};
}

/**
* Display a section.
*/
private function displaySection(string $name): void
{
$this->output->writeln(sprintf('<comment>%s</comment>', $name));
}

/**
* Display a section item.
*/
private function displaySectionItem(string $name, mixed $value): void
{
$this->output->writeln(sprintf(' - %s: <info>%s</info>', $name, $this->formatValue($value)));
}

/**
* Update the progress bar message.
*/
private function updateProgressBarMessage(array $tableInfo): void
{
$message = sprintf('<info>%s</info> (%s)', $tableInfo['name'], $this->formatRowCount($tableInfo['rowCount']));
$this->progressBar->setMessage($message, 'title');
}

/**
* Get max number of steps of the progress bar.
*/
private function getMaxSteps(DumperConfig $config, MetadataInterface $metadata): int
{
$includedTables = $config->getTablesWhitelist() ?: $metadata->getTableNames();
$excludedTables = $config->getTablesBlacklist();

return count(array_diff($includedTables, $excludedTables));
}

/**
* Format a value for console output.
*/
private function formatValue(mixed $value): string
{
if (is_bool($value)) {
return $value ? 'true' : 'false';
}

if (is_scalar($value)) {
return (string) $value;
}

return json_encode($value, JSON_THROW_ON_ERROR | JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_SLASHES);
}

/**
* Format the row count.
*/
private function formatRowCount(int $rowCount): string
{
return $rowCount > 1 ? sprintf('%d rows', $rowCount) : sprintf('%d row', $rowCount);
Copy link
Contributor

@staabm staabm Mar 4, 2024

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

what do you think about a thousands separator?

in my case the numbers get rather long and harder to read:

Dump progress
  57/405 [===>------------------------]  14% - my_table (101748356 rows) - 26 mins, 1 sec - 35.5 MiB

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

With a space character as the separator of course 😃

}
}
4 changes: 2 additions & 2 deletions src/Dumper/Event/DumpEvent.php
Original file line number Diff line number Diff line change
Expand Up @@ -39,15 +39,15 @@ public function getDatabase(): Database
}

/**
* Get the dump context.
* Get the dumper.
*/
public function getDumper(): Mysqldump
{
return $this->dumper;
}

/**
* Get the context.
* Get the dump context.
*/
public function getContext(): array
{
Expand Down
26 changes: 26 additions & 0 deletions src/Dumper/Event/DumpFinishedEvent.php
Original file line number Diff line number Diff line change
@@ -0,0 +1,26 @@
<?php

declare(strict_types=1);

namespace Smile\GdprDump\Dumper\Event;

use Smile\GdprDump\Dumper\Config\DumperConfig;
use Symfony\Contracts\EventDispatcher\Event;

/**
* Event dispatched after a dump creation.
*/
class DumpFinishedEvent extends Event
{
public function __construct(private DumperConfig $config)
{
}

/**
* Get the dumper config.
*/
public function getConfig(): DumperConfig
{
return $this->config;
}
}
13 changes: 8 additions & 5 deletions src/Dumper/MysqlDumper.php
Original file line number Diff line number Diff line change
Expand Up @@ -10,12 +10,15 @@
use Smile\GdprDump\Dumper\Config\ConfigProcessor;
use Smile\GdprDump\Dumper\Config\DumperConfig;
use Smile\GdprDump\Dumper\Event\DumpEvent;
use Symfony\Component\EventDispatcher\EventDispatcher;
use Smile\GdprDump\Dumper\Event\DumpFinishedEvent;
use Symfony\Contracts\EventDispatcher\EventDispatcherInterface;

class MysqlDumper implements DumperInterface
{
public function __construct(private DatabaseFactory $databaseFactory, private EventDispatcher $eventDispatcher)
{
public function __construct(
private DatabaseFactory $databaseFactory,
private EventDispatcherInterface $eventDispatcher
) {
}

/**
Expand Down Expand Up @@ -49,15 +52,15 @@ public function dump(ConfigInterface $config): void
$database->getConnectionParams()->get('driverOptions', [])
);

$event = new DumpEvent($dumper, $database, $config, $context);
$this->eventDispatcher->dispatch($event);
$this->eventDispatcher->dispatch(new DumpEvent($dumper, $database, $config, $context));

// Close the Doctrine connection before proceeding to the dump creation (MySQLDump-PHP uses its own connection)
$database->getConnection()->close();

// Create the dump
$output = $config->getDumpOutput();
$dumper->start($output);
$this->eventDispatcher->dispatch(new DumpFinishedEvent($config));
}

/**
Expand Down
Loading