From 2159ae9c213bbe5f65d2ba22965af98086bea1da Mon Sep 17 00:00:00 2001 From: Pierre Rineau Date: Fri, 22 Nov 2024 10:22:12 +0100 Subject: [PATCH] issue #194 - configuration basics documentation --- docs/.vitepress/config.ts | 2 +- docs/content/anonymization/essentials.md | 8 +- docs/content/backup_restore.md | 2 +- docs/content/configuration.md | 329 ---------- docs/content/configuration/basics.md | 657 +++++++++++++++++++ docs/content/configuration/reference.md | 32 +- docs/content/contribute/pack.md | 4 +- docs/content/getting-started/basics.md | 5 +- docs/content/getting-started/flavors.md | 2 +- docs/content/getting-started/installation.md | 2 +- 10 files changed, 686 insertions(+), 357 deletions(-) delete mode 100644 docs/content/configuration.md create mode 100644 docs/content/configuration/basics.md diff --git a/docs/.vitepress/config.ts b/docs/.vitepress/config.ts index e975c83..600553b 100644 --- a/docs/.vitepress/config.ts +++ b/docs/.vitepress/config.ts @@ -85,7 +85,7 @@ export default defineConfig({ items: [ { text: 'Backup & Restore', link: '/backup_restore' }, { text: 'Statistics', link: '/stats' }, - { text: 'Bundle configuration', link: '/configuration' }, + { text: 'Configuration basics', link: '/configuration/basics' }, { text: 'Configuration reference', link: '/configuration/reference' }, ] }, diff --git a/docs/content/anonymization/essentials.md b/docs/content/anonymization/essentials.md index 3c7784c..119ec13 100644 --- a/docs/content/anonymization/essentials.md +++ b/docs/content/anonymization/essentials.md @@ -12,13 +12,13 @@ With the Symfony bundle, there is two ways to tell *DbToolsBundle* how it should 2. you can declare it with a **YAML** file ::: tip -The *DbToolsBundle* does not only work with Doctrine Entities to anonymize data. You can use it with +*DbToolsBundle* does not only work with Doctrine Entities to anonymize data. You can use it with *any* database, all you need is a DBAL connection. In such case, the [YAML configuration](../configuration#anonymization) is the only available option. ::: -If Doctrine ORM is enabled, the *DbToolsBundle* will automatically look for attributes on your entities. +If Doctrine ORM is enabled, *DbToolsBundle* will automatically look for attributes on your entities. If you want to use YAML configuration, look at the [Bundle Configuration section](../configuration#anonymization) to see how to configure it. @@ -224,7 +224,7 @@ customer: ## Going further -The DbToolsBundle provides a bunch of *Anonymizers* that should cover most of your needs. You can find a +*DbToolsBundle* provides a bunch of *Anonymizers* that should cover most of your needs. You can find a complete description of each one of them in the next section. You can also add *Anonymizers* from [community packs](./packs). For example, to add the `pack-fr-fr` run: @@ -233,7 +233,7 @@ You can also add *Anonymizers* from [community packs](./packs). For example, to composer require db-tools-bundle/pack-fr-fr ``` -If you can't find what you need from core anonymizers and in available packs, the *DbToolsBundle* allows +If you can't find what you need from core anonymizers and in available packs, *DbToolsBundle* allows you to [create your own *Custom Anonymizers*](./custom-anonymizers). ::: tip diff --git a/docs/content/backup_restore.md b/docs/content/backup_restore.md index fa0ad28..d07bc4d 100644 --- a/docs/content/backup_restore.md +++ b/docs/content/backup_restore.md @@ -307,7 +307,7 @@ php bin/console db-tools:restore --ignore-default-options ## Storage -As mentioned earlier on this page, the *DbToolsBundle* can list existing backup files +As mentioned earlier on this page, *DbToolsBundle* can list existing backup files when you want to restore a previous one with the restore command. All backups are stored in a directory. By default this directory is `./var/db_tools` (relative to the yaml config file)`%kernel.project_dir%/var/db_tools` diff --git a/docs/content/configuration.md b/docs/content/configuration.md deleted file mode 100644 index ff8e076..0000000 --- a/docs/content/configuration.md +++ /dev/null @@ -1,329 +0,0 @@ -# Bundle configuration - -The *DbToolsBundle* let you configure some of its behaviors. As with any classic Symfony Bundle, -all will take place in the `config/packages/db_tools.yaml` file. - -:::tip -A complete example of this file can be found in the bundle sources in: `vendor/makinacorpus/db-tools-bundle/config/packages/db_tools.yaml` -::: - -For detailed information about configuration options, please see the -[configuration reference](configuration/reference). - -## Backup configuration - -Some options are available to customize how the `db-tools:backup` command works. - -### Storage - -#### Root directory - -The `db_tools.storage.root_dir` parameter let you choose where to put the generated dumps. - -Default value is `'%kernel.project_dir%/var/db_tools'`. - -#### File and directory naming strategy - -Default behavior will store your backup using this strategy: -`%db_tools.storage.root_dir%///-.` -where `` is the file extension depending upon the database backend (mostly `.sql` or `.dump`). - -Custom strategy can be implemented by extending the -`MakinaCorpus\DbToolsBundle\Storage\AbstractFilenameStrategy` abstract class: - -```php -namespace App\DbTools\Storage; - -use MakinaCorpus\DbToolsBundle\Storage\AbstractFilenameStrategy; - -class FooFilenameStrategy extends AbstractFilenameStrategy -{ - #[\Override] - public function generateFilename( - string $connectionName = 'default', - string $extension = 'sql', - bool $anonymized = false - ): string { - return '/some_folder/' . $connectionName . '.' . $extension; - } -} -``` - -Then registered this way, on a per-connection basis: - -```yaml -# config/packages/db_tools.yaml - -db_tools: - storage: - filename_strategy: - connection_name: App\DbTools\Storage\FooFilenameStrategy -``` - -Value can be a container service identifier, or directly a class name in case this -has no constructor arguments. - -If you need to store your dumps outside of the `%db_tools.storage.root_dir%` directory, -then implement the `MakinaCorpus\DbToolsBundle\Storage\FilenameStrategyInterface` directly -and add the following method: - -```php -namespace App\DbTools\Storage; - -use MakinaCorpus\DbToolsBundle\Storage\FilenameStrategyInterface; - -class FooFilenameStrategy implements FilenameStrategyInterface -{ - #[\Override] - public function generateFilename(/* ... */): string {} - - #[\Override] - public function getRootDir( - string $defaultRootDir, - string $connectionName = 'default', - ): string { - return '/some/path/' . $connectionName . '/foo'; - } -} -``` - -This will allow the restore command to find your backups. - -### Excluded tables - -The `excluded_tables` parameter let you configure tables to exclude from backups. You will need to give a -configuration per doctrine connection. - -Default value is `null`: no table are excluded. - -Here is an example for exclude `table1` and `table2` for the `default` doctrine connection: - -```yml -# config/packages/db_tools.yaml - -db_tools: - - #... - - excluded_tables: - default: ['table1', 'table2'] - - #... -``` - -:::tip -Note that you can override this configuration while running the `db-tools:backup` command using -the `--exclude` option. -::: - -### Binary options - -See the [default binary options](#default-binary-options) section. - -### Backup expiration age - -The `backup_expiration_age` parameter let you choose when a backup is considered as obsolete. - -Default value is `'3 months ago'`. - -Use [PHP relative date/time formats](https://www.php.net/manual/en/datetime.formats.relative.php) -for this value. - -```yml -# config/packages/db_tools.yaml - -db_tools: - - #... - - backup_expiration_age: '1 week ago' - - #... -``` - -### Backup and restore timeout - -The `backup_timeout` and `restore_timeout` options let you choose what is the backup and restore -processes timeout in seconds. - -Default value is `600` (seconds) for backup, `1800` (seconds) for restore. - -Value can be either a [\DateInterval::createFromDateString()](https://www.php.net/manual/en/dateinterval.createfromdatestring.php) -compatible string value or a number of seconds as an integer value. - -```yml -# config/packages/db_tools.yaml - -db_tools: - - #... - - # As a date interval string. - backup_timeout: '6 minutes 30 seconds' - restore_timeout: '3 minutes 15 seconds' - - # As a number of seconds integer value. - backup_timeout: 390 - restore_timeout: 195 - - #... -``` - -## Binaries - -`db-tools:backup` and `db-tools:restore` need your system/environment to provide some extra binaries -to be able to work. These binaries depend on the database vendor you use, you will need: -* for MariaDB: `mariadb-dump` and `mariadb` -* for MySQL: `mysqldump` and `mysql` -* for PostgreSQL: `pg_dump` and `pg_restore` -* for SQLite: `sqlite3` - -You can verify if those binaries are well found by the *DbToolsBundle*, -for each of your DBAL connections, by launching: - -```sh -php bin/console db-tools:check -``` - -If the `db-tools:check` command returns you some errors: - * if your binaries are present on your system but the *DbToolsBundle* can't find them: you will need - to specify path for these binaries: - - ```yml - # config/packages/db_tools.yaml - - db_tools: - - #... - - # Specify here paths to binaries, only if the system can't find them by himself - # platform are 'mysql', 'postgresql', 'sqlite' - backupper_binaries: - mariadb: '/usr/bin/mariadb-dump' # default 'mariadb-dump' - mysql: '/usr/bin/mysqldump' # default 'mysqldump' - postgresql: '/usr/bin/pg_dump' # default 'pg_dump' - sqlite: '/usr/bin/sqlite3' # default 'sqlite3' - restorer_binaries: - mariadb: '/usr/bin/mariadb' # default 'mariadb' - mysql: '/usr/bin/mysql' # default 'mysql' - postgresql: '/usr/bin/pg_restore' # default 'pg_restore' - sqlite: '/usr/bin/sqlite3' # default 'sqlite3' - - #... - ``` - * Or, if your binaries are not present on your system: you will need to install them - - - -:::tip -If your app lives in the [official PHP docker image](https://hub.docker.com/_/php/), -you can install correct binaries adding these lines to your Dockerfile, - -for PostgreSQL: - -``` -RUN apt-get update && \ - apt-get install -y --no-install-recommends postgresql-client -``` - -for MariaDB/MySQL: - -``` -RUN apt-get update && \ - apt-get install -y --no-install-recommends default-mysql-client -``` -::: - -:::warning -Dump and restore is not supported yet for SQL Server. -::: - -### Default binary options - -Apart from the essential options (credentials, database name, etc.), the bundle -also passes a few default options to the binary depending on the operation being -performed and the invoked binary itself. You can customize those default options -by configuring your own ones per operation type and DBAL connection: - -```yaml -# config/packages/db_tools.yaml -db_tools: - # ... - - backupper_options: - default: '--an-option' - another_connection: '-xyz --another' - restorer_options: - default: '--a-first-one --a-second-one' - another_connection: '-O sample-value' -``` - -If you do not define your own default options, the following ones will be used -according to the database vendor: - -* When backing up: - * MariaDB: `--no-tablespaces` - * MySQL: `--no-tablespaces` - * PostgreSQL: `-Z 5 --lock-wait-timeout=120` - * SQLite: `-bail` -* When restoring: - * MariaDB: None - * MySQL: None - * PostgreSQL: `-j 2 --clean --if-exists --disable-triggers` - * SQLite: None - -## Anonymizer paths - -By default, the *DbToolsBundle* will look for *anonymizers* in 2 directories - -* `%kernel.project_dir%/vendor/makinacorpus/db-tools-bundle/src/Anonymizer` -* `%kernel.project_dir%/src/Anonymizer` - -If you want to put custom anonymizers in another directory or if you want to load -a pack of anonymizers from an external library, you can modify/add paths: - - -```yml -# config/packages/db_tools.yaml - -db_tools: - # ... - - anonymizer_paths: - - '%kernel.project_dir%/vendor/makinacorpus/db-tools-bundle/src/Anonymization/Anonymizer' - - '%kernel.project_dir%/src/Anonymizer' - - '%kernel.project_dir%/vendor/myAnonymizerProvider/anonymizers/src' - - # ... -``` - -## Anonymization - -Per default, the **DbToolsBundle** will only look for anonymization configurations from PHP attributes on Doctrine Entities. - -But the **DbToolsBundle** does not necessary need Doctrine ORM to anonymize your data, it can do it just with a DBAL connection. -In this case (or if you prefer YAML over attributes): you can configure the DbToolsBundle to look for anonymization -configurations in a YAML file: - -```yml -# config/packages/db_tools.yaml - -db_tools: - # ... - - anonymization: - # If you want to load configuration from a yaml: - # 1/ If you want to configure anonymization only for the default - # DBAL connection, declare it like this: - yaml: '%kernel.project_dir%/config/anonymizations.yaml' - # 2/ If you use multiple connections, declare each configuration like this: - #yaml: - #- connection_one: '%kernel.project_dir%/config/anonymizations/connection_one.yaml' - #- connection_two: '%kernel.project_dir%/config/anonymizations/connection_two.yaml' - - #... -``` - -:::tip -For more information about anonymization, refer to the [Anonymization section](./anonymization/essentials). -::: diff --git a/docs/content/configuration/basics.md b/docs/content/configuration/basics.md new file mode 100644 index 0000000..574a796 --- /dev/null +++ b/docs/content/configuration/basics.md @@ -0,0 +1,657 @@ +# Configuration basics + +Configuration options will vary depending on which flavor you want to use. + +Select below your target: + + + +
+ +*DbToolsBundle* let you configure some of its behaviors. As with any classic Symfony Bundle, +all will take place in the `config/packages/db_tools.yaml` file. + +:::tip +A complete example of this file can be found in the bundle sources in: `vendor/makinacorpus/db-tools-bundle/config/packages/db_tools.yaml`. +::: + +
+ +
+ +*DbToolsBundle* let you configure some of its behaviors +all will take place in your configuration file, usually `db_tools.yaml`. + +:::tip +**In this page, all paths are relative to the `db_tools.yaml` configuration file.** + +A complete example of this file can be found in the library sources in: `vendor/makinacorpus/db-tools-bundle/config/db_tools.standalone.complete.sample.yaml`. +::: + +
+ +For detailed information about configuration options, please see the +[configuration reference](configuration/reference). + +:::tip +**Almost every configuration option can be configured at the connection level** for example the +backup excluded tables can either be configured top-level (for all connections): + +
+ +```yml +# config/packages/db_tools.yaml +db_tools: + backup_excluded_tables: ['table1', 'table2'] +``` + +Or for each connection: + +```yml +# config/packages/db_tools.yaml +db_tools: + connections: + connection_one: + backup_excluded_tables: ['table1', 'table2'] + connection_two: + backup_excluded_tables: ['table3', 'table4'] +``` + +
+
+ +```yml +# db_tools.yaml +backup_excluded_tables: ['table1', 'table2'] +``` + +Or for each connection: + +```yml +# db_tools.yaml +connections: + connection_one: + backup_excluded_tables: ['table1', 'table2'] + connection_two: + backup_excluded_tables: ['table3', 'table4'] +``` + +
+ +When working with multiple connections, any connection which does not specify the option +will inherit from the default. +::: + +## Backup configuration + +Some options are available to customize how the `db-tools:backup` command works. + +### Storage + +#### Root directory + +The `db_tools.storage.root_dir` parameter let you choose where to put the generated dumps. + +
+ +Default value is `'%kernel.project_dir%/var/db_tools'`. + +
+ +
+ +Default value is `./var/db_tools'`. + +
+ +#### File and directory naming strategy + +Default behavior will store your backup using this strategy: +`%db_tools.storage.root_dir%///-.` +where `` is the file extension depending upon the database vendor (mostly `.sql` or `.dump`). + +
+ +Custom strategy can be implemented by extending the +`MakinaCorpus\DbToolsBundle\Storage\AbstractFilenameStrategy` abstract class: + +```php +namespace App\DbTools\Storage; + +use MakinaCorpus\DbToolsBundle\Storage\AbstractFilenameStrategy; + +class FooFilenameStrategy extends AbstractFilenameStrategy +{ + #[\Override] + public function generateFilename( + string $connectionName = 'default', + string $extension = 'sql', + bool $anonymized = false + ): string { + return '/some_folder/' . $connectionName . '.' . $extension; + } +} +``` + +Then registered this way, on a per-connection basis: + +```yaml +# config/packages/db_tools.yaml +db_tools: + storage: + filename_strategy: + connection_name: App\DbTools\Storage\FooFilenameStrategy +``` + +Value can be a container service identifier, or directly a class name in case this +has no constructor arguments. + +If you need to store your dumps outside of the `%db_tools.storage.root_dir%` directory, +then implement the `MakinaCorpus\DbToolsBundle\Storage\FilenameStrategyInterface` directly +and add the following method: + +```php +namespace App\DbTools\Storage; + +use MakinaCorpus\DbToolsBundle\Storage\FilenameStrategyInterface; + +class FooFilenameStrategy implements FilenameStrategyInterface +{ + #[\Override] + public function generateFilename(/* ... */): string {} + + #[\Override] + public function getRootDir( + string $defaultRootDir, + string $connectionName = 'default', + ): string { + return '/some/path/' . $connectionName . '/foo'; + } +} +``` + +This will allow the restore command to find your backups. + +
+ +
+ +:::warning +There is as of now no way to implement a custom filename strategy when using *DbToolsBundle* as a standalone +CLI tool. + +If you need this feature, please let us know by [filing an issue](https://github.com/makinacorpus/DbToolsBundle/issues). +::: + +
+ +:::info +More filename strategies may be implemented in core in the future. If you have any +suggestions, please [open an discussion](https://github.com/makinacorpus/DbToolsBundle/issues) about it. +::: + +### Excluded tables + +The `backup_excluded_tables` parameter let you configure tables to exclude from backups. + +Default value is `null`: no table are excluded. + +
+ +Here is an example for exclude `table1` and `table2` for all connections: + +```yml +# config/packages/db_tools.yaml +db_tools: + backup_excluded_tables: ['table1', 'table2'] +``` + +Or set a specific table list for each connection: + +```yml +# config/packages/db_tools.yaml +db_tools: + connections: + connection_one: + backup_excluded_tables: ['table1', 'table2'] + connection_two: + backup_excluded_tables: ['table3', 'table4'] +``` +
+
+ +Here is an example for exclude `table1` and `table2` for all connections: + +```yml +# db_tools.yaml +backup_excluded_tables: ['table1', 'table2'] +``` + +Or set a specific table list for each connection: + +```yml +# db_tools.yaml +connections: + connection_one: + backup_excluded_tables: ['table1', 'table2'] + connection_two: + backup_excluded_tables: ['table3', 'table4'] +``` +
+ +:::tip +Note that you can override this configuration while running the `db-tools:backup` command using +the `--exclude` option. +::: + +### Binary options + +See the [default binary options](#default-binary-options) section. + +### Backup expiration age + +The `backup_expiration_age` parameter let you choose when a backup is considered as obsolete. + +Default value is `'3 months ago'`. + +Use [PHP relative date/time formats](https://www.php.net/manual/en/datetime.formats.relative.php) +for this value. + +
+ +Here is an example that sets 1 week lifetime for backups for all connections: + +```yml +# config/packages/db_tools.yaml +db_tools: + backup_expiration_age: '1 week ago' +``` + +Or set a specific value list for each connection: + +```yml +# config/packages/db_tools.yaml +db_tools: + connections: + connection_one: + backup_expiration_age: '1 week ago' + connection_two: + backup_expiration_age: '3 days ago' +``` +
+
+ +Here is an example that sets 1 week lifetime for backups for all connections: + +```yml +# db_tools.yaml +backup_expiration_age: '1 week ago' +``` + +Or set a specific value list for each connection: + +```yml +# db_tools.yaml +connections: + connection_one: + backup_expiration_age: '1 week ago' + connection_two: + backup_expiration_age: '3 days ago' +``` +
+ +### Backup and restore timeout + +The `backup_timeout` and `restore_timeout` options let you choose what is the backup and restore +processes timeout in seconds. + +Default value is `600` (seconds) for backup, `1800` (seconds) for restore. + +Value can be either a [\DateInterval::createFromDateString()](https://www.php.net/manual/en/dateinterval.createfromdatestring.php) +compatible string value or a number of seconds as an integer value. + +
+ +Here is an example that sets timeouts for all connection: + +```yml +# config/packages/db_tools.yaml +db_tools: + # As a date interval string. + backup_timeout: '6 minutes 30 seconds' + restore_timeout: '3 minutes 15 seconds' + + # As a number of seconds integer value. + backup_timeout: 390 + restore_timeout: 195 +``` + +Or set a different timeout for each connection: + +```yml +# config/packages/db_tools.yaml +db_tools: + connections: + connection_one: + backup_timeout: '6 minutes 30 seconds' + restore_timeout: '3 minutes 15 seconds' + connection_two: + backup_timeout: 390 + restore_timeout: 195 +``` +
+
+ +Here is an example that sets timeouts for all connection: + +```yml +# db_tools.yaml + +# As a date interval string. +backup_timeout: '6 minutes 30 seconds' +restore_timeout: '3 minutes 15 seconds' + +# As a number of seconds integer value. +backup_timeout: 390 +restore_timeout: 195 +``` + +Or set a different timeout for each connection: + +```yml +# db_tools.yaml +connections: + connection_one: + backup_timeout: '6 minutes 30 seconds' + restore_timeout: '3 minutes 15 seconds' + connection_two: + backup_timeout: 390 + restore_timeout: 195 +``` +
+ +## Binaries + +`db-tools:backup` and `db-tools:restore` need your system/environment to provide some extra binaries +to be able to work. These binaries depend on the database vendor you use, you will need: +* for MariaDB: `mariadb-dump` and `mariadb` +* for MySQL: `mysqldump` and `mysql` +* for PostgreSQL: `pg_dump` and `pg_restore` +* for SQLite: `sqlite3` + +You can verify if those binaries are well found by *DbToolsBundle*, +for each of your connections, by launching: + +
+ +```sh +php bin/console db-tools:check +``` +
+
+ +```sh +php vendor/bin/db-tools database:check +``` +
+ +If the `db-tools:check` command returns you some errors: + * if your binaries are present on your system but *DbToolsBundle* can't find them you will need + to specify path for these binaries: + +
+ + ```yml + # config/packages/db_tools.yaml + db_tools: + backup_binary: '/usr/local/bin/pg_dump' + restore_binary: '/usr/local/bin/pg_restore' + ``` +
+
+ + ```yml + # db_tools.yaml + backup_binary: '/usr/local/bin/pg_dump' + restore_binary: '/usr/local/bin/pg_restore' + ``` +
+ + * Backup and restore binaries, as well as command line arguments and options are + configured on a per-connection basis. If you have more than one connection, + use the following syntax instead: + +
+ + ```yml + # config/packages/db_tools.yaml + db_tools: + connections: + connection_one: + backup_binary: '/usr/local/bin/pg_dump' + restore_binary: '/usr/local/bin/pg_restore' + connection_two: + backup_binary: '/usr/local/bin/mysqldump' + restore_binary: '/usr/local/bin/mysql' + ``` +
+
+ + ```yml + # db_tools.yaml + connections: + connection_one: + backup_binary: '/usr/local/bin/pg_dump' + restore_binary: '/usr/local/bin/pg_restore' + connection_two: + backup_binary: '/usr/local/bin/mysqldump' + restore_binary: '/usr/local/bin/mysql' + ``` +
+ + * Or, if your binaries are not present on your system: you will need to install them. + +:::tip +If your app lives in the [official PHP docker image](https://hub.docker.com/_/php/), +you can install correct binaries adding these lines to your Dockerfile, + +for PostgreSQL: + +``` +RUN apt-get update && \ + apt-get install -y --no-install-recommends postgresql-client +``` + +for MariaDB/MySQL: + +``` +RUN apt-get update && \ + apt-get install -y --no-install-recommends default-mysql-client +``` +::: + +:::warning +Dump and restore is not supported yet for SQL Server. +::: + +### Default binary options + +Apart from the essential options (credentials, database name, etc.), the library +also passes a few default options to the binary depending on the operation being +performed and the invoked binary itself. You can customize those default options +by configuring your own ones per operation type and connection: + +
+ +Here is an example that sets options for all connections: + +```yml +# config/packages/db_tools.yaml +db_tools: + backup_options: '--an-option' + restore_options: '--a-first-one --a-second-one' +``` + +Or set a specific value list for each connection: + +```yml +# config/packages/db_tools.yaml +db_tools: + connections: + connection_one: + backup_options: '--an-option' + restor_options: '-xyz --another' + connection_two: + backup_options: '--a-first-one --a-second-one' + restor_options: '-O sample-value' +``` +
+
+ +Here is an example that sets options for all connections: + +```yml +# db_tools.yaml +backup_options: '--an-option' +restore_options: '--a-first-one --a-second-one' +``` + +Or set a specific value list for each connection: + +```yml +# db_tools.yaml +connections: + connection_one: + backup_options: '--an-option' + restor_options: '-xyz --another' + connection_two: + backup_options: '--a-first-one --a-second-one' + restor_options: '-O sample-value' +``` +
+ +If you do not define your own default options, the following ones will be used +according to the database vendor: + +* When backing up: + * MariaDB: `--no-tablespaces` + * MySQL: `--no-tablespaces` + * PostgreSQL: `-Z 5 --lock-wait-timeout=120` + * SQLite: `-bail` +* When restoring: + * MariaDB: None + * MySQL: None + * PostgreSQL: `-j 2 --clean --if-exists --disable-triggers` + * SQLite: None + +## Anonymizer paths + +
+ +By default, *DbToolsBundle* will look for custom *anonymizers* in 2 directories: + +* `%kernel.project_dir%/vendor/makinacorpus/db-tools-bundle/src/Anonymizer` +* `%kernel.project_dir%/src/Anonymizer` + +If you want to put custom anonymizers in another directory or if you want to load +a pack of anonymizers from an external library, you can modify/add paths: + + +```yml +# config/packages/db_tools.yaml +db_tools: + anonymizer_paths: + - '%kernel.project_dir%/src/Anonymizer' + - '%kernel.project_dir%/vendor/myAnonymizerProvider/anonymizers/src' + # ... +``` +
+
+ +By default, *DbToolsBundle* when used as a standalone CLI tool will not lookup for +custom *anonymizers*. + +If you want to write custom anonymizers, in order for them to be found, you must specify +paths where the source code lies: + + +```yml +# db_tools.yaml +db_tools: + anonymizer_paths: + - './src/Anonymizer' + - './vendor/myAnonymizerProvider/anonymizers/src' + # ... +``` +
+ +:::tip +Core provided anonymizers and anonymizers that are in packs installed using composer +will always be looked-up. +::: + +:::warning +Packs must be installed using composer: *DbToolsBundle* uses composer generated metadata +about installed packages to find them. +::: + +## Anonymization + +
+ +Per default, **DbToolsBundle** will only look for anonymization configurations from PHP attributes on Doctrine Entities. + +But *DbToolsBundle* does not necessary need Doctrine ORM to anonymize your data, it can do it just with a DBAL connection. +In this case (or if you prefer YAML over attributes): you can configure *DbToolsBundle* to look for anonymization +configurations in a YAML file: + +```yml +# config/packages/db_tools.yaml +db_tools: + + # When you have a single connection, and a single file: + anonymization_files: '%kernel.project_dir%/config/anonymizations.yaml' + + # Or with multiple connections: + anonymization_files: + connection_one: '%kernel.project_dir%/config/anonymizations/connection_one.yaml' + connection_two: '%kernel.project_dir%/config/anonymizations/connection_two.yaml' + + # Each connection may have multiple files: + anonymization_files: + connection_one: + - '%kernel.project_dir%/config/anonymizations/connection_one_1.yaml' + - '%kernel.project_dir%/config/anonymizations/connection_one_2.yaml' + # ... +``` + +
+
+ +You need to register your anonymization configuration for the anonymization feature to work: + +```yml +# config/packages/db_tools.yaml +db_tools: + + # When you have a single connection, and a single file: + anonymization_files: './anonymizations.yaml' + + # Or with multiple connections: + anonymization_files: + connection_one: './anonymizations/connection_one.yaml' + connection_two: './anonymizations/connection_two.yaml' + + # Each connection may have multiple files: + anonymization_files: + connection_one: + - './config/anonymizations/connection_one_1.yaml' + - './config/anonymizations/connection_one_2.yaml' + # ... +``` + +
+ +:::tip +For more information about anonymization and configuration file structure, refer to the [Anonymization section](./anonymization/essentials). +::: diff --git a/docs/content/configuration/reference.md b/docs/content/configuration/reference.md index a4a3fdd..05ebcb2 100644 --- a/docs/content/configuration/reference.md +++ b/docs/content/configuration/reference.md @@ -30,16 +30,16 @@ within. [`anonymization.tables` (standalone)](#anonymization-tables) | [`anonymization.yaml`](#anonymization-yaml) | [`anonymizer_paths`](#anonymizer-paths) | +[`backup_binary`](#backup-binary) | [`backup_expiration_age`](#backup-expiration-age) | +[`backup_options`](#backup-options) | [`backup_timeout`](#backup-timeout) | -[`backupper_binaries`](#backupper-binaries) | -[`backupper_options`](#backupper-options) | [`connections` (standalone)](#connections) | [`default_connection` (standalone)](#default-connection) | [`excluded_tables`](#excluded-tables) | +[`restore_binary`](#restore-binary) | +[`restore_options`](#restore-options) | [`restore_timeout`](#restore-timeout) | -[`restorer_binaries`](#restorer-binaries) | -[`restorer_options`](#restorer-options) | [`storage.filename_strategy`](#storage-filename-strategy) | [`workdir` (standalone)](#workdir) @@ -177,7 +177,7 @@ excluded_tables: ::: -### `backupper_binaries` +### `backup_binary` Path to backup command in filesystem. @@ -188,7 +188,7 @@ work in most Linux distributions. :::code-group ```yaml [Symfony] db_tools: - backupper_binaries: + backup_binary: mariadb: /usr/bin/mariadb-dump mysql: /usr/bin/mysqldump postgresql: /usr/bin/pg_dump @@ -196,7 +196,7 @@ db_tools: ``` ```yaml [Standalone] -backupper_binaries: +backup_binary: mariadb: /usr/bin/mariadb-dump mysql: /usr/bin/mysqldump postgresql: /usr/bin/pg_dump @@ -204,7 +204,7 @@ backupper_binaries: ``` ::: -### `restorer_binaries` +### `restore_binary` Path to restore command in filesystem. @@ -214,7 +214,7 @@ work in most Linux distributions. :::code-group ```yaml [Symfony] db_tools: - restorer_binaries: + restore_binary: mariadb: /usr/bin/mariadb mysql: /usr/bin/mysql postgresql: /usr/bin/pg_restore @@ -222,7 +222,7 @@ db_tools: ``` ```yaml [Standalone] -restorer_binaries: +restore_binary: mariadb: /usr/bin/mariadb mysql: /usr/bin/mysql postgresql: /usr/bin/pg_restore @@ -230,7 +230,7 @@ restorer_binaries: ``` ::: -### `backupper_options` +### `backup_options` Allows you to add specific command line options to the backup command, one for each connection. @@ -246,13 +246,13 @@ By specifying options, the default ones will be dropped. :::code-group ```yaml [Symfony] db_tools: - backupper_options: + backup_options: connection_one: '-Z 5 --lock-wait-timeout=120' connection_two: '--no-tablespaces' ``` ```yaml [Standalone] -backupper_options: +backup_options: connection_one: '-Z 5 --lock-wait-timeout=120' connection_two: '--no-tablespaces' ``` @@ -287,7 +287,7 @@ restore_timeout: 67 ``` ::: -### `restorer_options` +### `restore_options` Allows you to add specific command line options to the restore command, one for each connection. @@ -301,13 +301,13 @@ invoking the command, the following ones will be used according to the database :::code-group ```yaml [Symfony] db_tools: - backupper_options: + restore_options: connection_one: '-j 2 --clean --if-exists --disable-triggers' connection_two: '--no-tablespaces' ``` ```yaml [Standalone] -backupper_options: +restore_options: connection_one: '-j 2 --clean --if-exists --disable-triggers' connection_two: '--some-other-option ``` diff --git a/docs/content/contribute/pack.md b/docs/content/contribute/pack.md index ab3921a..1c9978b 100644 --- a/docs/content/contribute/pack.md +++ b/docs/content/contribute/pack.md @@ -20,7 +20,7 @@ First, you will need to adapt the provided `composer.json`: { "name": "db-tools-bundle/pack-template",// [!code --] "name": "my-vendor/pack-awesome",// [!code ++] - "description": "An example pack of anonymizers for the DbToolsBundle",// [!code --] + "description": "An example pack of anonymizers for DbToolsBundle",// [!code --] "description": "An awesome pack for anonymizing many things!",// [!code ++] "type": "db-tools-bundle-pack", "license": "MIT", @@ -130,7 +130,7 @@ Learn more about how to develop them reading the [Custom Anonymizers section](.. ## 3. Test your anonymizers After you built your anonymizers, don't forget to test them. We recommend doing at least one functionnal test per anonymizer. -To inspire you doing these tests, read [existing tests in the DbToolsBundle](https://github.com/makinacorpus/DbToolsBundle/tree/main/tests/Functional/Anonymizer/Core) +To inspire you doing these tests, read [existing tests in DbToolsBundle](https://github.com/makinacorpus/DbToolsBundle/tree/main/tests/Functional/Anonymizer/Core) or in official packs. To help you launchning these tests, use provided `dev.sh` script, see [Development guide section](./guide) to learn how to use it. diff --git a/docs/content/getting-started/basics.md b/docs/content/getting-started/basics.md index 81d3774..f8b9d6f 100644 --- a/docs/content/getting-started/basics.md +++ b/docs/content/getting-started/basics.md @@ -97,7 +97,8 @@ anonymization:
-With the DbToolsBundle, by adding some PHP attributes on your Doctrine Entities, + +With *DbToolsBundle*, by adding some PHP attributes on your Doctrine Entities, you can easily configure a complete anonymization for your sensitive data. ::: info @@ -167,7 +168,7 @@ to download and restore it on your local environment without any security concer We know that a slow anonymization process can be real pain. That's why a meticulous work has been carried out to make this operation as quick as possible. -Thanks to this work, the DbToolsBundle can now **anonymize 1 million rows in less than 20s**! +Thanks to this work, *DbToolsBundle* can now **anonymize 1 million rows in less than 20s**! Learn more about performance in the [dedicated section](../anonymization/performance). ::: diff --git a/docs/content/getting-started/flavors.md b/docs/content/getting-started/flavors.md index 193fb1f..23811fb 100644 --- a/docs/content/getting-started/flavors.md +++ b/docs/content/getting-started/flavors.md @@ -31,7 +31,7 @@ to find the database(s) you want to manage. be fully integrated into any Symfony project via its dedicated bridge. After you installed the bundle: -* All the *DbToolsBundle* commands will be accessible with +* All *DbToolsBundle* commands will be accessible with the [Symfony Console](https://symfony.com/doc/current/components/console.html). * Database connection URL will be autoconfigured based on available DBAL connections. * *DbToolsBundle* can be setup through its bundle configuration (`config/packages/db_tools.yaml`). diff --git a/docs/content/getting-started/installation.md b/docs/content/getting-started/installation.md index 1135457..f57fb9e 100644 --- a/docs/content/getting-started/installation.md +++ b/docs/content/getting-started/installation.md @@ -24,7 +24,7 @@ you should not be lost if you are a regular Symfony developer. - PHP 8.1 or higher - Symfony 6.0 or higher -- Doctrine/DBAL, the DbToolsBundle takes advantage of available DBAL connections +- Doctrine/DBAL, *DbToolsBundle* takes advantage of available DBAL connections