From 25806af91d1a58e67aec7a60adce5c7c31a40e4d Mon Sep 17 00:00:00 2001 From: Yonas Habteab Date: Wed, 19 Jul 2023 13:03:22 +0200 Subject: [PATCH 1/5] Provide migration hook & migrate jobs config to database --- library/X509/Model/Schema.php | 39 +++++++++ library/X509/ProvidedHook/DbMigration.php | 98 +++++++++++++++++++++++ run.php | 4 + schema/mysql-upgrades/1.3.0.sql | 30 ++++--- schema/mysql.schema.sql | 14 ++++ schema/pgsql-upgrades/1.3.0.sql | 30 ++++--- schema/pgsql.schema.sql | 14 ++++ 7 files changed, 211 insertions(+), 18 deletions(-) create mode 100644 library/X509/Model/Schema.php create mode 100644 library/X509/ProvidedHook/DbMigration.php diff --git a/library/X509/Model/Schema.php b/library/X509/Model/Schema.php new file mode 100644 index 00000000..c505e543 --- /dev/null +++ b/library/X509/Model/Schema.php @@ -0,0 +1,39 @@ +add(new BoolCast(['success'])); + $behaviors->add(new MillisecondTimestamp(['timestamp'])); + } +} diff --git a/library/X509/ProvidedHook/DbMigration.php b/library/X509/ProvidedHook/DbMigration.php new file mode 100644 index 00000000..cc422811 --- /dev/null +++ b/library/X509/ProvidedHook/DbMigration.php @@ -0,0 +1,98 @@ +translate('Icinga Certificate Monitoring'); + } + + public function providedDescriptions(): array + { + return [ + '1.0.0' => $this->translate( + 'Adjusts the database type of several columns and changes some composed primary keys.' + ), + '1.1.0' => $this->translate( + 'Changes the composed x509_target index and x509_certificate valid from/to types to bigint.' + ), + '1.2.0' => $this->translate( + 'Changes all timestamp columns to bigint and adjusts enum types of "yes/no" to "n/y".' + ), + '1.3.0' => $this->translate( + 'Introduces the required tables to store jobs and job schedules in the database.' + ) + ]; + } + + public function getVersion(): string + { + if ($this->version === null) { + $conn = $this->getDb(); + $schema = $this->getSchemaQuery() + ->columns(['version', 'success']) + ->orderBy('id', SORT_DESC) + ->limit(2); + + if (static::tableExists($conn, $schema->getModel()->getTableName())) { + /** @var Schema $version */ + foreach ($schema as $version) { + if ($version->success) { + $this->version = $version->version; + + break; + } + } + + if (! $this->version) { + // Schema version table exist, but the user has probably deleted the entry! + $this->version = '1.3.0'; + } + } elseif ( + $this->getDb()->getAdapter() instanceof Pgsql + || static::getColumnType($conn, 'x509_certificate', 'ctime') === 'bigint(20) unsigned' + ) { + // We modified a bunch of timestamp columns to bigint in x509 version 1.2.0. + // We have also added Postgres support with x509 version 1.2 and never had an upgrade scripts until now. + $this->version = '1.2.0'; + } elseif (static::getColumnType($conn, 'x509_certificate_subject_alt_name', 'hash') !== null) { + // We know for sure that x509 version 1.0 has been applied, though not whether x509 version 1.1.0 + // did too. Therefore, we have modified the 1.1.0 upgrade script to run multiple times without any + // errors so that we can use 1.0 as the last (migrated) version. + $this->version = '1.0.0'; + } else { + // X509 version 1.0 was the first release of this module, but due to some reason it also contains + // an upgrade script and adds `hash` column. However, if this column doesn't exist yet, we need + // to use the lowest possible release value as the initial (last migrated) version. + $this->version = '0.0.0'; + } + } + + return $this->version; + } + + public function getDb(): Sql\Connection + { + return $this->getX509Db(); + } + + protected function getSchemaQuery(): Query + { + return Schema::on($this->getDb()); + } +} diff --git a/run.php b/run.php index 3c041a2c..b1977a9e 100644 --- a/run.php +++ b/run.php @@ -2,5 +2,9 @@ // Icinga Web 2 X.509 Module | (c) 2018 Icinga GmbH | GPLv2 +/** @var \Icinga\Application\Modules\Module $this */ + +$this->provideHook('DbMigration', '\\Icinga\\Module\\X509\\ProvidedHook\\DbMigration'); + $this->provideHook('director/ImportSource', '\\Icinga\\Module\\X509\\ProvidedHook\\HostsImportSource'); $this->provideHook('director/ImportSource', '\\Icinga\\Module\\X509\\ProvidedHook\\ServicesImportSource'); diff --git a/schema/mysql-upgrades/1.3.0.sql b/schema/mysql-upgrades/1.3.0.sql index c886b0bd..f31e8bd2 100644 --- a/schema/mysql-upgrades/1.3.0.sql +++ b/schema/mysql-upgrades/1.3.0.sql @@ -1,4 +1,4 @@ -CREATE TABLE IF NOT EXISTS x509_job ( +CREATE TABLE x509_job ( id int(10) unsigned NOT NULL AUTO_INCREMENT, name varchar(255) NOT NULL COLLATE utf8mb4_unicode_ci, author varchar(255) NOT NULL COLLATE utf8mb4_unicode_ci, @@ -12,7 +12,7 @@ CREATE TABLE IF NOT EXISTS x509_job ( UNIQUE (name) ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; -CREATE TABLE IF NOT EXISTS x509_schedule ( +CREATE TABLE x509_schedule ( id int(10) unsigned NOT NULL AUTO_INCREMENT, job_id int(10) unsigned NOT NULL, name varchar(255) NOT NULL COLLATE utf8mb4_unicode_ci, @@ -27,13 +27,25 @@ CREATE TABLE IF NOT EXISTS x509_schedule ( DELETE FROM x509_job_run; ALTER TABLE x509_job_run - ADD COLUMN IF NOT EXISTS job_id int(10) unsigned NOT NULL AFTER id, - ADD COLUMN IF NOT EXISTS schedule_id int(10) unsigned DEFAULT NULL AFTER job_id, - DROP COLUMN IF EXISTS `name`, - DROP COLUMN IF EXISTS ctime, - DROP COLUMN IF EXISTS mtime, - DROP CONSTRAINT IF EXISTS fk_x509_job_run_job, - DROP CONSTRAINT IF EXISTS fk_x509_job_run_schedule; + ADD COLUMN job_id int(10) unsigned NOT NULL AFTER id, + ADD COLUMN schedule_id int(10) unsigned DEFAULT NULL AFTER job_id, + DROP COLUMN `name`, + DROP COLUMN ctime, + DROP COLUMN mtime; ALTER TABLE x509_job_run ADD CONSTRAINT fk_x509_job_run_job FOREIGN KEY (job_id) REFERENCES x509_job (id) ON DELETE CASCADE, ADD CONSTRAINT fk_x509_job_run_schedule FOREIGN KEY (schedule_id) REFERENCES x509_schedule (id) ON DELETE CASCADE; + +CREATE TABLE x509_schema ( + id int unsigned NOT NULL AUTO_INCREMENT, + version varchar(64) NOT NULL, + timestamp bigint unsigned NOT NULL, + success enum ('n', 'y') DEFAULT NULL, + reason text DEFAULT NULL, + + PRIMARY KEY (id), + CONSTRAINT idx_x509_schema_version UNIQUE (version) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin ROW_FORMAT=DYNAMIC; + +INSERT INTO x509_schema (version, timestamp, success, reason) + VALUES ('1.3.0', UNIX_TIMESTAMP() * 1000, 'y', NULL); diff --git a/schema/mysql.schema.sql b/schema/mysql.schema.sql index 8892b357..7e56746f 100644 --- a/schema/mysql.schema.sql +++ b/schema/mysql.schema.sql @@ -120,3 +120,17 @@ CREATE TABLE x509_job_run ( CONSTRAINT fk_x509_job_run_job FOREIGN KEY (job_id) REFERENCES x509_job (id) ON DELETE CASCADE, CONSTRAINT fk_x509_job_run_schedule FOREIGN KEY (schedule_id) REFERENCES x509_schedule (id) ON DELETE CASCADE ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; + +CREATE TABLE x509_schema ( + id int unsigned NOT NULL AUTO_INCREMENT, + version varchar(64) NOT NULL, + timestamp bigint unsigned NOT NULL, + success enum ('n', 'y') DEFAULT NULL, + reason text DEFAULT NULL, + + PRIMARY KEY (id), + CONSTRAINT idx_x509_schema_version UNIQUE (version) +) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin ROW_FORMAT=DYNAMIC; + +INSERT INTO x509_schema (version, timestamp, success) + VALUES ('1.3.0', UNIX_TIMESTAMP() * 1000, 'y'); diff --git a/schema/pgsql-upgrades/1.3.0.sql b/schema/pgsql-upgrades/1.3.0.sql index be8fcfaa..7e1f43a6 100644 --- a/schema/pgsql-upgrades/1.3.0.sql +++ b/schema/pgsql-upgrades/1.3.0.sql @@ -1,4 +1,4 @@ -CREATE TABLE IF NOT EXISTS x509_job ( +CREATE TABLE x509_job ( id serial PRIMARY KEY, name varchar(255) NOT NULL, author varchar(255) NOT NULL, @@ -11,7 +11,7 @@ CREATE TABLE IF NOT EXISTS x509_job ( UNIQUE (name) ); -CREATE TABLE IF NOT EXISTS x509_schedule ( +CREATE TABLE x509_schedule ( id serial PRIMARY KEY, job_id int NOT NULL, name varchar(255) NOT NULL, @@ -25,13 +25,25 @@ CREATE TABLE IF NOT EXISTS x509_schedule ( DELETE FROM x509_job_run; ALTER TABLE x509_job_run - ADD COLUMN IF NOT EXISTS job_id int NOT NULL, - ADD COLUMN IF NOT EXISTS schedule_id int DEFAULT NULL, - DROP COLUMN IF EXISTS name, - DROP COLUMN IF EXISTS ctime, - DROP COLUMN IF EXISTS mtime, - DROP CONSTRAINT IF EXISTS fk_x509_job_run_job, - DROP CONSTRAINT IF EXISTS fk_x509_job_run_schedule; + ADD COLUMN job_id int NOT NULL, + ADD COLUMN schedule_id int DEFAULT NULL, + DROP COLUMN name, + DROP COLUMN ctime, + DROP COLUMN mtime; ALTER TABLE x509_job_run ADD CONSTRAINT fk_x509_job_run_job FOREIGN KEY (job_id) REFERENCES x509_job (id) ON DELETE CASCADE, ADD CONSTRAINT fk_x509_job_run_schedule FOREIGN KEY (schedule_id) REFERENCES x509_schedule (id) ON DELETE CASCADE; + +CREATE TABLE x509_schema ( + id serial, + version varchar(64) NOT NULL, + timestamp bigint NOT NULL, + success boolenum DEFAULT NULL, + reason text DEFAULT NULL, + + CONSTRAINT pk_x509_schema PRIMARY KEY (id), + CONSTRAINT idx_x509_schema_version UNIQUE (version) +); + +INSERT INTO x509_schema (version, timestamp, success, reason) + VALUES ('1.3.0', UNIX_TIMESTAMP() * 1000, 'y', NULL); diff --git a/schema/pgsql.schema.sql b/schema/pgsql.schema.sql index cfaa3793..1d93ef3d 100644 --- a/schema/pgsql.schema.sql +++ b/schema/pgsql.schema.sql @@ -146,3 +146,17 @@ CREATE TABLE x509_job_run ( CONSTRAINT fk_x509_job_run_job FOREIGN KEY (job_id) REFERENCES x509_job (id) ON DELETE CASCADE, CONSTRAINT fk_x509_job_run_schedule FOREIGN KEY (schedule_id) REFERENCES x509_schedule (id) ON DELETE CASCADE ); + +CREATE TABLE x509_schema ( + id serial, + version varchar(64) NOT NULL, + timestamp bigint NOT NULL, + success boolenum DEFAULT NULL, + reason text DEFAULT NULL, + + CONSTRAINT pk_x509_schema PRIMARY KEY (id), + CONSTRAINT idx_x509_schema_version UNIQUE (version) +); + +INSERT INTO x509_schema (version, timestamp, success) + VALUES ('1.3.0', UNIX_TIMESTAMP() * 1000, 'y'); From ef3e4beaa0a0b2cb2da511d1f05ffc9f0058ec3a Mon Sep 17 00:00:00 2001 From: Yonas Habteab Date: Wed, 19 Jul 2023 16:53:50 +0200 Subject: [PATCH 2/5] Introduce `MigrateCommand` --- application/clicommands/MigrateCommand.php | 120 +++++++++++++++++++++ 1 file changed, 120 insertions(+) create mode 100644 application/clicommands/MigrateCommand.php diff --git a/application/clicommands/MigrateCommand.php b/application/clicommands/MigrateCommand.php new file mode 100644 index 00000000..5b947360 --- /dev/null +++ b/application/clicommands/MigrateCommand.php @@ -0,0 +1,120 @@ + + * + * OPTIONS + * + * --author= + * An Icinga Web 2 user used to mark as an author for all the migrated jobs. + */ + public function jobsAction(): void + { + /** @var string $author */ + $author = $this->params->getRequired('author'); + /** @var User $user */ + $user = Auth::getInstance()->getUser(); + $user->setUsername($author); + + $this->migrateJobs(); + + Logger::info('Successfully applied all pending migrations'); + } + + protected function migrateJobs(): void + { + $repo = new class () extends IniRepository { + /** @var array> */ + protected $queryColumns = [ + 'jobs' => ['name', 'cidrs', 'ports', 'exclude_targets', 'schedule', 'frequencyType'] + ]; + + /** @var array> */ + protected $configs = [ + 'jobs' => [ + 'module' => 'x509', + 'name' => 'jobs', + 'keyColumn' => 'name' + ] + ]; + }; + + $conn = $this->getDb(); + $conn->transaction(function (Connection $conn) use ($repo) { + /** @var User $user */ + $user = Auth::getInstance()->getUser(); + /** @var stdClass $data */ + foreach ($repo->select() as $data) { + $config = []; + if (! isset($data->frequencyType) && ! empty($data->schedule)) { + $frequency = new Cron($data->schedule); + $config = [ + 'type' => get_php_type($frequency), + 'frequency' => Json::encode($frequency) + ]; + } elseif (! empty($data->schedule)) { + $config = [ + 'type' => $data->frequencyType, + 'frequency' => $data->schedule // Is already json encoded + ]; + } + + $excludes = $data->exclude_targets; + if (empty($excludes)) { + $excludes = new Expression('NULL'); + } + + $conn->insert('x509_job', [ + 'name' => $data->name, + 'author' => $user->getUsername(), + 'cidrs' => $data->cidrs, + 'ports' => $data->ports, + 'exclude_targets' => $excludes, + 'ctime' => (new DateTime())->getTimestamp() * 1000, + 'mtime' => (new DateTime())->getTimestamp() * 1000 + ]); + + $jobId = (int) $conn->lastInsertId(); + if (! empty($config)) { + $config['rescan'] = 'n'; + $config['full_scan'] = 'n'; + $config['since_last_scan'] = Job::DEFAULT_SINCE_LAST_SCAN; + + $conn->insert('x509_schedule', [ + 'job_id' => $jobId, + 'name' => $data->name . ' Schedule', + 'author' => $user->getUsername(), + 'config' => Json::encode($config), + 'ctime' => (new DateTime())->getTimestamp() * 1000, + 'mtime' => (new DateTime())->getTimestamp() * 1000, + ]); + } + } + }); + } +} From 1cfeb8d38b6ed586e31c44cbefd1be96b3109750 Mon Sep 17 00:00:00 2001 From: Yonas Habteab Date: Wed, 19 Jul 2023 17:15:25 +0200 Subject: [PATCH 3/5] Add `1.3.0` upgrading docs --- doc/80-Upgrading.md | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/doc/80-Upgrading.md b/doc/80-Upgrading.md index 511f67c6..a66ddc46 100644 --- a/doc/80-Upgrading.md +++ b/doc/80-Upgrading.md @@ -3,6 +3,42 @@ Upgrading Icinga Certificate Monitoring is straightforward. Usually the only manual steps involved are schema updates for the database. +## Upgrading to version 1.3.0 + +Icinga Certificate Monitoring version `1.3.0` requires a schema update for the database. We have dropped the use of **INI** +files to store jobs and are using the database instead. So you need to migrate your job configs to the database. + +If you're already using Icinga Web 2 version `>= 2.12`, then you don't need to import the sql upgrade scripts manually. +Icinga Web provides you the ability to perform such migrations in a simple way. You may be familiar with such an automation +if you're an Icinga Director user. + +> **Note** +> +> Please note that it doesn't matter if you import the database upgrade script manually or via the new automation, +> you will have to migrate your [Jobs config](#migrate-jobs) from INI to the database manually afterwards. + +Before migrating your jobs from **INI** to the database, you need to first apply the migration script. This will create +the tables needed to store the jobs and schedules in the database. + +You may use the following command to apply the database schema upgrade file: + + +**Note:** If you haven't installed this module from packages, then please adapt the schema path to the correct installation path. + + +```sql +# mysql -u root -p x509 < /usr/share/icingaweb2/modules/x509/schema/mysql-upgrades/1.3.0.sql +``` + +### Migrate Jobs + +Afterwards, you can safely migrate your jobs with the following command. Keep in mind that you need to specify an +Icinga Web username that will be used as the author of these jobs in the database. + +``` +# icingacli x509 migrate jobs --author "icingaadmin" +``` + ## Upgrading to version 1.2.0 Icinga Certificate Monitoring version 1.2.0 requires a schema update for the database. We have changed all `timestamp` From 88e1d17f30ed50e92040c3df55bb36740f3e102f Mon Sep 17 00:00:00 2001 From: Yonas Habteab Date: Mon, 18 Sep 2023 10:02:53 +0200 Subject: [PATCH 4/5] DbMigration: Determine not applied version `1.0.0` & `1.1.0` correctly --- library/X509/ProvidedHook/DbMigration.php | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/library/X509/ProvidedHook/DbMigration.php b/library/X509/ProvidedHook/DbMigration.php index cc422811..03a26ba1 100644 --- a/library/X509/ProvidedHook/DbMigration.php +++ b/library/X509/ProvidedHook/DbMigration.php @@ -71,10 +71,11 @@ public function getVersion(): string // We have also added Postgres support with x509 version 1.2 and never had an upgrade scripts until now. $this->version = '1.2.0'; } elseif (static::getColumnType($conn, 'x509_certificate_subject_alt_name', 'hash') !== null) { - // We know for sure that x509 version 1.0 has been applied, though not whether x509 version 1.1.0 - // did too. Therefore, we have modified the 1.1.0 upgrade script to run multiple times without any - // errors so that we can use 1.0 as the last (migrated) version. - $this->version = '1.0.0'; + if (static::getColumnType($conn, 'x509_certificate', 'valid_from') === 'bigint(20) unsigned') { + $this->version = '1.0.0'; + } else { + $this->version = '1.1.0'; + } } else { // X509 version 1.0 was the first release of this module, but due to some reason it also contains // an upgrade script and adds `hash` column. However, if this column doesn't exist yet, we need From 1ca678bbeca530b756a559d6975def6bf2750d8f Mon Sep 17 00:00:00 2001 From: Yonas Habteab Date: Tue, 19 Sep 2023 15:18:58 +0200 Subject: [PATCH 5/5] Schema: Add class property type hints --- library/X509/Model/Schema.php | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/library/X509/Model/Schema.php b/library/X509/Model/Schema.php index c505e543..02ec0c08 100644 --- a/library/X509/Model/Schema.php +++ b/library/X509/Model/Schema.php @@ -4,11 +4,21 @@ namespace Icinga\Module\X509\Model; +use DateTime; use ipl\Orm\Behavior\BoolCast; use ipl\Orm\Behavior\MillisecondTimestamp; use ipl\Orm\Behaviors; use ipl\Orm\Model; +/** + * A database model for x509 schema version table + * + * @property int $id Unique identifier of the database schema entries + * @property string $version The current schema version of Icinga Web + * @property DateTime $timestamp The insert/modify time of the schema entry + * @property bool $success Whether the database migration of the current version was successful + * @property ?string $reason The reason why the database migration has failed + */ class Schema extends Model { public function getTableName(): string