diff --git a/.env b/.env index 41a098719..b6597fc5d 100644 --- a/.env +++ b/.env @@ -22,6 +22,7 @@ APP_EUDONET_PARIS_BASE_URL=https://eudonet-partage.apps.paris.fr APP_BAC_IDF_DECREES_FILE=data/bac_idf/decrees.json APP_BAC_IDF_CITIES_FILE=data/bac_idf/cities.csv DATABASE_URL="postgresql://dialog:dialog@database:5432/dialog" +METABASE_DATABASE_URL="postgresql://dialog:dialog@database:5432/dialog" REDIS_URL="redis://redis:6379" API_ADRESSE_BASE_URL=https://api-adresse.data.gouv.fr APP_IGN_GEOCODER_BASE_URL=https://data.geopf.fr diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7da4e5736..25189801e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -61,6 +61,7 @@ jobs: run: | echo "DATABASE_URL=postgresql://dialog:dialog@localhost:5432/dialog" >> .env echo "BDTOPO_DATABASE_URL=${{ secrets.BDTOPO_DATABASE_URL }}" >> .env + echo "METABASE_DATABASE_URL=postgresql://dialog:dialog@localhost:5432/dialog" >> .env echo "REDIS_URL=redis://localhost:6379" >> .env echo "APP_STORAGE_SOURCE=memory.storage" >> .env diff --git a/.github/workflows/metabase_export.yml b/.github/workflows/metabase_export.yml index 98380a560..a17681b47 100644 --- a/.github/workflows/metabase_export.yml +++ b/.github/workflows/metabase_export.yml @@ -26,10 +26,10 @@ jobs: run: | ssh-keyscan -H ssh.osc-fr1.scalingo.com >> ~/.ssh/known_hosts + - name: Init CI environment variables + run: | + echo "DATABASE_URL=${{ secrets.METABASE_EXPORT_DATABASE_URL }}" >> .env + echo "METABASE_DATABASE_URL=${{ secrets.METABASE_EXPORT_METABASE_DATABASE_URL }}" >> .env + - name: Run export run: make ci_metabase_export - env: - METABASE_SRC_APP: ${{ vars.METABASE_EXPORT_SRC_APP }} - METABASE_SRC_DATABASE_URL: ${{ secrets.METABASE_EXPORT_SRC_DATABASE_URL }} - METABASE_DEST_APP: ${{ vars.METABASE_EXPORT_DEST_APP }} - METABASE_DEST_DATABASE_URL: ${{ secrets.METABASE_EXPORT_DEST_DATABASE_URL }} diff --git a/.github/workflows/metabase_migrate.yml b/.github/workflows/metabase_migrate.yml new file mode 100644 index 000000000..68edc0672 --- /dev/null +++ b/.github/workflows/metabase_migrate.yml @@ -0,0 +1,40 @@ +name: Metabase Migrate + +on: + workflow_dispatch: + push: + branches: + - main + paths: + - 'src/Infrastructure/Persistence/Doctrine/MetabaseMigrations/**' + +jobs: + build: + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v1 + + - name: Setup PHP with PECL extension + uses: shivammathur/setup-php@v2 + with: + php-version: '8.2' + + - name: Get Composer Cache Directory + id: composer-cache + run: | + echo "dir=$(composer config cache-files-dir)" >> $GITHUB_OUTPUT + + - uses: actions/cache@v3 + with: + path: ${{ steps.composer-cache.outputs.dir }} + key: ${{ runner.os }}-composer-${{ hashFiles('**/composer.lock') }} + restore-keys: | + ${{ runner.os }}-composer- + + - name: Init environment variables + run: | + echo "METABASE_DATABASE_URL=${{ secrets.METABASE_MIGRATIONS_METABASE_DATABASE_URL }}" >> .env + + - name: CI + run: make ci_metabase_migrate BIN_COMPOSER="composer" BIN_CONSOLE="php bin/console" diff --git a/Makefile b/Makefile index 8398fbd4b..ad72a1919 100644 --- a/Makefile +++ b/Makefile @@ -67,6 +67,7 @@ dbinstall: ## Setup databases make data_install make console CMD="doctrine:database:create --env=test --if-not-exists" make dbmigrate ARGS="--env=test" + make metabase_migrate ARGS="--env=test" make data_install ARGS="--env=test" make dbfixtures @@ -107,6 +108,12 @@ bdtopo_migrate_redo: ## Revert db migrations for bdtopo and run them again # Re-run migrations from there make bdtopo_migrate +metabase_migration: ## Generate new migration for metabase + ${BIN_CONSOLE} doctrine:migrations:generate --configuration ./config/packages/metabase/doctrine_migrations.yaml + +metabase_migrate: ## Run db migrations for metabase + ${BIN_CONSOLE} doctrine:migrations:migrate -n --all-or-nothing --configuration ./config/packages/metabase/doctrine_migrations.yaml ${ARGS} + dbshell: ## Connect to the database docker compose exec database psql postgresql://dialog:dialog@database:5432/dialog @@ -280,10 +287,14 @@ ci_bdtopo_migrate: ## Run CI steps for BD TOPO Migrate workflow make composer CMD="install -n --prefer-dist" make bdtopo_migrate +ci_metabase_migrate: ## Run CI steps for Metabase Migrate workflow + make composer CMD="install -n --prefer-dist" + make metabase_migrate + ci_metabase_export: ## Export data to Metabase scalingo login --ssh --ssh-identity ~/.ssh/id_rsa - ./tools/scalingodbtunnel ${METABASE_DEST_APP} --host-url --port 10001 & ./tools/wait-for-it.sh 127.0.0.1:10001 - ./tools/metabase-export.sh ${METABASE_SRC_DATABASE_URL} ${METABASE_DEST_DATABASE_URL} + ./tools/scalingodbtunnel dialog-metabase --host-url --port 10001 & ./tools/wait-for-it.sh 127.0.0.1:10001 + make console CMD="app:metabase:export" ## ## ---------------- diff --git a/config/packages/doctrine.yaml b/config/packages/doctrine.yaml index 5ccb4a623..070489211 100644 --- a/config/packages/doctrine.yaml +++ b/config/packages/doctrine.yaml @@ -10,6 +10,9 @@ doctrine: bdtopo: url: '%env(BDTOPO_DATABASE_URL)%' use_savepoints: true + metabase: + url: '%env(METABASE_DATABASE_URL)%' + use_savepoints: true orm: auto_generate_proxy_classes: true default_entity_manager: default @@ -33,12 +36,18 @@ doctrine: alias: 'App\Domain' bdtopo: connection: bdtopo + metabase: + connection: metabase when@test: doctrine: dbal: - # "TEST_TOKEN" is typically set by ParaTest - dbname_suffix: '_test%env(default::TEST_TOKEN)%' + connections: + default: + # "TEST_TOKEN" is typically set by ParaTest + dbname_suffix: '_test%env(default::TEST_TOKEN)%' + metabase: + dbname_suffix: '_test%env(default::TEST_TOKEN)%' when@prod: doctrine: diff --git a/config/packages/metabase/doctrine_migrations.yaml b/config/packages/metabase/doctrine_migrations.yaml new file mode 100644 index 000000000..bca5018be --- /dev/null +++ b/config/packages/metabase/doctrine_migrations.yaml @@ -0,0 +1,3 @@ +migrations_paths: + App\Infrastructure\Persistence\Doctrine\MetabaseMigrations: 'src/Infrastructure/Persistence/Doctrine/MetabaseMigrations' +em: metabase diff --git a/docs/tools/metabase.md b/docs/tools/metabase.md index 087bbe7ad..ebbc535a8 100644 --- a/docs/tools/metabase.md +++ b/docs/tools/metabase.md @@ -12,20 +12,31 @@ Le Metabase de DiaLog est hébergé sur Scalingo sous l'application `dialog-meta Cette application dispose de sa propre base de données où nous stockons les données nécessaires au calcul des indicateurs, conformément aux [recommendations Beta](https://doc.incubateur.net/communaute/les-outils-de-la-communaute/autres-services/metabase/metabase#connecter-metabase-a-une-base-de-donnees-anonymisee) -La collecte des données d'indicateurs est réalisée au moyen d'un [script](../../tools/metabase-export.sh). Ce script exécute des requêtes SQL depuis la base Metabase vers la base applicative. Pour cela un utilisateur `dialog_metabase` avec droits en lecture seule a été créé sur la base applicative (identifiants dans le Vaultwarden de l'équipe DiaLog). +La collecte des données d'indicateurs est réalisée au moyen d'une la commande Symfony `app:metabase:export`. Cette commande rassemble les données sources (requêtes à la base de données, requêtes HTTP, ou autres opérations...) puis les upload vers la base de données PostgreSQL de l'instance Metabase. ## Lancer l'export depuis GitHub Actions L'export Metabase peut être déclenché via [GitHub Actions](./github_actions.md) à l'aide du workflow [`metabase_export.yml`](../../.github/workflows/metabase_export.yml). +## Tester l'export en local + +Vous pouvez tester l'export en local en configurant votre `.env.local` comme ceci : + +```bash +METABASE_DATABASE_URL="postgresql://dialog:dialog@database:5432/dialog" +``` + +Lancez ensuite `make console CMD="app:metabase:export"`. Cela aura pour effet de calculer et charger les indicateurs directement dans votre base locale `dialog`. + +La visualisation des graphiques Metabase à partir de ces données n'est pas possible, mais vous pourrez au moins explorer les données brutes dans les tables commençant par `analytics_`. + ### Configuration de la GitHub Action La configuration de la GitHub Action passe par diverses variables d'environnement listées ci-dessous : | Variable d'environnement | Configuration | Description | |---|---|---| -| `METABASE_EXPORT_SRC_APP` | [Variable](https://docs.github.com/fr/actions/learn-github-actions/variables) au sens GitHub Actions | `dialog` (pour la production) | -| `METABASE_EXPORT_SRC_DATABASE_URL` | [Secret](https://docs.github.com/fr/actions/security-guides/using-secrets-in-github-actions) au sens GitHub Actions | L'URL d'accès à la base de données applicative par la DB Metabase : utiliser la `METABASE_EXPORT_SRC_DATABASE_URL` de l'app `dialog` | -| `METABASE_EXPORT_DEST_APP` | Variable | `dialog-metabase` | -| `METABASE_EXPORT_DEST_DATABASE_URL` | Secret | L'URL d'accès à la base de données Metabase par la CI (`./tools/scalingodbtunnel dialog-metabase --host-url --port 10001`) | +| `METABASE_MIGRATIONS_METABASE_DATABASE_URL` | [Secret](https://docs.github.com/fr/actions/security-guides/using-secrets-in-github-actions) au sens GitHub Actions | L'URL d'accès à la base de données Metabase par la CI, afin d'exécuter les migrations (`./tools/scalingodbtunnel dialog-metabase --host-url --port 10001`) | +| `METABASE_EXPORT_DATABASE_URL` | [Secret](https://docs.github.com/fr/actions/security-guides/using-secrets-in-github-actions) au sens GitHub Actions | L'URL d'accès à la base de données applicative par la CI (`./tools/scalingodbtunnel dialog --host-url --port 10001`) | +| `METABASE_EXPORT_METABASE_DATABASE_URL` | Secret | L'URL d'accès à la base de données Metabase par la CI (`./tools/scalingodbtunnel dialog-metabase --host-url --port 10001`) | | `GH_SCALINGO_SSH_PRIVATE_KEY` | Secret | Clé SSH privée permettant l'accès à Scalingo par la CI | diff --git a/phpunit.xml.dist b/phpunit.xml.dist index 6a5374492..b9f5071dc 100644 --- a/phpunit.xml.dist +++ b/phpunit.xml.dist @@ -25,6 +25,7 @@ src/Infrastructure/Persistence/Doctrine/Mapping src/Infrastructure/Persistence/Doctrine/Migrations src/Infrastructure/Persistence/Doctrine/BdTopoMigrations + src/Infrastructure/Persistence/Doctrine/MetabaseMigrations src/Infrastructure/Persistence/Doctrine/PostGIS/Event src/Infrastructure/Adapter/CommandBus.php src/Infrastructure/Adapter/IdFactory.php diff --git a/src/Domain/Statistics/Repository/StatisticsRepositoryInterface.php b/src/Domain/Statistics/Repository/StatisticsRepositoryInterface.php new file mode 100644 index 000000000..eece58d35 --- /dev/null +++ b/src/Domain/Statistics/Repository/StatisticsRepositoryInterface.php @@ -0,0 +1,10 @@ +lastActiveAt; } - public function setLastActiveAt(\DateTimeInterface $date): void + public function setLastActiveAt(\DateTimeInterface $date): self { $this->lastActiveAt = $date; + + return $this; } public function __toString(): string diff --git a/src/Infrastructure/Persistence/Doctrine/Fixtures/UserFixture.php b/src/Infrastructure/Persistence/Doctrine/Fixtures/UserFixture.php index b5c6cb1d5..dfcc90a81 100644 --- a/src/Infrastructure/Persistence/Doctrine/Fixtures/UserFixture.php +++ b/src/Infrastructure/Persistence/Doctrine/Fixtures/UserFixture.php @@ -23,14 +23,16 @@ public function load(ObjectManager $manager): void ->setEmail(self::MAIN_ORG_USER_EMAIL) ->setPassword(self::PASSWORD) ->setRoles([UserRolesEnum::ROLE_USER->value]) - ->setRegistrationDate(new \DateTimeImmutable('2024-03-01')); + ->setRegistrationDate(new \DateTimeImmutable('2024-03-01')) + ->setLastActiveAt(new \DateTimeImmutable('2024-06-07')); $mainOtherAdmin = (new User('5bc831a3-7a09-44e9-aefa-5ce3588dac33')) ->setFullName('Mathieu FERNANDEZ') ->setEmail(self::MAIN_ORG_ADMIN_EMAIL) ->setPassword(self::PASSWORD) ->setRoles([UserRolesEnum::ROLE_SUPER_ADMIN->value]) - ->setRegistrationDate(new \DateTimeImmutable('2024-04-02')); + ->setRegistrationDate(new \DateTimeImmutable('2024-04-02')) + ->setLastActiveAt(new \DateTimeImmutable('2024-06-08')); $otherOrgUser = (new User('d47badd9-989e-472b-a80e-9df642e93880')) ->setFullName('Florimond MANCA') diff --git a/src/Infrastructure/Persistence/Doctrine/MetabaseMigrations/Version20241216132451.php b/src/Infrastructure/Persistence/Doctrine/MetabaseMigrations/Version20241216132451.php new file mode 100644 index 000000000..da34e807c --- /dev/null +++ b/src/Infrastructure/Persistence/Doctrine/MetabaseMigrations/Version20241216132451.php @@ -0,0 +1,38 @@ +addSql( + 'CREATE TABLE IF NOT EXISTS analytics_user_active ( + id UUID NOT NULL, + uploaded_at TIMESTAMP(0), + last_active_at TIMESTAMP(0), + PRIMARY KEY(id) + );', + ); + + $this->addSql( + 'CREATE INDEX IF NOT EXISTS idx_analytics_user_active_uploaded_at + ON analytics_user_active (uploaded_at);', + ); + } + + public function down(Schema $schema): void + { + $this->addSql('DROP TABLE IF EXISTS analytics_user_active'); + } +} diff --git a/src/Infrastructure/Persistence/Doctrine/Repository/Statistics/StatisticsRepository.php b/src/Infrastructure/Persistence/Doctrine/Repository/Statistics/StatisticsRepository.php new file mode 100644 index 000000000..f2ab74dc7 --- /dev/null +++ b/src/Infrastructure/Persistence/Doctrine/Repository/Statistics/StatisticsRepository.php @@ -0,0 +1,42 @@ += [uploaded_at] - 7 jours", puis en groupant sur le uploaded_at.) + $userRows = $this->userRepository->findAllForStatistics(); + $this->bulkInsertUserActiveStatistics($now, $userRows); + } + + private function bulkInsertUserActiveStatistics(\DateTimeInterface $now, array $userRows): void + { + $stmt = $this->metabaseConnection->prepare( + 'INSERT INTO analytics_user_active(id, uploaded_at, last_active_at) + VALUES (:id, (:uploadedAt)::timestamp(0), (:lastActiveAt)::timestamp(0))', + ); + + foreach ($userRows as $row) { + $stmt->bindValue('id', $row['uuid']); + $stmt->bindValue('uploadedAt', $now->format(\DateTimeInterface::ATOM)); + $stmt->bindValue('lastActiveAt', $row['lastActiveAt']?->format(\DateTimeInterface::ATOM)); + $stmt->execute(); + } + } +} diff --git a/src/Infrastructure/Persistence/Doctrine/Repository/User/UserRepository.php b/src/Infrastructure/Persistence/Doctrine/Repository/User/UserRepository.php index 6db376c5d..fb8902e30 100644 --- a/src/Infrastructure/Persistence/Doctrine/Repository/User/UserRepository.php +++ b/src/Infrastructure/Persistence/Doctrine/Repository/User/UserRepository.php @@ -48,4 +48,12 @@ public function countUsers(): int ->getQuery() ->getSingleScalarResult(); } + + public function findAllForStatistics(): array + { + return $this->createQueryBuilder('u') + ->select('u.uuid, u.lastActiveAt') + ->getQuery() + ->getResult(); + } } diff --git a/src/Infrastructure/Symfony/Command/RunMetabaseExportCommand.php b/src/Infrastructure/Symfony/Command/RunMetabaseExportCommand.php new file mode 100644 index 000000000..548560b7e --- /dev/null +++ b/src/Infrastructure/Symfony/Command/RunMetabaseExportCommand.php @@ -0,0 +1,36 @@ +dateUtils->getNow(); + + $this->statisticsRepository->addUserActiveStatistics($now); + + return Command::SUCCESS; + } +} diff --git a/tests/Integration/Infrastructure/Controller/Regulation/AddRegulationControllerTest.php b/tests/Integration/Infrastructure/Controller/Regulation/AddRegulationControllerTest.php index 803b2d865..118da7d65 100644 --- a/tests/Integration/Infrastructure/Controller/Regulation/AddRegulationControllerTest.php +++ b/tests/Integration/Infrastructure/Controller/Regulation/AddRegulationControllerTest.php @@ -38,7 +38,7 @@ public function testAdd(): void /** @var UserRepositoryInterface */ $userRepository = static::getContainer()->get(UserRepositoryInterface::class); - $this->assertNull($userRepository->findOneByEmail($email)->getLastActiveAt()); + $this->assertEquals(new \DateTimeImmutable('2024-06-07'), $userRepository->findOneByEmail($email)->getLastActiveAt()); // Get the raw values. $values = $form->getPhpValues(); @@ -56,6 +56,7 @@ public function testAdd(): void $crawler = $client->request($form->getMethod(), $form->getUri(), $values, $form->getPhpFiles()); $this->assertResponseStatusCodeSame(303); + // Filled with DateUtilsMock::getNow() $this->assertEquals(new \DateTimeImmutable('2023-06-09'), $userRepository->findOneByEmail($email)->getLastActiveAt()); $client->followRedirect(); diff --git a/tests/Integration/Infrastructure/Symfony/Command/RunMetabaseExportCommandTest.php b/tests/Integration/Infrastructure/Symfony/Command/RunMetabaseExportCommandTest.php new file mode 100644 index 000000000..8c3102161 --- /dev/null +++ b/tests/Integration/Infrastructure/Symfony/Command/RunMetabaseExportCommandTest.php @@ -0,0 +1,38 @@ +get(RunMetabaseExportCommand::class); + $commandTester = new CommandTester($command); + $commandTester->execute([]); + $commandTester->assertCommandIsSuccessful(); + + /** @var \Doctrine\DBAL\Connection */ + $metabaseConnection = $container->get('doctrine.dbal.metabase_connection'); + $rows = $metabaseConnection->fetchAllAssociative('SELECT * FROM analytics_user_active'); + $this->assertCount(3, $rows); + $this->assertEquals(['id', 'uploaded_at', 'last_active_at'], array_keys($rows[0])); + + $this->assertSame('2023-06-09 00:00:00', $rows[0]['uploaded_at']); + $this->assertSame('2024-06-07 00:00:00', $rows[0]['last_active_at']) + ; + $this->assertSame('2023-06-09 00:00:00', $rows[1]['uploaded_at']); + $this->assertSame('2024-06-08 00:00:00', $rows[1]['last_active_at']); + + $this->assertSame('2023-06-09 00:00:00', $rows[2]['uploaded_at']); + $this->assertSame(null, $rows[2]['last_active_at']); + } +} diff --git a/tools/metabase-export.sh b/tools/metabase-export.sh deleted file mode 100755 index 4e56c92bd..000000000 --- a/tools/metabase-export.sh +++ /dev/null @@ -1,16 +0,0 @@ -#!/bin/bash -# Inspiré de : https://doc.incubateur.net/communaute/les-outils-de-la-communaute/autres-services/metabase/metabase#connecter-metabase-a-une-base-de-donnees-anonymisee -set -euxo pipefail - -# URL de la DB DiaLog du point de vue de la DB Metabase (sera utilisé avec dblink) -SRC_DATABASE_URL=$1 - -# URL de la DB Metabase du point de vue de ce script -DEST_DATABASE_URL=$2 - -export PGOPTIONS="-c custom.src_database_url=${SRC_DATABASE_URL}" - -# ON_ERROR_STOP=1 s'assure que cette commande échoue (return code != 0) si le script SQL a des statements qui échouent. -# (Par défaut avec -f on a toujours un return code 0 et un statement en échec n'empêche pas les suivants de s'exécuter.) -# https://engineering.nordeus.com/psql-exit-on-first-error/ -psql $DEST_DATABASE_URL -v ON_ERROR_STOP=1 -f ./tools/metabase-export.sql diff --git a/tools/metabase-export.sql b/tools/metabase-export.sql deleted file mode 100644 index 4b2e50c04..000000000 --- a/tools/metabase-export.sql +++ /dev/null @@ -1,31 +0,0 @@ --- metabase-export.sql --- Ce script est conçu pour être exécuté sur la base de données PostgreSQL de l'instance Metabase (destination). --- Il consiste à extraire des données de la base applicative (source) pour les charger dans des tables Metabase. - --- Configuration générale -CREATE EXTENSION IF NOT EXISTS "uuid-ossp"; - --- CONNEXION À LA DB APPLICATIVE (source) --- Voir : https://www.postgresql.org/docs/current/contrib-dblink-connect.html -CREATE EXTENSION IF NOT EXISTS dblink; -SELECT dblink_connect('src', current_setting('custom.src_database_url')); - --- COLLECTE DES DONNÉES D'INDICATEURS - --- # Utilisateurs actifs --- À chaque exécution, on ajoute la liste des dates de dernière activité pour chaque utilisateur, assortie de la date d'exécution. --- Dans Metabase cela permet de calculer le nombre d'utilisateurs actif au moment de chaque exécution. --- (Par exemple avec un filtre : "[last_active_at] >= [uploaded_at] - 7 jours", puis en groupant sur le uploaded_at.) -CREATE TABLE IF NOT EXISTS analytics_user_active (id UUID NOT NULL, uploaded_at TIMESTAMP(0), last_active_at TIMESTAMP(0), PRIMARY KEY(id)); -CREATE INDEX IF NOT EXISTS idx_analytics_user_active_uploaded_at ON analytics_user_active (uploaded_at); - -WITH params AS ( - -- Calculé 1 bonne fois pour toute pour que toutes les lignes utilisent exactement la même valeur à des fins de groupement dans Metabase - SELECT NOW() as current_date -) -INSERT INTO analytics_user_active(id, uploaded_at, last_active_at) -SELECT uuid_generate_v4() AS id, p.current_date AS uploaded_at, u.last_active_at AS last_active_at -FROM - dblink('src', 'SELECT last_active_at FROM "user"') AS u(last_active_at TIMESTAMP(0) WITH TIME ZONE), - params AS p -;