From f4db67fd5bc26fb5124bafc31a6a53a11e9ab435 Mon Sep 17 00:00:00 2001 From: Sebastian Date: Fri, 27 Oct 2023 14:36:12 +0200 Subject: [PATCH 1/4] fix: reduce census API computation duration by ~93% --- src/Service/DataProvider/CensusDataProvider.php | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/Service/DataProvider/CensusDataProvider.php b/src/Service/DataProvider/CensusDataProvider.php index 83f2d8e..443bef3 100644 --- a/src/Service/DataProvider/CensusDataProvider.php +++ b/src/Service/DataProvider/CensusDataProvider.php @@ -312,10 +312,12 @@ public function getTreemapData(Group $group, CensusRequestData $censusRequestDat */ private function filterGroups(array $statisticGroups, CensusRequestData $censusRequestData) { - return array_filter($statisticGroups, function ($group) use ($censusRequestData) { - return sizeof(array_filter($censusRequestData->getGroups(), function ($groupId) use ($group) { - return $groupId == $group->getId(); - })) === 0; + // For faster lookups we swap array index with value so that array goes from [1 => 23, 2 => 352] to [23 => null, 352 => null] + $groupIdsToFilterOut = array_flip($censusRequestData->getGroups()); + $filteredGroups = array_filter($statisticGroups, function (StatisticGroup $group) use ($groupIdsToFilterOut) { + return !isset($groupIdsToFilterOut[$group->getId()]); }); + // Ensure that they are sequential. + return array_values($filteredGroups); } } From b5ceb1b8eb294741c064d9fc7bc716513a933a93 Mon Sep 17 00:00:00 2001 From: Sebastian Date: Fri, 27 Oct 2023 16:18:52 +0200 Subject: [PATCH 2/4] fix: prevent 0 division and null errors --- src/Controller/Api/Apps/CensusController.php | 12 ++++++++---- src/DTO/Mapper/CensusMapper.php | 5 +++-- src/Service/DataProvider/CensusDataProvider.php | 4 +++- 3 files changed, 14 insertions(+), 7 deletions(-) diff --git a/src/Controller/Api/Apps/CensusController.php b/src/Controller/Api/Apps/CensusController.php index 98a3efa..3ebd63a 100644 --- a/src/Controller/Api/Apps/CensusController.php +++ b/src/Controller/Api/Apps/CensusController.php @@ -45,7 +45,8 @@ public function getPreview(Group $group) public function getTableData(Group $group, CensusRequestData $censusRequestData) { $this->denyAccessUnlessGranted(PermissionVoter::VIEWER, $group); - return $this->json($this->censusDataProvider->getTableData($group, $censusRequestData)); + $data = $this->censusDataProvider->getTableData($group, $censusRequestData); + return $this->json($data); } /** @@ -57,7 +58,8 @@ public function getTableData(Group $group, CensusRequestData $censusRequestData) public function getDevelopmentData(Group $group, CensusRequestData $censusRequestData) { $this->denyAccessUnlessGranted(PermissionVoter::VIEWER, $group); - return $this->json($this->censusDataProvider->getDevelopmentData($group, $censusRequestData)); + $data = $this->censusDataProvider->getDevelopmentData($group, $censusRequestData); + return $this->json($data); } @@ -70,7 +72,8 @@ public function getDevelopmentData(Group $group, CensusRequestData $censusReques public function getMembersData(Group $group, CensusRequestData $censusRequestData) { $this->denyAccessUnlessGranted(PermissionVoter::VIEWER, $group); - return $this->json($this->censusDataProvider->getMembersData($group, $censusRequestData)); + $data = $this->censusDataProvider->getMembersData($group, $censusRequestData); + return $this->json($data); } /** @@ -82,7 +85,8 @@ public function getMembersData(Group $group, CensusRequestData $censusRequestDat public function getTreemapData(Group $group, CensusRequestData $censusRequestData) { $this->denyAccessUnlessGranted(PermissionVoter::VIEWER, $group); - return $this->json($this->censusDataProvider->getTreemapData($group, $censusRequestData)); + $data = $this->censusDataProvider->getTreemapData($group, $censusRequestData); + return $this->json($data); } /** diff --git a/src/DTO/Mapper/CensusMapper.php b/src/DTO/Mapper/CensusMapper.php index 8cdb308..4e15067 100644 --- a/src/DTO/Mapper/CensusMapper.php +++ b/src/DTO/Mapper/CensusMapper.php @@ -66,8 +66,9 @@ public static function mapToCensusTable(StatisticGroup $statisticGroup, array $c $improvementVs3YearsAgo = (100 / $totalCounts[count($totalCounts) - 4]) * $totalCounts[count($totalCounts) - 1] - 100; } } - if (!$incomplete) { - $improvementVsAvg5Years = (100 / (($totalCounts[0] + $totalCounts[1] + $totalCounts[2] + $totalCounts[3] + $totalCounts[4]) / 5)) * $totalCounts[count($totalCounts) - 1] - 100; + $fiveYearTotal = $totalCounts[0] + $totalCounts[1] + $totalCounts[2] + $totalCounts[3] + $totalCounts[4]; + if (!$incomplete && $fiveYearTotal !== 0) { + $improvementVsAvg5Years = (100 / ($fiveYearTotal / 5)) * $totalCounts[count($totalCounts) - 1] - 100; } $dto->setRelativeMemberCounts([$improvementVsLastYear, $improvementVs3YearsAgo, $improvementVsAvg5Years]); return $dto; diff --git a/src/Service/DataProvider/CensusDataProvider.php b/src/Service/DataProvider/CensusDataProvider.php index 443bef3..d77ee4f 100644 --- a/src/Service/DataProvider/CensusDataProvider.php +++ b/src/Service/DataProvider/CensusDataProvider.php @@ -238,7 +238,6 @@ public function getDevelopmentData(Group $group, CensusRequestData $censusReques $relative[] = $dto->getRelative()[0]; } } - $return = new DevelopmentWidgetDTO(); $return->setYears($relevantYears); $return->setAbsolute($absolute); @@ -313,6 +312,9 @@ public function getTreemapData(Group $group, CensusRequestData $censusRequestDat private function filterGroups(array $statisticGroups, CensusRequestData $censusRequestData) { // For faster lookups we swap array index with value so that array goes from [1 => 23, 2 => 352] to [23 => null, 352 => null] + if (is_null($censusRequestData->getGroups())) { + return $statisticGroups; + } $groupIdsToFilterOut = array_flip($censusRequestData->getGroups()); $filteredGroups = array_filter($statisticGroups, function (StatisticGroup $group) use ($groupIdsToFilterOut) { return !isset($groupIdsToFilterOut[$group->getId()]); From 8de02049c378db527059e5759dbcfbe9785ab369 Mon Sep 17 00:00:00 2001 From: Sebastian Date: Fri, 27 Oct 2023 16:31:27 +0200 Subject: [PATCH 3/4] fix: use proper entities for preview --- src/Service/DataProvider/CensusDataProvider.php | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/Service/DataProvider/CensusDataProvider.php b/src/Service/DataProvider/CensusDataProvider.php index d77ee4f..bccfa95 100644 --- a/src/Service/DataProvider/CensusDataProvider.php +++ b/src/Service/DataProvider/CensusDataProvider.php @@ -44,7 +44,7 @@ public function __construct( public function getPreviewData(Group $group) { - $groups = $this->groupRepository->findAllRelevantSubGroupsByParentGroupId($group->getId(), ['Group::Abteilung', 'Group::Kantonalverband', 'Group::Region']); // Replace with group endpoint + $flattenedGroups = $this->getRelevantGroups($group); $return = [ 'm' => [ 'leiter' => 0, @@ -65,8 +65,8 @@ public function getPreviewData(Group $group) 'pta' => 0 ] ]; - foreach ($groups as $group) { - $censusGroup = $this->censusGroupRepository->findOneBy(['group_id' => $group['id'], 'year' => date('Y')]); + foreach ($flattenedGroups as $group) { + $censusGroup = $this->censusGroupRepository->findOneBy(['group_id' => $group->getId(), 'year' => date('Y')]); if (!is_null($censusGroup)) { $return['m']['leiter'] += $censusGroup->getLeiterMCount(); $return['m']['biber'] += $censusGroup->getBiberMCount(); From 445e4292911d1981a5e728976f9c6ea69b150f6f Mon Sep 17 00:00:00 2001 From: Sebastian Date: Fri, 27 Oct 2023 16:48:39 +0200 Subject: [PATCH 4/4] add crontab and finish census command --- crontab | 3 +++ src/Command/FetchCensusCommand.php | 12 ++++-------- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/crontab b/crontab index 89059cb..8ef3e02 100644 --- a/crontab +++ b/crontab @@ -3,3 +3,6 @@ # Run every 3 months at 00:00 (AM) 0 0 1 */3 * /usr/local/bin/php /srv/bin/console app:import-geo-addresses + +# Run once a year on the first of february at 01:00 (AM) +0 1 1 2 * /usr/local/bin/php /srv/bin/console app:fetch-census diff --git a/src/Command/FetchCensusCommand.php b/src/Command/FetchCensusCommand.php index 6382700..ccdf6c8 100644 --- a/src/Command/FetchCensusCommand.php +++ b/src/Command/FetchCensusCommand.php @@ -20,6 +20,7 @@ class FetchCensusCommand extends StatisticsCommand protected GroupTypeRepository $groupTypeRepository; private SymfonyStyle $io; + private $start; public function __construct( CensusAPIService $apiService, @@ -36,16 +37,16 @@ public function __construct( public function configure() { $this->setName('app:fetch-census') - ->setDescription('Not implemented'); + ->setDescription('Fetch and aggregate census data'); } public function execute(InputInterface $input, OutputInterface $output) { + $this->start = microtime(true); $this->io = new SymfonyStyle($input, $output); $year = (int) date('Y'); $minYear = $year - 6; - $groupsToAggregate = []; // Fetch groups while ($year > $minYear) { $this->io->writeln('year ' . $year); @@ -54,15 +55,11 @@ public function execute(InputInterface $input, OutputInterface $output) foreach ($rawCensusGroups as $rawCensusGroup) { $exists = $this->censusGroupRepository->findOneBy(['group_id' => $rawCensusGroup['group_id'], 'year' => $year]); if (is_null($exists)) { - $groupsToAggregate[] = $rawCensusGroup['group_id']; $this->mapRawCensusGroupToCensusGroup($rawCensusGroup, $year); } } $year--; } - // Aggregate Groups - foreach (array_unique($groupsToAggregate) as $groupId) { - } return Command::SUCCESS; } @@ -102,9 +99,8 @@ private function sanitizeValue($raw): int } - // TODO: Implement the statistics public function getStats(): CommandStatistics { - return new CommandStatistics(0, 'Statistics not yet implemented.'); + return new CommandStatistics(microtime(true) - $this->start, ''); } }