From 1a479839804a7df88c1c868d5be31bb9145c035b Mon Sep 17 00:00:00 2001 From: Carlo Beltrame Date: Mon, 23 Aug 2021 20:52:57 +0200 Subject: [PATCH 01/12] WIP use the normal hitobito API instead of the group_health endpoints for groups, people and roles --- config/routing_api.yml | 6 + src/Command/FetchDataCommand.php | 2 +- src/Command/ImportFromJsonCommand.php | 160 ------------------ src/Controller/Api/SyncController.php | 92 ++++++++++ src/Entity/Group.php | 9 + src/Entity/Person.php | 26 +++ src/Service/PbsApi/Fetcher/GroupFetcher.php | 101 +++++++++++ src/Service/PbsApi/Fetcher/PeopleFetcher.php | 127 ++++++++++++++ .../PbsApi/Fetcher/PersonRoleMapper.php | 93 ++++++++++ src/Service/PbsApiService.php | 16 ++ src/Service/SyncService.php | 57 +++++++ translations/messages+intl-icu.de.yml | 2 + translations/messages+intl-icu.fr.yml | 2 + translations/messages+intl-icu.it.yml | 2 + 14 files changed, 534 insertions(+), 161 deletions(-) create mode 100644 src/Controller/Api/SyncController.php create mode 100644 src/Service/PbsApi/Fetcher/GroupFetcher.php create mode 100644 src/Service/PbsApi/Fetcher/PeopleFetcher.php create mode 100644 src/Service/PbsApi/Fetcher/PersonRoleMapper.php create mode 100644 src/Service/SyncService.php diff --git a/config/routing_api.yml b/config/routing_api.yml index 2ef9737..40816a4 100644 --- a/config/routing_api.yml +++ b/config/routing_api.yml @@ -24,6 +24,12 @@ invite_delete: methods: DELETE controller: App\Controller\Api\InviteController:deleteInvite +# Sync routes +sync_start: + path: /groups/{groupId}/sync + methods: POST + controller: App\Controller\Api\SyncController:startSync + # Filter data routes filter_data: path: /groups/{groupId}/filter-data diff --git a/src/Command/FetchDataCommand.php b/src/Command/FetchDataCommand.php index 0519105..69b30f6 100644 --- a/src/Command/FetchDataCommand.php +++ b/src/Command/FetchDataCommand.php @@ -14,7 +14,7 @@ class FetchDataCommand extends StatisticsCommand public const STAT_TABLE_HEADERS = ['Table Name', 'Duration (s)', 'Items Fetched']; public const STAT_TOTAL_TABLE_HEADERS = ['Total Duration (s)', 'Total Items Fetched']; - private const PAGINATED = ['people', 'groups', 'roles', 'courses', 'camps', 'participations', 'qualifications']; + private const PAGINATED = ['courses', 'camps', 'participations', 'qualifications']; private const NOT_PAGINATED = ['group_types', 'role_types', 'participation_types', 'j_s_kinds', 'camp_states', 'qualification_kinds', 'event_kinds']; diff --git a/src/Command/ImportFromJsonCommand.php b/src/Command/ImportFromJsonCommand.php index f4706f2..0031189 100644 --- a/src/Command/ImportFromJsonCommand.php +++ b/src/Command/ImportFromJsonCommand.php @@ -101,13 +101,10 @@ protected function execute(InputInterface $input, OutputInterface $output): int $this->importYouthSportTypes($output); $this->importCampStates($output); $this->importPersonEventTypes($output); - $this->importGroups($output); $this->importCourses($output); $this->importCamps($output); - $this->importPeople($output); $this->importParticipations($output); $this->importQualifications($output); - $this->importRoles($output); $this->em->getConnection()->commit(); @@ -361,50 +358,6 @@ private function importPersonEventTypes(OutputInterface $output) $output->writeln([sprintf('%s rows imported from participation_types.json', $i)]); } - /** - * @param OutputInterface $output - * @throws Exception - */ - private function importGroups(OutputInterface $output) - { - $start = microtime(true); - $groups = JsonMachine::fromFile(sprintf('%s/groups.json', $this->params->get('import_data_dir'))); - $i = 0; - foreach ($groups as $gr) { - $group = $this->em->getRepository(Group::class)->findOneBy(['id' => $gr['id']]); - if (!$group) { - $group = new Group(); - $group->setId($gr['id']); - $metadata = $this->em->getClassMetaData(get_class($group)); - $metadata->setIdGenerator(new AssignedGenerator()); - } - - $group->setName($gr['name']); - $group->setCantonId($gr['canton_id']); - $group->setCantonName($gr['canton_name']); - $group->setCreatedAt(new DateTimeImmutable($gr['created_at'])); - if ($gr['deleted_at']) { - $group->setDeletedAt(new DateTimeImmutable($gr['deleted_at'])); - } - - /** @var GroupType $gt */ - $gt = $this->em->getRepository(GroupType::class)->findOneBy(['groupType' => $gr['type']]); - $group->setGroupType($gt); - - if ($gr['parent_id'] !== null) { - $pg = $this->em->getRepository(Group::class)->find($gr['parent_id']); - $group->setParentGroup($pg); - } - - $this->em->persist($group); - $this->em->flush(); - $i++; - } - $timeElapsed = microtime(true) - $start; - $this->stats[] = ['groups.json', $timeElapsed, $i]; - $output->writeln([sprintf('%s rows imported from groups.json', $i)]); - } - /** * @param OutputInterface $output * @throws Exception @@ -534,67 +487,6 @@ private function importCamps(OutputInterface $output) $output->writeln([sprintf('%s rows imported from camps.json', $i)]); } - /** - * @param OutputInterface $output - * @throws Exception - */ - private function importPeople(OutputInterface $output) - { - $this->personRepository->markAllAsLeft(); - - $start = microtime(true); - $people = JsonMachine::fromFile(sprintf('%s/people.json', $this->params->get('import_data_dir'))); - $i = 0; - foreach ($people as $p) { - $person = $this->em->getRepository(Person::class)->findOneBy(['id' => $p['id']]); - if (!$person) { - $person = new Person(); - $person->setId($p['id']); - $metadata = $this->em->getClassMetaData(get_class($person)); - $metadata->setIdGenerator(new AssignedGenerator()); - } - $person->setNickname($p['name']); - $person->setGender($p['gender']); - $person->setAddress($p['address']); - $person->setCountry($p['country']); - $person->setZip(intval($p['zip_code'])); - if ($p['birthday']) { - $person->setBirthday(new DateTimeImmutable($p['birthday'])); - } - $person->setPbsNumber($p['pbs_number']); - if ($p['entry_date']) { - $person->setEntryDate(new DateTimeImmutable($p['entry_date'])); - } - if ($p['leaving_date']) { - $person->setLeavingDate(new DateTimeImmutable($p['leaving_date'])); - } else { - $person->setLeavingDate(null); - } - $person->setTown($p['town']); - - if ($p['primary_group_id']) { - $group = $this->em->getRepository(Group::class)->find($p['primary_group_id']); - if ($group) { - $person->setGroup($group); - } - } - - $this->em->persist($person); - $i++; - - if (($i % $this->batchSize) === 0) { - $this->em->flush(); - $this->em->clear(); - } - } - - $this->em->flush(); - - $timeElapsed = microtime(true) - $start; - $this->stats[] = ['people.json', $timeElapsed, $i]; - $output->writeln([sprintf('%s rows imported from people.json', $i)]); - } - /** * @param OutputInterface $output */ @@ -706,58 +598,6 @@ private function importQualifications(OutputInterface $output) $output->writeln([sprintf('%s rows imported from qualifications.json', $i)]); } - /** - * @param OutputInterface $output - * @throws Exception - */ - private function importRoles(OutputInterface $output) - { - $start = microtime(true); - $roles = JsonMachine::fromFile(sprintf('%s/roles.json', $this->params->get('import_data_dir'))); - $i = 0; - foreach ($roles as $r) { - $personRole = $this->em->getRepository(PersonRole::class)->findOneBy(['id' => $r['id']]); - if (!$personRole) { - $personRole = new PersonRole(); - $personRole->setId($r['id']); - $metadata = $this->em->getClassMetaData(get_class($personRole)); - $metadata->setIdGenerator(new AssignedGenerator()); - } - $person = $this->em->getRepository(Person::class)->find($r['person_id']); - if (!$person) { - continue; - } - $personRole->setPerson($person); - - $role = $this->em->getRepository(Role::class)->getOneByRoleType($r['type']); - if ($role) { - $personRole->setRole($role); - } - - $group = $this->em->getRepository(Group::class)->find($r['group_id']); - if ($group) { - $personRole->setGroup($group); - } - - $personRole->setCreatedAt(new DateTimeImmutable($r['created_at'])); - if ($r['deleted_at']) { - $personRole->setDeletedAt(new DateTimeImmutable($r['deleted_at'])); - } - - $this->em->persist($personRole); - $i++; - - if (($i % $this->batchSize) === 0) { - $this->em->flush(); - $this->em->clear(); - } - } - $this->em->flush(); - $timeElapsed = microtime(true) - $start; - $this->stats[] = ['roles.json', $timeElapsed, $i]; - $output->writeln([sprintf('%s rows imported from roles.json', $i)]); - } - public function getStats(): CommandStatistics { $totalItems = 0; diff --git a/src/Controller/Api/SyncController.php b/src/Controller/Api/SyncController.php new file mode 100644 index 0000000..6112dfb --- /dev/null +++ b/src/Controller/Api/SyncController.php @@ -0,0 +1,92 @@ +syncService = $syncService; + $this->translator = $translator; + } + + /** + * @param Request $request + * @param Group $group + * @param SerializerInterface $serializer + * @param ValidatorInterface $validator + * @return JsonResponse + * @ParamConverter(name="group", options={"mapping":{"groupId":"id"}}) + * @IsGranted("create", subject="group") + */ + public function startSync( + Request $request, + Group $group, + SerializerInterface $serializer, + ValidatorInterface $validator + ) { + $action = $this->translator->trans('api.action.started'); + $entity = $this->translator->trans('api.entity.sync'); + $message = $this->translator->trans('api.success', ['entityName' => $entity, 'action' => $action]); + $this->syncService->startSync($group); + return $this->json($message, JsonResponse::HTTP_CREATED); + } + + /** + * @param Group $group + * @return JsonResponse + * @ParamConverter(name="group", options={"mapping":{"groupId":"id"}}) + * @IsGranted("view", subject="group") + */ + public function getInvites(Group $group) + { + return $this->json($this->inviteService->getAllInvites($group)); + } + + /** + * @param Group $group + * @param Invite $invite + * @return JsonResponse + * @ParamConverter(name="group", options={"mapping":{"groupId":"id"}}) + * @ParamConverter(name="invite", options={"mapping":{"inviteId":"id"}}) + * @IsGranted("delete", subject="group") + */ + public function deleteInvite(Group $group, Invite $invite) + { + $this->inviteService->deleteInvite($invite, $group); + $action = $this->translator->trans('api.action.deleted'); + $entity = $this->translator->trans('api.entity.invite'); + $message = $this->translator->trans('api.success', ['entityName' => $entity, 'action' => $action]); + return $this->json($message); + } +} diff --git a/src/Entity/Group.php b/src/Entity/Group.php index 636c9dc..7bb7dd4 100644 --- a/src/Entity/Group.php +++ b/src/Entity/Group.php @@ -210,6 +210,15 @@ public function setGroupType(?GroupType $groupType) $this->groupType = $groupType; } + public function addChild(Group $child): self { + if (!$this->children->contains($child)) { + $this->children[] = $child; + $child->setParentGroup($this); + } + + return $this; + } + public function __toString() { return (string)$this->id; diff --git a/src/Entity/Person.php b/src/Entity/Person.php index c4797db..9226d38 100644 --- a/src/Entity/Person.php +++ b/src/Entity/Person.php @@ -4,6 +4,7 @@ use DateTimeImmutable; use DateTimeInterface; +use Doctrine\Common\Collections\ArrayCollection; use Doctrine\ORM\Mapping as ORM; /** @@ -86,6 +87,11 @@ class Person */ private $group; + /** + * @ORM\OneToMany(targetEntity="PersonRole", mappedBy="person") + */ + private $personRoles; + /** * @ORM\OneToMany(targetEntity="PersonEvent", mappedBy="person", cascade={"persist", "remove"}) */ @@ -102,6 +108,11 @@ class Person */ private $geoAddress; + public function __construct() + { + $this->personRoles = new ArrayCollection(); + } + /** * @param int $id */ @@ -309,4 +320,19 @@ public function setGeoAddress(GeoAddress $geoAddress): void { $this->geoAddress = $geoAddress; } + + public function addPersonRole(PersonRole $personRole): self { + if (!$this->personRoles->contains($personRole)) { + $this->personRoles[] = $personRole; + $personRole->setPerson($this); + } + + return $this; + } + + public function clearPersonRoles(): self { + $this->personRoles = new ArrayCollection(); + + return $this; + } } diff --git a/src/Service/PbsApi/Fetcher/GroupFetcher.php b/src/Service/PbsApi/Fetcher/GroupFetcher.php new file mode 100644 index 0000000..325ca40 --- /dev/null +++ b/src/Service/PbsApi/Fetcher/GroupFetcher.php @@ -0,0 +1,101 @@ +em = $em; + $this->groupTypeRepository = $this->em->getRepository(GroupType::class); + $this->groupRepository = $this->em->getRepository(Group::class); + $this->pbsApiService = $pbsApiService; + } + + public function fetchAndPersistGroup(string $id) + { + $this->em->persist($this->fetchGroup($id)); + $this->em->flush(); + } + + private function fetchGroup(string $id): Group + { + $groupData = $this->pbsApiService->getApiData('/groups/'.$id); + return $this->mapJsonToGroup($groupData); + } + + private function mapJsonToGroup(array $json): Group + { + $groupJson = $json['groups'][0] ?? []; + $linked = $json['linked'] ?? []; + + $group = $this->groupRepository->findOneBy(['id' => $groupJson['id']]); + if (!$group) { + $group = new Group(); + $group->setId($groupJson['id']); + $metadata = $this->em->getClassMetaData(Group::class); + $metadata->setIdGenerator(new AssignedGenerator()); + } + $group->setName($this->gr['name'] ?? null); + + $cantonId = $groupJson['links']['hierarchies'][1] ?? null; + $group->setCantonId($cantonId); + $group->setCantonName($this->getLinked($linked, 'groups', $cantonId)); + + $group->setCreatedAt(new DateTimeImmutable($groupJson['created_at'])); + + // TODO expose deleted groups in MiData API + if ($groupJson['deleted_at']) { + $group->setDeletedAt(new DateTimeImmutable($groupJson['deleted_at'])); + } + + /** @var GroupType $gt */ + $gt = $this->groupTypeRepository->findOneBy(['groupType' => $groupJson['type']]); + $group->setGroupType($gt); + + if ($groupJson['links']['parent'] ?? false) { + /** @var Group $pg */ + $pg = $this->groupRepository->find($groupJson['links']['parent']); + if ($pg) { + $group->setParentGroup($pg); + } + } + + foreach ($groupJson['links']['children'] ?? [] as $child) { + $group->addChild($this->fetchGroup($child)); + } + + return $group; + } + + private function getLinked(array $linked, string $rel, string $id) { + return array_filter($linked[$rel] ?? [], function($linkedEntity) use ($id) { + return $linkedEntity['id'] === $id; + })[0] ?? null; + } +} diff --git a/src/Service/PbsApi/Fetcher/PeopleFetcher.php b/src/Service/PbsApi/Fetcher/PeopleFetcher.php new file mode 100644 index 0000000..c28a797 --- /dev/null +++ b/src/Service/PbsApi/Fetcher/PeopleFetcher.php @@ -0,0 +1,127 @@ +em = $em; + $this->personRepository = $this->em->getRepository(Person::class); + $this->groupRepository = $this->em->getRepository(Group::class); + $this->pbsApiService = $pbsApiService; + $this->roleMapper = $roleMapper; + } + + public function fetchAndPersistPeople(string $groupId) + { + $i = 0; + foreach ($this->fetchPeople($groupId) as $person) { + $this->em->persist($person); + $i++; + + if (($i % $this->batchSize) === 0) { + $this->em->flush(); + $this->em->clear(); + } + } + $this->em->flush(); + } + + private function fetchPeople(string $groupId): array + { + $peopleData = $this->pbsApiService->getApiData('/groups/'.$groupId.'/people?filters[role][kind]=with_deleted&range=layer'); + return $this->mapJsonToPeople($peopleData); + } + + private function mapJsonToPeople(array $json): array + { + $peopleJson = $json['people'] ?? []; + $linked = $json['linked'] ?? []; + + $people = []; + foreach ($peopleJson as $personJson) { + $person = $this->personRepository->findOneBy(['id' => $personJson['id']]); + if (!$person) { + $person = new Person(); + $person->setId($personJson['id']); + $metadata = $this->em->getClassMetaData(get_class($person)); + $metadata->setIdGenerator(new AssignedGenerator()); + } + $person->setNickname($personJson['nickname']); + $person->setGender($personJson['gender']); + $person->setAddress($personJson['address']); + $person->setCountry($personJson['country']); + $person->setZip(intval($personJson['zip_code'])); + if ($personJson['birthday']) { + $person->setBirthday(new DateTimeImmutable($personJson['birthday'])); + } + $person->setPbsNumber($personJson['pbs_number']); + if ($personJson['entry_date']) { + $person->setEntryDate(new DateTimeImmutable($personJson['entry_date'])); + } + if ($personJson['leaving_date']) { + $person->setLeavingDate(new DateTimeImmutable($personJson['leaving_date'])); + } else { + $person->setLeavingDate(null); + } + $person->setTown($personJson['town']); + + if ($personJson['primary_group_id']) { + /** @var Group $group */ + $group = $this->groupRepository->find($personJson['primary_group_id']); + if ($group) { + $person->setGroup($group); + } + } + + foreach ($personJson['links']['roles'] ?? [] as $roleId) { + $person->clearPersonRoles(); + $person->addPersonRole($this->roleMapper->mapFromJson($this->getLinked($linked, 'roles', $roleId))); + } + + $people[] = $person; + } + + return $people; + } + + private function getLinked(array $linked, string $rel, string $id) { + return array_filter($linked[$rel] ?? [], function($linkedEntity) use ($id) { + return $linkedEntity['id'] === $id; + })[0] ?? null; + } +} diff --git a/src/Service/PbsApi/Fetcher/PersonRoleMapper.php b/src/Service/PbsApi/Fetcher/PersonRoleMapper.php new file mode 100644 index 0000000..cc7c41a --- /dev/null +++ b/src/Service/PbsApi/Fetcher/PersonRoleMapper.php @@ -0,0 +1,93 @@ +em = $em; + $this->personRoleRepository = $this->em->getRepository(PersonRole::class); + $this->personRepository = $this->em->getRepository(Person::class); + $this->groupRepository = $this->em->getRepository(Group::class); + $this->roleRepository = $this->em->getRepository(Role::class); + $this->pbsApiService = $pbsApiService; + } + + /** + * @param array $roleJson + * @return PersonRole|null + * @throws \Exception + */ + public function mapFromJson(array $roleJson): ?PersonRole + { + $personRole = $this->personRoleRepository->findOneBy(['id' => $roleJson['id']]); + if (!$personRole) { + $personRole = new PersonRole(); + $personRole->setId($roleJson['id']); + $metadata = $this->em->getClassMetaData(get_class($personRole)); + $metadata->setIdGenerator(new AssignedGenerator()); + } + /** @var Person $person */ + $person = $this->personRepository->find($roleJson['person_id']); + if (!$person) { + return null; + } + $personRole->setPerson($person); + + $role = $this->roleRepository->getOneByRoleType($roleJson['type']); + if ($role) { + $personRole->setRole($role); + } + + /** @var Group $group */ + $group = $this->groupRepository->find($roleJson['group_id']); + if ($group) { + $personRole->setGroup($group); + } + + $personRole->setCreatedAt(new DateTimeImmutable($roleJson['created_at'])); + if ($roleJson['deleted_at']) { + $personRole->setDeletedAt(new DateTimeImmutable($roleJson['deleted_at'])); + } + + return $personRole; + } +} diff --git a/src/Service/PbsApiService.php b/src/Service/PbsApiService.php index c557524..5b06f7c 100644 --- a/src/Service/PbsApiService.php +++ b/src/Service/PbsApiService.php @@ -42,4 +42,20 @@ public function getTableData(string $tableName, int $page = null, int $itemsPerP $additionalHeaders = ['X-Token' => $this->apiKey]; return $this->guzzleWrapper->getJson($endpoint, null, $additionalHeaders); } + + /** + * @param string $tableName + * @param int|null $page + * @param int|null $itemsPerPage + * @return array + */ + public function getApiData(string $uri) + { + $endpoint = $this->url . $uri; + $additionalHeaders = [ + 'Authorization' => 'Bearer '. $this->accessToken(), + 'Accept' => 'application/json', + ]; + return $this->guzzleWrapper->getJson($endpoint, null, $additionalHeaders)->getContent(); + } } diff --git a/src/Service/SyncService.php b/src/Service/SyncService.php new file mode 100644 index 0000000..b13304e --- /dev/null +++ b/src/Service/SyncService.php @@ -0,0 +1,57 @@ +em = $em; + $this->pbsApiService = $pbsApiService; + $this->groupFetcher = $groupMapper; + $this->peopleFetcher = $peopleFetcher; + } + + /** + * @param Group $group + * @return void + */ + public function startSync(Group $group) + { + $this->groupFetcher->fetchAndPersistGroup($group->getId()); + $this->peopleFetcher->fetchAndPersistPeople($group->getId()); + } +} diff --git a/translations/messages+intl-icu.de.yml b/translations/messages+intl-icu.de.yml index 781fb08..84dcce9 100644 --- a/translations/messages+intl-icu.de.yml +++ b/translations/messages+intl-icu.de.yml @@ -43,11 +43,13 @@ api: entity: invite: 'Freigabe' group: 'Gruppe' + sync: 'Synchronisation' action: created: 'erstellt' updated: 'aktualisiert' deleted: 'gelöscht' + started: 'begonnen' error: unknown: 'Ein Fehler ist aufgetreten, versuche es später nochmals.' diff --git a/translations/messages+intl-icu.fr.yml b/translations/messages+intl-icu.fr.yml index 16fa7e0..f7b3e83 100644 --- a/translations/messages+intl-icu.fr.yml +++ b/translations/messages+intl-icu.fr.yml @@ -43,11 +43,13 @@ api: entity: invite: "Invité.e.s" group: "Membres" + sync: "Synchronisation" action: created: "Crée" updated: "Actualisé" deleted: "Supprimé" + started: "démarré" error: unknown: "Une erreur s'est produite, réessaie plus tard." diff --git a/translations/messages+intl-icu.it.yml b/translations/messages+intl-icu.it.yml index ce32058..eb339f7 100644 --- a/translations/messages+intl-icu.it.yml +++ b/translations/messages+intl-icu.it.yml @@ -43,11 +43,13 @@ api: entity: invite: 'Liberalizzazione' group: 'Gruppo' + sync: 'Sincronizzazione' action: created: 'creato' updated: 'attualizzato' deleted: 'cancellato' + started: 'iniziata' error: unknown: 'Si è verificato un errore, prova di nuovo più tardi.' From a55442522762817db9573257181c2b12fbcb7ecd Mon Sep 17 00:00:00 2001 From: Carlo Beltrame Date: Tue, 24 Aug 2021 13:31:38 +0200 Subject: [PATCH 02/12] Set up OAuth flow for syncing a single Abteilung --- config/routing_api.yml | 2 +- config/services.yaml | 2 ++ src/Controller/Api/SyncController.php | 36 ++++---------------- src/Service/PbsApi/Fetcher/GroupFetcher.php | 14 ++++---- src/Service/PbsApi/Fetcher/PeopleFetcher.php | 8 ++--- src/Service/PbsApiService.php | 4 +-- src/Service/PbsAuthService.php | 11 +++--- src/Service/Security/PbsAuthenticator.php | 2 +- src/Service/SyncService.php | 16 +++------ 9 files changed, 36 insertions(+), 59 deletions(-) diff --git a/config/routing_api.yml b/config/routing_api.yml index 40816a4..f6fbe32 100644 --- a/config/routing_api.yml +++ b/config/routing_api.yml @@ -28,7 +28,7 @@ invite_delete: sync_start: path: /groups/{groupId}/sync methods: POST - controller: App\Controller\Api\SyncController:startSync + controller: App\Controller\Api\SyncController::startSync # Filter data routes filter_data: diff --git a/config/services.yaml b/config/services.yaml index 80068e5..319b079 100644 --- a/config/services.yaml +++ b/config/services.yaml @@ -96,6 +96,8 @@ services: $pbsCallbackUrl: '%env(PBS_CALLBACK_URL)%' $specialAccessEmails: '%env(SPECIAL_ACCESS)%' + App\Service\SyncService: + App\Service\Aggregator\DemographicGroupAggregator: tags: - { name: 'widget.aggregator', key: 'widget.demographic-group' } diff --git a/src/Controller/Api/SyncController.php b/src/Controller/Api/SyncController.php index 6112dfb..03a7fd7 100644 --- a/src/Controller/Api/SyncController.php +++ b/src/Controller/Api/SyncController.php @@ -7,11 +7,13 @@ use App\Entity\Invite; use App\Exception\ApiException; use App\Service\InviteService; +use App\Service\SyncService; use Sensio\Bundle\FrameworkExtraBundle\Configuration\IsGranted; use Sensio\Bundle\FrameworkExtraBundle\Configuration\ParamConverter; use Symfony\Bundle\FrameworkBundle\Controller\AbstractController; use Symfony\Component\HttpFoundation\JsonResponse; use Symfony\Component\HttpFoundation\Request; +use Symfony\Component\HttpKernel\Exception\BadRequestHttpException; use Symfony\Component\Serializer\Normalizer\AbstractNormalizer; use Symfony\Component\Serializer\SerializerInterface; use Symfony\Component\Validator\Validator\ValidatorInterface; @@ -58,35 +60,11 @@ public function startSync( $action = $this->translator->trans('api.action.started'); $entity = $this->translator->trans('api.entity.sync'); $message = $this->translator->trans('api.success', ['entityName' => $entity, 'action' => $action]); - $this->syncService->startSync($group); + $code = $request->toArray()['code'] ?? null; + if ($code === null) { + throw new BadRequestHttpException('Authorization code is missing'); + } + $this->syncService->startSync($group, $request->toArray()['code'] ?? ''); return $this->json($message, JsonResponse::HTTP_CREATED); } - - /** - * @param Group $group - * @return JsonResponse - * @ParamConverter(name="group", options={"mapping":{"groupId":"id"}}) - * @IsGranted("view", subject="group") - */ - public function getInvites(Group $group) - { - return $this->json($this->inviteService->getAllInvites($group)); - } - - /** - * @param Group $group - * @param Invite $invite - * @return JsonResponse - * @ParamConverter(name="group", options={"mapping":{"groupId":"id"}}) - * @ParamConverter(name="invite", options={"mapping":{"inviteId":"id"}}) - * @IsGranted("delete", subject="group") - */ - public function deleteInvite(Group $group, Invite $invite) - { - $this->inviteService->deleteInvite($invite, $group); - $action = $this->translator->trans('api.action.deleted'); - $entity = $this->translator->trans('api.entity.invite'); - $message = $this->translator->trans('api.success', ['entityName' => $entity, 'action' => $action]); - return $this->json($message); - } } diff --git a/src/Service/PbsApi/Fetcher/GroupFetcher.php b/src/Service/PbsApi/Fetcher/GroupFetcher.php index 325ca40..5f5763c 100644 --- a/src/Service/PbsApi/Fetcher/GroupFetcher.php +++ b/src/Service/PbsApi/Fetcher/GroupFetcher.php @@ -37,19 +37,19 @@ public function __construct(EntityManagerInterface $em, PbsApiService $pbsApiSer $this->pbsApiService = $pbsApiService; } - public function fetchAndPersistGroup(string $id) + public function fetchAndPersistGroup(string $id, string $accessToken) { - $this->em->persist($this->fetchGroup($id)); + $this->em->persist($this->fetchGroup($id, $accessToken)); $this->em->flush(); } - private function fetchGroup(string $id): Group + private function fetchGroup(string $id, string $accessToken): Group { - $groupData = $this->pbsApiService->getApiData('/groups/'.$id); - return $this->mapJsonToGroup($groupData); + $groupData = $this->pbsApiService->getApiData('/groups/'.$id, $accessToken); + return $this->mapJsonToGroup($groupData, $accessToken); } - private function mapJsonToGroup(array $json): Group + private function mapJsonToGroup(array $json, string $accessToken): Group { $groupJson = $json['groups'][0] ?? []; $linked = $json['linked'] ?? []; @@ -87,7 +87,7 @@ private function mapJsonToGroup(array $json): Group } foreach ($groupJson['links']['children'] ?? [] as $child) { - $group->addChild($this->fetchGroup($child)); + $group->addChild($this->fetchGroup($child, $accessToken)); } return $group; diff --git a/src/Service/PbsApi/Fetcher/PeopleFetcher.php b/src/Service/PbsApi/Fetcher/PeopleFetcher.php index c28a797..221a089 100644 --- a/src/Service/PbsApi/Fetcher/PeopleFetcher.php +++ b/src/Service/PbsApi/Fetcher/PeopleFetcher.php @@ -46,10 +46,10 @@ public function __construct(EntityManagerInterface $em, PbsApiService $pbsApiSer $this->roleMapper = $roleMapper; } - public function fetchAndPersistPeople(string $groupId) + public function fetchAndPersistPeople(string $groupId, string $accessToken) { $i = 0; - foreach ($this->fetchPeople($groupId) as $person) { + foreach ($this->fetchPeople($groupId, $accessToken) as $person) { $this->em->persist($person); $i++; @@ -61,9 +61,9 @@ public function fetchAndPersistPeople(string $groupId) $this->em->flush(); } - private function fetchPeople(string $groupId): array + private function fetchPeople(string $groupId, string $accessToken): array { - $peopleData = $this->pbsApiService->getApiData('/groups/'.$groupId.'/people?filters[role][kind]=with_deleted&range=layer'); + $peopleData = $this->pbsApiService->getApiData('/groups/'.$groupId.'/people?filters[role][kind]=with_deleted&range=layer', $accessToken); return $this->mapJsonToPeople($peopleData); } diff --git a/src/Service/PbsApiService.php b/src/Service/PbsApiService.php index 5b06f7c..96b0a48 100644 --- a/src/Service/PbsApiService.php +++ b/src/Service/PbsApiService.php @@ -49,11 +49,11 @@ public function getTableData(string $tableName, int $page = null, int $itemsPerP * @param int|null $itemsPerPage * @return array */ - public function getApiData(string $uri) + public function getApiData(string $uri, string $accessToken) { $endpoint = $this->url . $uri; $additionalHeaders = [ - 'Authorization' => 'Bearer '. $this->accessToken(), + 'Authorization' => 'Bearer '. $accessToken, 'Accept' => 'application/json', ]; return $this->guzzleWrapper->getJson($endpoint, null, $additionalHeaders)->getContent(); diff --git a/src/Service/PbsAuthService.php b/src/Service/PbsAuthService.php index 0a196c7..e79d2a9 100644 --- a/src/Service/PbsAuthService.php +++ b/src/Service/PbsAuthService.php @@ -105,11 +105,12 @@ public function __construct( /** * @param string $code * @param string $locale + * @param string|null $action * @return PbsUserDTO */ - public function getUser(string $code, string $locale): PbsUserDTO + public function getUser(string $code, string $locale, ?string $action = null): PbsUserDTO { - $token = $this->getTokenUsingCode($code); + $token = $this->getTokenUsingCode($code, $action); $user = $this->getUserWithToken($token); switch ($this->environment) { case 'dev': @@ -139,15 +140,17 @@ public function getUser(string $code, string $locale): PbsUserDTO /** * @param string $code + * @param string|null $action * @return string */ - private function getTokenUsingCode(string $code): string + public function getTokenUsingCode(string $code, ?string $action = null): string { + $callbackUrl = $this->pbsCallbackUrl . ($action ? '?action='.$action : ''); $body = [ 'grant_type' => 'authorization_code', 'client_id' => $this->pbsClientId, 'client_secret' => $this->pbsClientSecret, - 'redirect_uri' => $this->pbsCallbackUrl, + 'redirect_uri' => $callbackUrl, 'code' => $code ]; $response = $this->guzzleWrapper->postJson($this->pbsUrl . '/oauth/token', json_encode($body)); diff --git a/src/Service/Security/PbsAuthenticator.php b/src/Service/Security/PbsAuthenticator.php index bb1f82b..d545a91 100644 --- a/src/Service/Security/PbsAuthenticator.php +++ b/src/Service/Security/PbsAuthenticator.php @@ -97,7 +97,7 @@ public function getUser($credentials, UserProviderInterface $userProvider) return null; } - return $this->pbsAuthService->getUser($credentials['code'], $this->request->getLocale()); + return $this->pbsAuthService->getUser($credentials['code'], $this->request->getLocale(), $credentials['action'] ?? null); } /** diff --git a/src/Service/SyncService.php b/src/Service/SyncService.php index b13304e..78b697a 100644 --- a/src/Service/SyncService.php +++ b/src/Service/SyncService.php @@ -2,18 +2,10 @@ namespace App\Service; -use App\DTO\Mapper\InviteMapper; -use App\DTO\Model\InviteDTO; use App\Entity\Group; -use App\Entity\GroupType; -use App\Entity\Invite; -use App\Repository\InviteRepository; use App\Service\PbsApi\Fetcher\GroupFetcher; use App\Service\PbsApi\Fetcher\PeopleFetcher; -use DateTime; use Doctrine\ORM\EntityManagerInterface; -use Doctrine\ORM\Id\AssignedGenerator; -use Symfony\Component\HttpKernel\Exception\NotFoundHttpException; class SyncService { @@ -49,9 +41,11 @@ public function __construct(EntityManagerInterface $em, PbsApiService $pbsApiSer * @param Group $group * @return void */ - public function startSync(Group $group) + public function startSync(Group $group, $accessToken) { - $this->groupFetcher->fetchAndPersistGroup($group->getId()); - $this->peopleFetcher->fetchAndPersistPeople($group->getId()); + $this->groupFetcher->fetchAndPersistGroup($group->getId(), $accessToken); + $this->peopleFetcher->fetchAndPersistPeople($group->getId(), $accessToken); + + // TODO run aggregations here, but only for the fetched group } } From 3bf0e0acca708c3e0330244ef65be6c44052e1c2 Mon Sep 17 00:00:00 2001 From: Carlo Beltrame Date: Sat, 25 Sep 2021 17:41:03 +0200 Subject: [PATCH 03/12] Remove all syncing of critical personal data from nightly import --- README.md | 4 +- run-import.sh | 4 +- src/Command/FetchDataCommand.php | 2 +- src/Command/ImportFromJsonCommand.php | 512 ++++++++++++-------------- 4 files changed, 245 insertions(+), 277 deletions(-) diff --git a/README.md b/README.md index c3929d6..79ed47f 100644 --- a/README.md +++ b/README.md @@ -66,8 +66,8 @@ Make sure you execute all the migrations so the schema and tables are in sync wi #### Import Data -To run the import you can execute the `run-import.sh` script inside the healthcheck-core service container. -Notice: This might take a while to finish. +To manually run the nightly import of all non-personal data, you can execute the `run-import.sh` script inside the +healthcheck-core service container. `docker exec healthcheck-core-local ./run-import.sh` diff --git a/run-import.sh b/run-import.sh index ac8618d..ce90b05 100755 --- a/run-import.sh +++ b/run-import.sh @@ -6,5 +6,5 @@ set -e # Run import commands from symfony project /usr/local/bin/php /srv/bin/console app:fetch-data /usr/local/bin/php /srv/bin/console app:import-data -/usr/local/bin/php /srv/bin/console app:map-peoples-addresses -/usr/local/bin/php /srv/bin/console app:aggregate-data +#/usr/local/bin/php /srv/bin/console app:map-peoples-addresses +#/usr/local/bin/php /srv/bin/console app:aggregate-data diff --git a/src/Command/FetchDataCommand.php b/src/Command/FetchDataCommand.php index 69b30f6..f3209f0 100644 --- a/src/Command/FetchDataCommand.php +++ b/src/Command/FetchDataCommand.php @@ -14,7 +14,7 @@ class FetchDataCommand extends StatisticsCommand public const STAT_TABLE_HEADERS = ['Table Name', 'Duration (s)', 'Items Fetched']; public const STAT_TOTAL_TABLE_HEADERS = ['Total Duration (s)', 'Total Items Fetched']; - private const PAGINATED = ['courses', 'camps', 'participations', 'qualifications']; + private const PAGINATED = []; private const NOT_PAGINATED = ['group_types', 'role_types', 'participation_types', 'j_s_kinds', 'camp_states', 'qualification_kinds', 'event_kinds']; diff --git a/src/Command/ImportFromJsonCommand.php b/src/Command/ImportFromJsonCommand.php index 0031189..fc2bfb0 100644 --- a/src/Command/ImportFromJsonCommand.php +++ b/src/Command/ImportFromJsonCommand.php @@ -2,29 +2,17 @@ namespace App\Command; -use App\Entity\EventGroup; -use App\Entity\YouthSportType; -use App\Entity\Camp; use App\Entity\CampState; -use App\Entity\Course; -use App\Entity\Event; -use App\Entity\EventDate; use App\Entity\EventType; use App\Entity\EventTypeQualificationType; -use App\Entity\Group; use App\Entity\GroupType; -use App\Entity\Person; -use App\Entity\PersonEvent; use App\Entity\PersonEventType; -use App\Entity\PersonQualification; -use App\Entity\PersonRole; use App\Entity\QualificationType; use App\Entity\Role; +use App\Entity\YouthSportType; use App\Model\CommandStatistics; use App\Repository\PersonRepository; -use DateTimeImmutable; use Doctrine\DBAL\ConnectionException; -use Doctrine\DBAL\DBALException; use Doctrine\ORM\EntityManagerInterface; use Doctrine\ORM\Id\AssignedGenerator; use Exception; @@ -93,7 +81,6 @@ protected function execute(InputInterface $input, OutputInterface $output): int $this->em->getConnection()->beginTransaction(); try { $output->writeln(['Start importing...']); - $this->cleaningUpEntities($output); $this->importRoleTypes($output); $this->importGroupTypes($output); $this->importQualificationTypes($output); @@ -101,10 +88,6 @@ protected function execute(InputInterface $input, OutputInterface $output): int $this->importYouthSportTypes($output); $this->importCampStates($output); $this->importPersonEventTypes($output); - $this->importCourses($output); - $this->importCamps($output); - $this->importParticipations($output); - $this->importQualifications($output); $this->em->getConnection()->commit(); @@ -118,21 +101,6 @@ protected function execute(InputInterface $input, OutputInterface $output): int return 0; } - /** - * @param OutputInterface $output - * @throws \Doctrine\DBAL\Exception - */ - private function cleaningUpEntities(OutputInterface $output) - { - $connection = $this->em->getConnection(); - - $connection->executeQuery("DELETE FROM midata_event_date"); - - $connection->executeQuery("DELETE FROM midata_event_group"); - - $output->writeln("cleaned some entities from the db"); - } - /** * @param OutputInterface $output */ @@ -358,245 +326,245 @@ private function importPersonEventTypes(OutputInterface $output) $output->writeln([sprintf('%s rows imported from participation_types.json', $i)]); } - /** - * @param OutputInterface $output - * @throws Exception - */ - private function importCourses(OutputInterface $output) - { - $start = microtime(true); - $courses = JsonMachine::fromFile(sprintf('%s/courses.json', $this->params->get('import_data_dir'))); - $i = 0; - foreach ($courses as $c) { - $course = $this->em->getRepository(Course::class)->findOneBy(['id' => $c['id']]); - if (!$course) { - $course = new Course(); - $course->setId($c['id']); - $metadata = $this->em->getClassMetaData(get_class($course)); - $metadata->setIdGenerator(new AssignedGenerator()); - } - - /** @var EventType $eventType */ - $eventType = $this->em->getRepository(EventType::class)->find($c['kind_id']); - $course->setEventType($eventType); - - if (isset($c['name'])) { - $course->setName($c['name']); - } - - if ($c['groups']) { - foreach ($c['groups'] as $g) { - $group = $this->em->getRepository(Group::class)->find($g['id']); - if ($group) { - $eventGroup = new EventGroup(); - $eventGroup->setGroup($group); - $eventGroup->setEvent($course); - $course->addGroup($eventGroup); - } - } - } - - if ($c['dates']) { - foreach ($c['dates'] as $date) { - $eventDate = new EventDate(); - $eventDate->setEvent($course); - $eventDate->setStartAt(new DateTimeImmutable($date['start_at'])); - $eventDate->setEndAt(new DateTimeImmutable($date['finish_at'])); - $this->em->persist($eventDate); - } - } - - $this->em->persist($course); - $this->em->flush(); - $i++; - } - $timeElapsed = microtime(true) - $start; - $this->stats[] = ['courses.json', $timeElapsed, $i]; - $output->writeln([sprintf('%s rows imported from courses.json', $i)]); - } - - /** - * @param OutputInterface $output - * @throws Exception - */ - private function importCamps(OutputInterface $output) - { - $start = microtime(true); - $camps = JsonMachine::fromFile(sprintf('%s/camps.json', $this->params->get('import_data_dir'))); - $i = 0; - - $groupData = $this->em->getRepository(Group::class)->findAll(); - $groups = []; - - /** @var Group $group */ - foreach ($groupData as $group) { - $groups[$group->getId()] = $group; - } - - foreach ($camps as $c) { - $camp = $this->em->getRepository(Camp::class)->findOneBy(['id' => $c['id']]); - if (!$camp) { - $camp = new Camp(); - $camp->setId($c['id']); - $metadata = $this->em->getClassMetaData(get_class($camp)); - $metadata->setIdGenerator(new AssignedGenerator()); - } - $camp->setState($c['state']); - $camp->setLocation(substr($c['location'], 0, 255)); - - if (isset($c['name'])) { - $camp->setName($c['name']); - } - - /** @var YouthSportType $ageSportType */ - $ageSportType = $this->em->getRepository(YouthSportType::class)->findOneBy(['type' => $c['j_s_kind']]); - $camp->setYouthSportType($ageSportType); - - if ($c['dates']) { - foreach ($c['dates'] as $date) { - $eventDate = new EventDate(); - $eventDate->setEvent($camp); - $eventDate->setStartAt(new DateTimeImmutable($date['start_at'])); - $eventDate->setEndAt(new DateTimeImmutable($date['finish_at'])); - $this->em->persist($eventDate); - } - } - - if ($c['groups']) { - foreach ($c['groups'] as $g) { - if (array_key_exists($g['id'], $groups)) { - $group = $groups[$g['id']]; - $eventGroup = new EventGroup(); - $eventGroup->setGroup($group); - $eventGroup->setEvent($camp); - $camp->addGroup($eventGroup); - } - } - } - - $this->em->persist($camp); - if ($i % 10) { - $this->em->flush(); - } - $i++; - } - $this->em->flush(); - - $timeElapsed = microtime(true) - $start; - $this->stats[] = ['camps.json', $timeElapsed, $i]; - $output->writeln([sprintf('%s rows imported from camps.json', $i)]); - } - - /** - * @param OutputInterface $output - */ - private function importParticipations(OutputInterface $output) - { - $start = microtime(true); - $participations = JsonMachine::fromFile( - sprintf('%s/participations.json', $this->params->get('import_data_dir')) - ); - $i = 0; - foreach ($participations as $participation) { - $personEvent = $this->em->getRepository(PersonEvent::class)->findOneBy( - ['id' => $participation['id']] - ); - if (!$personEvent) { - $personEvent = new PersonEvent(); - $personEvent->setId($participation['id']); - $metadata = $this->em->getClassMetaData(get_class($personEvent)); - $metadata->setIdGenerator(new AssignedGenerator()); - } - $person = $this->em->getRepository(Person::class)->find($participation['person_id']); - $event = $this->em->getRepository(Event::class)->find($participation['event_id']); - $personEvent->setQualified($participation['qualified']); - - if ($participation['roles']) { - foreach ($participation['roles'] as $role) { - $personEventType = $this->em->getRepository(PersonEventType::class)->findOneBy( - ['type' => $role['type']] - ); - if ($personEventType) { - $personEvent->addPersonEventType($personEventType); - } - } - } - - if ($person && $event) { - $personEvent->setPerson($person); - $personEvent->setEvent($event); - $this->em->persist($personEvent); - } - $i++; - - if (($i % $this->batchSize) === 0) { - $this->em->flush(); - $this->em->clear(); - } - } - $this->em->flush(); - - $timeElapsed = microtime(true) - $start; - $this->stats[] = ['participations.json', $timeElapsed, $i]; - $output->writeln([sprintf('%s rows imported from participations.json', $i)]); - } - - /** - * @param OutputInterface $output - * @throws Exception - */ - private function importQualifications(OutputInterface $output) - { - $start = microtime(true); - $qualifications = JsonMachine::fromFile( - sprintf('%s/qualifications.json', $this->params->get('import_data_dir')) - ); - $i = 0; - foreach ($qualifications as $qualification) { - $personQualification = $this->em->getRepository(PersonQualification::class)->findOneBy( - ['id' => $qualification['id']] - ); - if (!$personQualification) { - $personQualification = new PersonQualification(); - $personQualification->setId($qualification['id']); - $metadata = $this->em->getClassMetaData(get_class($personQualification)); - $metadata->setIdGenerator(new AssignedGenerator()); - } - $personQualification->setEventOrigin($qualification['origin']); - $personQualification->setStartAt(new DateTimeImmutable($qualification['start_at'])); - $personQualification->setEndAt( - $qualification['finish_at'] ? new DateTimeImmutable($qualification['finish_at']) : null - ); - - $person = $this->em->getRepository(Person::class)->find($qualification['person_id']); - if (!$person) { - continue; - } - - $qualificationType = $this->em->getRepository(QualificationType::class)->find( - $qualification['qualification_kind_id'] - ); - - $personQualification->setPerson($person); - - if ($qualificationType) { - $personQualification->setQualificationType($qualificationType); - } - - $this->em->persist($personQualification); - $i++; - - if (($i % $this->batchSize) === 0) { - $this->em->flush(); - $this->em->clear(); - } - } - $this->em->flush(); - - $timeElapsed = microtime(true) - $start; - $this->stats[] = ['qualifications.json', $timeElapsed, $i]; - $output->writeln([sprintf('%s rows imported from qualifications.json', $i)]); - } +// /** +// * @param OutputInterface $output +// * @throws Exception +// */ +// private function importCourses(OutputInterface $output) +// { +// $start = microtime(true); +// $courses = JsonMachine::fromFile(sprintf('%s/courses.json', $this->params->get('import_data_dir'))); +// $i = 0; +// foreach ($courses as $c) { +// $course = $this->em->getRepository(Course::class)->findOneBy(['id' => $c['id']]); +// if (!$course) { +// $course = new Course(); +// $course->setId($c['id']); +// $metadata = $this->em->getClassMetaData(get_class($course)); +// $metadata->setIdGenerator(new AssignedGenerator()); +// } +// +// /** @var EventType $eventType */ +// $eventType = $this->em->getRepository(EventType::class)->find($c['kind_id']); +// $course->setEventType($eventType); +// +// if (isset($c['name'])) { +// $course->setName($c['name']); +// } +// +// if ($c['groups']) { +// foreach ($c['groups'] as $g) { +// $group = $this->em->getRepository(Group::class)->find($g['id']); +// if ($group) { +// $eventGroup = new EventGroup(); +// $eventGroup->setGroup($group); +// $eventGroup->setEvent($course); +// $course->addGroup($eventGroup); +// } +// } +// } +// +// if ($c['dates']) { +// foreach ($c['dates'] as $date) { +// $eventDate = new EventDate(); +// $eventDate->setEvent($course); +// $eventDate->setStartAt(new DateTimeImmutable($date['start_at'])); +// $eventDate->setEndAt(new DateTimeImmutable($date['finish_at'])); +// $this->em->persist($eventDate); +// } +// } +// +// $this->em->persist($course); +// $this->em->flush(); +// $i++; +// } +// $timeElapsed = microtime(true) - $start; +// $this->stats[] = ['courses.json', $timeElapsed, $i]; +// $output->writeln([sprintf('%s rows imported from courses.json', $i)]); +// } +// +// /** +// * @param OutputInterface $output +// * @throws Exception +// */ +// private function importCamps(OutputInterface $output) +// { +// $start = microtime(true); +// $camps = JsonMachine::fromFile(sprintf('%s/camps.json', $this->params->get('import_data_dir'))); +// $i = 0; +// +// $groupData = $this->em->getRepository(Group::class)->findAll(); +// $groups = []; +// +// /** @var Group $group */ +// foreach ($groupData as $group) { +// $groups[$group->getId()] = $group; +// } +// +// foreach ($camps as $c) { +// $camp = $this->em->getRepository(Camp::class)->findOneBy(['id' => $c['id']]); +// if (!$camp) { +// $camp = new Camp(); +// $camp->setId($c['id']); +// $metadata = $this->em->getClassMetaData(get_class($camp)); +// $metadata->setIdGenerator(new AssignedGenerator()); +// } +// $camp->setState($c['state']); +// $camp->setLocation(substr($c['location'], 0, 255)); +// +// if (isset($c['name'])) { +// $camp->setName($c['name']); +// } +// +// /** @var YouthSportType $ageSportType */ +// $ageSportType = $this->em->getRepository(YouthSportType::class)->findOneBy(['type' => $c['j_s_kind']]); +// $camp->setYouthSportType($ageSportType); +// +// if ($c['dates']) { +// foreach ($c['dates'] as $date) { +// $eventDate = new EventDate(); +// $eventDate->setEvent($camp); +// $eventDate->setStartAt(new DateTimeImmutable($date['start_at'])); +// $eventDate->setEndAt(new DateTimeImmutable($date['finish_at'])); +// $this->em->persist($eventDate); +// } +// } +// +// if ($c['groups']) { +// foreach ($c['groups'] as $g) { +// if (array_key_exists($g['id'], $groups)) { +// $group = $groups[$g['id']]; +// $eventGroup = new EventGroup(); +// $eventGroup->setGroup($group); +// $eventGroup->setEvent($camp); +// $camp->addGroup($eventGroup); +// } +// } +// } +// +// $this->em->persist($camp); +// if ($i % 10) { +// $this->em->flush(); +// } +// $i++; +// } +// $this->em->flush(); +// +// $timeElapsed = microtime(true) - $start; +// $this->stats[] = ['camps.json', $timeElapsed, $i]; +// $output->writeln([sprintf('%s rows imported from camps.json', $i)]); +// } +// +// /** +// * @param OutputInterface $output +// */ +// private function importParticipations(OutputInterface $output) +// { +// $start = microtime(true); +// $participations = JsonMachine::fromFile( +// sprintf('%s/participations.json', $this->params->get('import_data_dir')) +// ); +// $i = 0; +// foreach ($participations as $participation) { +// $personEvent = $this->em->getRepository(PersonEvent::class)->findOneBy( +// ['id' => $participation['id']] +// ); +// if (!$personEvent) { +// $personEvent = new PersonEvent(); +// $personEvent->setId($participation['id']); +// $metadata = $this->em->getClassMetaData(get_class($personEvent)); +// $metadata->setIdGenerator(new AssignedGenerator()); +// } +// $person = $this->em->getRepository(Person::class)->find($participation['person_id']); +// $event = $this->em->getRepository(Event::class)->find($participation['event_id']); +// $personEvent->setQualified($participation['qualified']); +// +// if ($participation['roles']) { +// foreach ($participation['roles'] as $role) { +// $personEventType = $this->em->getRepository(PersonEventType::class)->findOneBy( +// ['type' => $role['type']] +// ); +// if ($personEventType) { +// $personEvent->addPersonEventType($personEventType); +// } +// } +// } +// +// if ($person && $event) { +// $personEvent->setPerson($person); +// $personEvent->setEvent($event); +// $this->em->persist($personEvent); +// } +// $i++; +// +// if (($i % $this->batchSize) === 0) { +// $this->em->flush(); +// $this->em->clear(); +// } +// } +// $this->em->flush(); +// +// $timeElapsed = microtime(true) - $start; +// $this->stats[] = ['participations.json', $timeElapsed, $i]; +// $output->writeln([sprintf('%s rows imported from participations.json', $i)]); +// } +// +// /** +// * @param OutputInterface $output +// * @throws Exception +// */ +// private function importQualifications(OutputInterface $output) +// { +// $start = microtime(true); +// $qualifications = JsonMachine::fromFile( +// sprintf('%s/qualifications.json', $this->params->get('import_data_dir')) +// ); +// $i = 0; +// foreach ($qualifications as $qualification) { +// $personQualification = $this->em->getRepository(PersonQualification::class)->findOneBy( +// ['id' => $qualification['id']] +// ); +// if (!$personQualification) { +// $personQualification = new PersonQualification(); +// $personQualification->setId($qualification['id']); +// $metadata = $this->em->getClassMetaData(get_class($personQualification)); +// $metadata->setIdGenerator(new AssignedGenerator()); +// } +// $personQualification->setEventOrigin($qualification['origin']); +// $personQualification->setStartAt(new DateTimeImmutable($qualification['start_at'])); +// $personQualification->setEndAt( +// $qualification['finish_at'] ? new DateTimeImmutable($qualification['finish_at']) : null +// ); +// +// $person = $this->em->getRepository(Person::class)->find($qualification['person_id']); +// if (!$person) { +// continue; +// } +// +// $qualificationType = $this->em->getRepository(QualificationType::class)->find( +// $qualification['qualification_kind_id'] +// ); +// +// $personQualification->setPerson($person); +// +// if ($qualificationType) { +// $personQualification->setQualificationType($qualificationType); +// } +// +// $this->em->persist($personQualification); +// $i++; +// +// if (($i % $this->batchSize) === 0) { +// $this->em->flush(); +// $this->em->clear(); +// } +// } +// $this->em->flush(); +// +// $timeElapsed = microtime(true) - $start; +// $this->stats[] = ['qualifications.json', $timeElapsed, $i]; +// $output->writeln([sprintf('%s rows imported from qualifications.json', $i)]); +// } public function getStats(): CommandStatistics { From 09fbe4191346a9145f0d68867f05ce56752c6ed8 Mon Sep 17 00:00:00 2001 From: Carlo Beltrame Date: Sat, 25 Sep 2021 17:42:08 +0200 Subject: [PATCH 04/12] Refactor PbsAuthService to only rely on personal data from the OAuth profile --- src/DTO/Mapper/GroupMapper.php | 12 ++ src/DTO/Mapper/GroupTypeMapper.php | 3 +- src/DTO/Mapper/PbsUserMapper.php | 8 +- src/DTO/Model/PbsUserDTO.php | 36 ++++++ src/Repository/GroupRepository.php | 22 ---- src/Service/PbsAuthService.php | 176 ++++++----------------------- 6 files changed, 89 insertions(+), 168 deletions(-) diff --git a/src/DTO/Mapper/GroupMapper.php b/src/DTO/Mapper/GroupMapper.php index 67af5d8..3b494af 100644 --- a/src/DTO/Mapper/GroupMapper.php +++ b/src/DTO/Mapper/GroupMapper.php @@ -21,4 +21,16 @@ public static function createFromEntity(Group $group, string $locale): GroupDTO $groupDTO->setGroupType(GroupTypeMapper::createGroupTypeFromEntity($group->getGroupType(), $locale)); return $groupDTO; } + + public static function createFromMidataOauthProfile(array $group, GroupType $groupType, string $locale): GroupDTO + { + $groupDTO = new GroupDTO(); + $groupDTO->setId($group['group_id']); + $groupDTO->setName($group['group_name']); + $groupDTO->setCantonName('' /* TODO really needed? */ ); + $groupDTO->setCreatedAt('' /* TODO really needed? */); + $groupDTO->setDeletedAt('' /* TODO really needed? */); + $groupDTO->setGroupType(GroupTypeMapper::createGroupTypeFromEntity($groupType, $locale)); + return $groupDTO; + } } diff --git a/src/DTO/Mapper/GroupTypeMapper.php b/src/DTO/Mapper/GroupTypeMapper.php index e75eaf5..8c36b2d 100644 --- a/src/DTO/Mapper/GroupTypeMapper.php +++ b/src/DTO/Mapper/GroupTypeMapper.php @@ -14,8 +14,7 @@ public static function createGroupTypeFromEntity(GroupType $groupType, string $l $groupTypeDTO->setId($groupType->getId()); self::setLabelForLocale($groupTypeDTO, $groupType, $locale); $groupTypeDTO->setGroupType($groupType->getGroupType()); - $color = array_key_exists($groupType->getGroupType(), WidgetDataProvider::RELEVANT_SUB_GROUP_TYPES) ? - WidgetDataProvider::GROUP_TYPE_COLORS[$groupType->getGroupType()] : ''; + $color = WidgetDataProvider::GROUP_TYPE_COLORS[$groupType->getGroupType()] ?? ''; $groupTypeDTO->setColor($color); return $groupTypeDTO; } diff --git a/src/DTO/Mapper/PbsUserMapper.php b/src/DTO/Mapper/PbsUserMapper.php index 823dafc..2a0620a 100644 --- a/src/DTO/Mapper/PbsUserMapper.php +++ b/src/DTO/Mapper/PbsUserMapper.php @@ -17,7 +17,7 @@ public static function createFromArray(array $user): PbsUserDTO $user['email'], $user['first_name'], $user['last_name'], - $user['nickname'] + $user['nickname'] ?? '' ); $pbsUser->setGender($user['gender'] ?? ''); @@ -27,10 +27,8 @@ public static function createFromArray(array $user): PbsUserDTO $pbsUser->setTown($user['town'] ?? ''); $pbsUser->setZipCode($user['zip_code'] ?? ''); $pbsUser->setCorrespondenceLanguage($user['correspondence_language'] ?? ''); - - foreach ($user['roles'] ?? [] as $role) { - $pbsUser->addPersonRole(PbsRoleMapper::createFromArray($role)); - } + $pbsUser->setSyncableGroups($user['syncable_groups'] ?? []); + $pbsUser->setReadableGroups($user['readable_groups'] ?? []); return $pbsUser; } diff --git a/src/DTO/Model/PbsUserDTO.php b/src/DTO/Model/PbsUserDTO.php index 8f98e06..641b946 100644 --- a/src/DTO/Model/PbsUserDTO.php +++ b/src/DTO/Model/PbsUserDTO.php @@ -34,6 +34,10 @@ class PbsUserDTO implements UserInterface private $roles; /** @var array|GroupDTO[] */ private $groups; + /** @var array */ + private $syncableGroups; + /** @var array */ + private $readableGroups; /** * PbsUserDTO constructor. @@ -166,6 +170,22 @@ public function getPersonRoles() return $this->roles; } + /** + * @return array + */ + public function getSyncableGroups(): array + { + return $this->syncableGroups; + } + + /** + * @return array + */ + public function getReadableGroups(): array + { + return $this->readableGroups; + } + /** * @param string $birthday */ @@ -248,6 +268,22 @@ public function addGroup(GroupDTO $group) $this->groups[] = $group; } + /** + * @param array $syncableGroups + */ + public function setSyncableGroups(array $syncableGroups): void + { + $this->syncableGroups = $syncableGroups; + } + + /** + * @param array $readableGroups + */ + public function setReadableGroups(array $readableGroups): void + { + $this->readableGroups = $readableGroups; + } + /** * @inheritDoc */ diff --git a/src/Repository/GroupRepository.php b/src/Repository/GroupRepository.php index 7daa665..568d9e0 100644 --- a/src/Repository/GroupRepository.php +++ b/src/Repository/GroupRepository.php @@ -32,28 +32,6 @@ public function findParentGroupById(int $groupId) ->getResult(); } - public function findParentGroupsForPerson(int $personId) - { - return $this->createQueryBuilder('g') - ->join('g.groupType', 'groupType') - ->join('g.personRoles', 'personRoles') - ->join('personRoles.person', 'person') - ->join('personRoles.role', 'role') - ->where('groupType.groupType = :name') - ->andWhere('role.roleType IN (:roleTypes)') - ->andWhere('person.id = :personId') - ->andWhere('personRoles.deletedAt IS NULL') - ->setParameter('name', 'Group::Abteilung', ParameterType::STRING) - ->setParameter( - 'roleTypes', - array_merge(WidgetAggregator::$mainGroupRoleTypes, ['Group::Abteilung::Coach']), - Connection::PARAM_STR_ARRAY - ) - ->setParameter('personId', $personId, ParameterType::INTEGER) - ->getQuery() - ->getResult(); - } - public function findAllParentGroups() { return $this->createQueryBuilder('g') diff --git a/src/Service/PbsAuthService.php b/src/Service/PbsAuthService.php index e79d2a9..5272f90 100644 --- a/src/Service/PbsAuthService.php +++ b/src/Service/PbsAuthService.php @@ -6,11 +6,9 @@ use App\DTO\Mapper\PbsUserMapper; use App\DTO\Model\GroupDTO; use App\DTO\Model\PbsUserDTO; -use App\Entity\PersonRole; -use App\Repository\GroupRepository; +use App\Entity\GroupType; +use App\Repository\GroupTypeRepository; use App\Repository\InviteRepository; -use App\Repository\PersonRoleRepository; -use App\Service\Aggregator\WidgetAggregator; use App\Service\Http\GuzzleWrapper; class PbsAuthService @@ -20,20 +18,15 @@ class PbsAuthService */ private $guzzleWrapper; - /** - * @var PersonRoleRepository - */ - private $personRoleRepository; - /** * @var InviteRepository */ private $inviteRepository; /** - * @var GroupRepository + * @var GroupTypeRepository */ - private $groupRepository; + private $groupTypeRepository; /** * @var string @@ -68,8 +61,7 @@ class PbsAuthService /** * PbsAuthService constructor. * @param GuzzleWrapper $guzzleWrapper - * @param PersonRoleRepository $personRoleRepository - * @param GroupRepository $groupRepository + * @param GroupTypeRepository $groupTypeRepository * @param InviteRepository $inviteRepository * @param string $environment * @param string $pbsUrl @@ -80,8 +72,7 @@ class PbsAuthService */ public function __construct( GuzzleWrapper $guzzleWrapper, - PersonRoleRepository $personRoleRepository, - GroupRepository $groupRepository, + GroupTypeRepository $groupTypeRepository, InviteRepository $inviteRepository, string $environment, string $pbsUrl, @@ -91,8 +82,7 @@ public function __construct( string $specialAccessEmails ) { $this->guzzleWrapper = $guzzleWrapper; - $this->personRoleRepository = $personRoleRepository; - $this->groupRepository = $groupRepository; + $this->groupTypeRepository = $groupTypeRepository; $this->inviteRepository = $inviteRepository; $this->environment = $environment; $this->pbsUrl = $pbsUrl; @@ -112,30 +102,11 @@ public function getUser(string $code, string $locale, ?string $action = null): P { $token = $this->getTokenUsingCode($code, $action); $user = $this->getUserWithToken($token); - switch ($this->environment) { - case 'dev': - $this->processRolesForDev($user); - $pbsUser = PbsUserMapper::createFromArray($user); - $this->processGroupsForDev($pbsUser, $locale); - break; - case 'stage': - $this->processRolesForStage($user); - $pbsUser = PbsUserMapper::createFromArray($user); - $this->processGroupsForStage($pbsUser, $locale); - break; - default: - $this->processRoles($user); - $pbsUser = PbsUserMapper::createFromArray($user); - $this->processGroups($pbsUser, $locale); - } - $allGroups = $pbsUser->getGroups(); - usort($allGroups, function (GroupDTO $a, GroupDTO $b) { - return strcmp($a->getName(), $b->getName()); - }); - $pbsUser->setGroups($allGroups); + $user['syncable_groups'] = $this->getSyncableGroups($user, $locale); + $user['readable_groups'] = $this->getReadableGroups($user, $locale); - return $pbsUser; + return PbsUserMapper::createFromArray($user); } /** @@ -172,117 +143,44 @@ private function getUserWithToken(string $token): array } /** + * The user can sync any group of the correct type, in which they have a role + * with enough permissions. + * * @param array $user - */ - private function processRoles(array &$user) - { - $groupIds = array_unique(array_map(function ($role) { - return $role['group_id']; - }, $user['roles'])); - $user['roles'] = []; - - foreach ($groupIds as $groupId) { - $personRoles = $this->personRoleRepository->findRolesForPersonInGroup($groupId, $user['id']); - - if (!$personRoles) { - continue; - } - - /** @var PersonRole $personRole */ - foreach ($personRoles as $personRole) { - $user['roles'][] = [ - 'group_id' => $personRole->getGroup()->getId(), - 'group_name' => $personRole->getGroup()->getName(), - 'role_type' => $personRole->getRole()->getRoleType() - ]; - } - } - } - - /** - * This will add all groups to the user object where the user has a main-group leader role. - * Additionally, we get all groups to where the user was invited to and them to the user object as well. - * @param PbsUserDTO $pbsUser * @param string $locale + * @return GroupDTO[] */ - private function processGroups(PbsUserDTO $pbsUser, string $locale) + private function getSyncableGroups(array $user, string $locale) { - $groups = $this->groupRepository->findParentGroupsForPerson($pbsUser->getId()); - - $invites = $this->inviteRepository->findAllValidByEmail($pbsUser->getEmail()); - if ($invites) { - foreach ($invites as $invite) { - $groups[] = $invite->getGroup(); - } - } - - foreach ($groups as $group) { - $pbsUser->addGroup(GroupMapper::createFromEntity($group, $locale)); - } - } - - /** - * @param array $user - */ - private function processRolesForStage(array &$user) - { - if (!in_array($user['email'], $this->specialAccessEmails)) { - $this->processRoles($user); - return; - } - $this->processRolesForDev($user); - } - - /** - * @param PbsUserDTO $pbsUser - * @param string $locale - */ - private function processGroupsForStage(PbsUserDTO $pbsUser, string $locale) - { - if (!in_array($pbsUser->getEmail(), $this->specialAccessEmails)) { - $this->processGroups($pbsUser, $locale); - return; - } - $this->processGroupsForDev($pbsUser, $locale); + // Only keep roles that are allowed to sync a group + $groups = array_filter($user['roles'], function ($role) { + return $role['group_type'] === 'Group::Abteilung' && in_array($role['role_type'], [ + 'Group::Abteilung::Abteilungsleitung', + 'Group::Abteilung::AbteilungsleitungStv', + ]); + }); + // Map Midata response to DTOs + $groups = array_map(function ($group) use ($locale) { + /** @var GroupType $groupType */ + $groupType = $this->groupTypeRepository->findOneBy(['groupType' => $group['group_type']]); + return GroupMapper::createFromMidataOauthProfile($group, $groupType, $locale); + }, $groups); + + return array_values($groups); } /** - * This will assign a main-group leader role to every existing main group to the user. - * We do this so we can select any main-group in the front-end. + * All syncable groups are readable, as well as any groups for which the user has an invitation. * @param array $user - */ - private function processRolesForDev(array &$user) - { - $groups = $this->groupRepository->findAllParentGroups(); - $user['roles'] = []; - foreach ($groups as $group) { - $user['roles'][] = [ - 'group_id' => $group->getId(), - 'group_name' => $group->getName(), - 'role_type' => WidgetAggregator::$mainGroupRoleTypes[0] - ]; - } - } - - /** - * In dev we add all main-groups to the user object so that we can select all of them in the front-end. - * This behaviour is only for dev, since we are working with test data from the MiData INT environment. - * @param PbsUserDTO $pbsUser * @param string $locale + * @return GroupDTO[] */ - private function processGroupsForDev(PbsUserDTO $pbsUser, string $locale) + private function getReadableGroups(array $user, string $locale) { - $groups = $this->groupRepository->findAllParentGroups(); - - $invites = $this->inviteRepository->findAllValidByEmail($pbsUser->getEmail()); - if ($invites) { - foreach ($invites as $invite) { - $groups[] = $invite->getGroup(); - } - } + $invites = array_map(function ($invite) use ($locale) { + return GroupMapper::createFromEntity($invite->getGroup(), $locale); + }, $this->inviteRepository->findAllValidByEmail($user['email'])); - foreach ($groups as $group) { - $pbsUser->addGroup(GroupMapper::createFromEntity($group, $locale)); - } + return $user['syncable_groups'] + array_values($invites); } } From b844acfbdd4a4ef4306f0129a08a567fc541ab25 Mon Sep 17 00:00:00 2001 From: Carlo Beltrame Date: Sat, 25 Sep 2021 18:31:56 +0200 Subject: [PATCH 05/12] No need to fill in unused fields --- src/DTO/Mapper/GroupMapper.php | 3 --- src/DTO/Model/GroupDTO.php | 6 +++--- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/src/DTO/Mapper/GroupMapper.php b/src/DTO/Mapper/GroupMapper.php index 3b494af..5610bdb 100644 --- a/src/DTO/Mapper/GroupMapper.php +++ b/src/DTO/Mapper/GroupMapper.php @@ -27,9 +27,6 @@ public static function createFromMidataOauthProfile(array $group, GroupType $gro $groupDTO = new GroupDTO(); $groupDTO->setId($group['group_id']); $groupDTO->setName($group['group_name']); - $groupDTO->setCantonName('' /* TODO really needed? */ ); - $groupDTO->setCreatedAt('' /* TODO really needed? */); - $groupDTO->setDeletedAt('' /* TODO really needed? */); $groupDTO->setGroupType(GroupTypeMapper::createGroupTypeFromEntity($groupType, $locale)); return $groupDTO; } diff --git a/src/DTO/Model/GroupDTO.php b/src/DTO/Model/GroupDTO.php index eeaee8d..df1554d 100644 --- a/src/DTO/Model/GroupDTO.php +++ b/src/DTO/Model/GroupDTO.php @@ -7,13 +7,13 @@ class GroupDTO /** @var int|null */ private $id; /** @var string|null */ - private $cantonName; + private $cantonName = null; /** @var string|null */ private $name; /** @var string|null */ - private $createdAt; + private $createdAt = null; /** @var string|null */ - private $deletedAt; + private $deletedAt = null; /** @var GroupTypeDTO|null */ private $groupType; From 44afc1d4305a477c9bb142ee9af578eced71bb30 Mon Sep 17 00:00:00 2001 From: Carlo Beltrame Date: Sat, 25 Sep 2021 23:41:14 +0200 Subject: [PATCH 06/12] Get syncing of people, groups and roles to work --- src/Controller/Api/SyncController.php | 17 +++++++---- src/Entity/Group.php | 3 +- src/Entity/Person.php | 4 +-- src/Migrations/Version20210925185300.php | 29 +++++++++++++++++++ src/Service/PbsApi/Fetcher/GroupFetcher.php | 8 ++--- src/Service/PbsApi/Fetcher/PeopleFetcher.php | 6 ++-- .../PbsApi/Fetcher/PersonRoleMapper.php | 11 ++----- src/Service/SyncService.php | 9 +++--- 8 files changed, 60 insertions(+), 27 deletions(-) create mode 100644 src/Migrations/Version20210925185300.php diff --git a/src/Controller/Api/SyncController.php b/src/Controller/Api/SyncController.php index 03a7fd7..6eb8e9f 100644 --- a/src/Controller/Api/SyncController.php +++ b/src/Controller/Api/SyncController.php @@ -7,6 +7,7 @@ use App\Entity\Invite; use App\Exception\ApiException; use App\Service\InviteService; +use App\Service\PbsAuthService; use App\Service\SyncService; use Sensio\Bundle\FrameworkExtraBundle\Configuration\IsGranted; use Sensio\Bundle\FrameworkExtraBundle\Configuration\ParamConverter; @@ -31,29 +32,34 @@ class SyncController extends AbstractController */ private $translator; + /** + * @var PbsAuthService + */ + private $pbsAuthService; + /** * InviteController constructor. * @param SyncService $syncService * @param TranslatorInterface $translator */ - public function __construct(SyncService $syncService, TranslatorInterface $translator) + public function __construct(SyncService $syncService, TranslatorInterface $translator, PbsAuthService $pbsAuthService) { $this->syncService = $syncService; $this->translator = $translator; + $this->pbsAuthService = $pbsAuthService; } /** * @param Request $request - * @param Group $group + * @param int $groupId * @param SerializerInterface $serializer * @param ValidatorInterface $validator * @return JsonResponse * @ParamConverter(name="group", options={"mapping":{"groupId":"id"}}) - * @IsGranted("create", subject="group") */ public function startSync( Request $request, - Group $group, + int $groupId, SerializerInterface $serializer, ValidatorInterface $validator ) { @@ -64,7 +70,8 @@ public function startSync( if ($code === null) { throw new BadRequestHttpException('Authorization code is missing'); } - $this->syncService->startSync($group, $request->toArray()['code'] ?? ''); + $accessToken = $this->pbsAuthService->getTokenUsingCode($code, 'sync'); + $this->syncService->startSync($groupId, $accessToken); return $this->json($message, JsonResponse::HTTP_CREATED); } } diff --git a/src/Entity/Group.php b/src/Entity/Group.php index 7bb7dd4..1434d41 100644 --- a/src/Entity/Group.php +++ b/src/Entity/Group.php @@ -25,7 +25,7 @@ class Group private $id; /** - * @ORM\OneToMany(targetEntity="Group", mappedBy="parentGroup") + * @ORM\OneToMany(targetEntity="Group", mappedBy="parentGroup", cascade={"persist"}) */ private $children; @@ -79,6 +79,7 @@ class Group public function __construct() { + $this->children = new ArrayCollection(); $this->events = new ArrayCollection(); } diff --git a/src/Entity/Person.php b/src/Entity/Person.php index 9226d38..5e3035c 100644 --- a/src/Entity/Person.php +++ b/src/Entity/Person.php @@ -32,7 +32,7 @@ class Person private $id; /** - * @ORM\Column(type="string", length=255) + * @ORM\Column(type="string", length=255, nullable=true) */ private $nickname; @@ -88,7 +88,7 @@ class Person private $group; /** - * @ORM\OneToMany(targetEntity="PersonRole", mappedBy="person") + * @ORM\OneToMany(targetEntity="PersonRole", mappedBy="person", cascade={"persist"}) */ private $personRoles; diff --git a/src/Migrations/Version20210925185300.php b/src/Migrations/Version20210925185300.php new file mode 100644 index 0000000..f627d50 --- /dev/null +++ b/src/Migrations/Version20210925185300.php @@ -0,0 +1,29 @@ +addSql('ALTER TABLE midata_person ALTER nickname DROP NOT NULL'); + } + + public function down(Schema $schema) : void + { + $this->addSql('ALTER TABLE midata_person ALTER nickname SET NOT NULL'); + } +} diff --git a/src/Service/PbsApi/Fetcher/GroupFetcher.php b/src/Service/PbsApi/Fetcher/GroupFetcher.php index 5f5763c..402555f 100644 --- a/src/Service/PbsApi/Fetcher/GroupFetcher.php +++ b/src/Service/PbsApi/Fetcher/GroupFetcher.php @@ -61,11 +61,11 @@ private function mapJsonToGroup(array $json, string $accessToken): Group $metadata = $this->em->getClassMetaData(Group::class); $metadata->setIdGenerator(new AssignedGenerator()); } - $group->setName($this->gr['name'] ?? null); + $group->setName($groupJson['name'] ?? null); $cantonId = $groupJson['links']['hierarchies'][1] ?? null; $group->setCantonId($cantonId); - $group->setCantonName($this->getLinked($linked, 'groups', $cantonId)); + $group->setCantonName($this->getLinked($linked, 'groups', $cantonId)['name'] ?? null); $group->setCreatedAt(new DateTimeImmutable($groupJson['created_at'])); @@ -94,8 +94,8 @@ private function mapJsonToGroup(array $json, string $accessToken): Group } private function getLinked(array $linked, string $rel, string $id) { - return array_filter($linked[$rel] ?? [], function($linkedEntity) use ($id) { + return array_values(array_filter($linked[$rel] ?? [], function($linkedEntity) use ($id) { return $linkedEntity['id'] === $id; - })[0] ?? null; + }))[0] ?? null; } } diff --git a/src/Service/PbsApi/Fetcher/PeopleFetcher.php b/src/Service/PbsApi/Fetcher/PeopleFetcher.php index 221a089..7a85358 100644 --- a/src/Service/PbsApi/Fetcher/PeopleFetcher.php +++ b/src/Service/PbsApi/Fetcher/PeopleFetcher.php @@ -110,7 +110,7 @@ private function mapJsonToPeople(array $json): array foreach ($personJson['links']['roles'] ?? [] as $roleId) { $person->clearPersonRoles(); - $person->addPersonRole($this->roleMapper->mapFromJson($this->getLinked($linked, 'roles', $roleId))); + $person->addPersonRole($this->roleMapper->mapFromJson($this->getLinked($linked, 'roles', $roleId), $person)); } $people[] = $person; @@ -120,8 +120,8 @@ private function mapJsonToPeople(array $json): array } private function getLinked(array $linked, string $rel, string $id) { - return array_filter($linked[$rel] ?? [], function($linkedEntity) use ($id) { + return array_values(array_filter($linked[$rel] ?? [], function($linkedEntity) use ($id) { return $linkedEntity['id'] === $id; - })[0] ?? null; + }))[0] ?? null; } } diff --git a/src/Service/PbsApi/Fetcher/PersonRoleMapper.php b/src/Service/PbsApi/Fetcher/PersonRoleMapper.php index cc7c41a..eabda32 100644 --- a/src/Service/PbsApi/Fetcher/PersonRoleMapper.php +++ b/src/Service/PbsApi/Fetcher/PersonRoleMapper.php @@ -56,7 +56,7 @@ public function __construct(EntityManagerInterface $em, PbsApiService $pbsApiSer * @return PersonRole|null * @throws \Exception */ - public function mapFromJson(array $roleJson): ?PersonRole + public function mapFromJson(array $roleJson, Person $person): ?PersonRole { $personRole = $this->personRoleRepository->findOneBy(['id' => $roleJson['id']]); if (!$personRole) { @@ -65,20 +65,15 @@ public function mapFromJson(array $roleJson): ?PersonRole $metadata = $this->em->getClassMetaData(get_class($personRole)); $metadata->setIdGenerator(new AssignedGenerator()); } - /** @var Person $person */ - $person = $this->personRepository->find($roleJson['person_id']); - if (!$person) { - return null; - } $personRole->setPerson($person); - $role = $this->roleRepository->getOneByRoleType($roleJson['type']); + $role = $this->roleRepository->getOneByRoleType($roleJson['role_class'] ?? ''); if ($role) { $personRole->setRole($role); } /** @var Group $group */ - $group = $this->groupRepository->find($roleJson['group_id']); + $group = $this->groupRepository->find($roleJson['links']['group'] ?? ''); if ($group) { $personRole->setGroup($group); } diff --git a/src/Service/SyncService.php b/src/Service/SyncService.php index 78b697a..4cdfc09 100644 --- a/src/Service/SyncService.php +++ b/src/Service/SyncService.php @@ -38,13 +38,14 @@ public function __construct(EntityManagerInterface $em, PbsApiService $pbsApiSer } /** - * @param Group $group + * @param int $groupId + * @param $accessToken * @return void */ - public function startSync(Group $group, $accessToken) + public function startSync(int $groupId, $accessToken) { - $this->groupFetcher->fetchAndPersistGroup($group->getId(), $accessToken); - $this->peopleFetcher->fetchAndPersistPeople($group->getId(), $accessToken); + $this->groupFetcher->fetchAndPersistGroup($groupId, $accessToken); + $this->peopleFetcher->fetchAndPersistPeople($groupId, $accessToken); // TODO run aggregations here, but only for the fetched group } From 48c38ebd87adad0001fe0ccb59595659addb65c6 Mon Sep 17 00:00:00 2001 From: Carlo Beltrame Date: Sun, 26 Sep 2021 02:16:36 +0200 Subject: [PATCH 07/12] Fetch camps from the normal hitobito API --- src/Entity/Event.php | 17 ++++- src/Entity/Person.php | 2 +- .../PbsApi/Fetcher/AbstractFetcher.php | 56 ++++++++++++++ src/Service/PbsApi/Fetcher/CampsFetcher.php | 75 +++++++++++++++++++ .../PbsApi/Fetcher/EventDateMapper.php | 52 +++++++++++++ src/Service/PbsApi/Fetcher/GroupFetcher.php | 22 ++---- src/Service/PbsApi/Fetcher/PeopleFetcher.php | 44 ++--------- .../PbsApi/Fetcher/PersonRoleMapper.php | 14 +--- src/Service/SyncService.php | 11 ++- 9 files changed, 222 insertions(+), 71 deletions(-) create mode 100644 src/Service/PbsApi/Fetcher/AbstractFetcher.php create mode 100644 src/Service/PbsApi/Fetcher/CampsFetcher.php create mode 100644 src/Service/PbsApi/Fetcher/EventDateMapper.php diff --git a/src/Entity/Event.php b/src/Entity/Event.php index 6d86917..356947f 100644 --- a/src/Entity/Event.php +++ b/src/Entity/Event.php @@ -41,7 +41,7 @@ abstract class Event private $persons; /** - * @ORM\OneToMany(targetEntity="App\Entity\EventDate", mappedBy="event") + * @ORM\OneToMany(targetEntity="App\Entity\EventDate", mappedBy="event", cascade={"persist", "remove"}, orphanRemoval=true) */ private $eventDates; @@ -129,4 +129,19 @@ public function setEventDates(Collection $eventDates): void { $this->eventDates = $eventDates; } + + public function addEventDate(EventDate $eventDate): self { + if (!$this->eventDates->contains($eventDate)) { + $this->eventDates[] = $eventDate; + $eventDate->setEvent($this); + } + + return $this; + } + + public function clearEventDates(): self { + $this->eventDates = new ArrayCollection(); + + return $this; + } } diff --git a/src/Entity/Person.php b/src/Entity/Person.php index 5e3035c..5c077ae 100644 --- a/src/Entity/Person.php +++ b/src/Entity/Person.php @@ -88,7 +88,7 @@ class Person private $group; /** - * @ORM\OneToMany(targetEntity="PersonRole", mappedBy="person", cascade={"persist"}) + * @ORM\OneToMany(targetEntity="PersonRole", mappedBy="person", cascade={"persist", "remove"}, orphanRemoval=true) */ private $personRoles; diff --git a/src/Service/PbsApi/Fetcher/AbstractFetcher.php b/src/Service/PbsApi/Fetcher/AbstractFetcher.php new file mode 100644 index 0000000..a9cf8e8 --- /dev/null +++ b/src/Service/PbsApi/Fetcher/AbstractFetcher.php @@ -0,0 +1,56 @@ +em = $em; + $this->pbsApiService = $pbsApiService; + } + + public function fetchAndPersist(string $groupId, string $accessToken) + { + $i = 0; + foreach ($this->fetch($groupId, $accessToken) as $entity) { + $this->em->persist($entity); + $i++; + + if (($i % $this->batchSize) === 0) { + $this->em->flush(); + $this->em->clear(); + } + } + $this->em->flush(); + } + + protected abstract function fetch(string $groupId, string $accessToken): array; + + protected function getLinked(array $linked, string $rel, string $id) { + return array_values(array_filter($linked[$rel] ?? [], function($linkedEntity) use ($id) { + return $linkedEntity['id'] === $id; + }))[0] ?? null; + } +} diff --git a/src/Service/PbsApi/Fetcher/CampsFetcher.php b/src/Service/PbsApi/Fetcher/CampsFetcher.php new file mode 100644 index 0000000..03b0499 --- /dev/null +++ b/src/Service/PbsApi/Fetcher/CampsFetcher.php @@ -0,0 +1,75 @@ +campRepository = $this->em->getRepository(Camp::class); + $this->youthSportTypeRepository = $this->em->getRepository(YouthSportType::class); + $this->eventDateMapper = $eventDateMapper; + } + + protected function fetch(string $groupId, string $accessToken): array + { + $startDate = date('d-m-Y', strtotime('-10 years')); + $endDate = date('d-m-Y', strtotime('+5 years')); + $campData = $this->pbsApiService->getApiData('/groups/'.$groupId.'/events?type=Event::Camp&start_date='.$startDate.'&end_date='.$endDate, $accessToken); + return $this->mapJsonToCamps($campData); + } + + private function mapJsonToCamps(array $json): array + { + $campsJson = $json['events'] ?? []; + $linked = $json['linked'] ?? []; + + $camps = []; + foreach ($campsJson as $campJson) { + $camp = $this->campRepository->findOneBy(['id' => $campJson['id']]); + if (!$camp) { + $camp = new Camp(); + $camp->setId($campJson['id']); + $metadata = $this->em->getClassMetaData(get_class($camp)); + $metadata->setIdGenerator(new AssignedGenerator()); + } + $camp->setName($campJson['name']); + $camp->setLocation($campJson['location']); + $camp->setState($campJson['state']); + + /** @var YouthSportType $youthYouthType */ + $youthYouthType = $this->youthSportTypeRepository->findOneBy(['type' => $campJson['j_s_kind'] ?? 'j_s_kind_none']); + $camp->setYouthSportType($youthYouthType); + + $camp->clearEventDates(); + foreach ($campJson['links']['dates'] ?? [] as $dateId) { + $camp->addEventDate($this->eventDateMapper->mapFromJson($this->getLinked($linked, 'event_dates', $dateId), $camp)); + } + + $camps[] = $camp; + } + + return $camps; + } +} diff --git a/src/Service/PbsApi/Fetcher/EventDateMapper.php b/src/Service/PbsApi/Fetcher/EventDateMapper.php new file mode 100644 index 0000000..a645741 --- /dev/null +++ b/src/Service/PbsApi/Fetcher/EventDateMapper.php @@ -0,0 +1,52 @@ +em = $em; + $this->eventDateRepository = $this->em->getRepository(PersonRole::class); + } + + /** + * @param array $dateJson + * @return PersonRole|null + * @throws \Exception + */ + public function mapFromJson(array $dateJson, Event $event): ?EventDate + { + $eventDate = $this->eventDateRepository->findOneBy(['id' => $dateJson['id']]); + if (!$eventDate) { + $eventDate = new EventDate(); + $eventDate->setId($dateJson['id']); + $metadata = $this->em->getClassMetaData(get_class($eventDate)); + $metadata->setIdGenerator(new AssignedGenerator()); + } + $eventDate->setEvent($event); + + $eventDate->setStartAt(new DateTimeImmutable($dateJson['start_at'])); + if ($dateJson['finish_at']) { + $eventDate->setEndAt(new DateTimeImmutable($dateJson['finish_at'])); + } + + return $eventDate; + } +} diff --git a/src/Service/PbsApi/Fetcher/GroupFetcher.php b/src/Service/PbsApi/Fetcher/GroupFetcher.php index 402555f..8518a61 100644 --- a/src/Service/PbsApi/Fetcher/GroupFetcher.php +++ b/src/Service/PbsApi/Fetcher/GroupFetcher.php @@ -11,12 +11,8 @@ use Doctrine\ORM\EntityManagerInterface; use Doctrine\ORM\Id\AssignedGenerator; -class GroupFetcher +class GroupFetcher extends AbstractFetcher { - /** - * @var EntityManagerInterface - */ - private $em; /** * @var GroupTypeRepository */ @@ -25,16 +21,11 @@ class GroupFetcher * @var GroupRepository */ private $groupRepository; - /** - * @var PbsApiService - */ - private $pbsApiService; public function __construct(EntityManagerInterface $em, PbsApiService $pbsApiService) { - $this->em = $em; + parent::__construct($em, $pbsApiService); $this->groupTypeRepository = $this->em->getRepository(GroupType::class); $this->groupRepository = $this->em->getRepository(Group::class); - $this->pbsApiService = $pbsApiService; } public function fetchAndPersistGroup(string $id, string $accessToken) @@ -43,7 +34,7 @@ public function fetchAndPersistGroup(string $id, string $accessToken) $this->em->flush(); } - private function fetchGroup(string $id, string $accessToken): Group + protected function fetchGroup(string $id, string $accessToken): Group { $groupData = $this->pbsApiService->getApiData('/groups/'.$id, $accessToken); return $this->mapJsonToGroup($groupData, $accessToken); @@ -93,9 +84,8 @@ private function mapJsonToGroup(array $json, string $accessToken): Group return $group; } - private function getLinked(array $linked, string $rel, string $id) { - return array_values(array_filter($linked[$rel] ?? [], function($linkedEntity) use ($id) { - return $linkedEntity['id'] === $id; - }))[0] ?? null; + protected function fetch(string $groupId, string $accessToken): array { + // Not implemented because not needed + return []; } } diff --git a/src/Service/PbsApi/Fetcher/PeopleFetcher.php b/src/Service/PbsApi/Fetcher/PeopleFetcher.php index 7a85358..bc85c0b 100644 --- a/src/Service/PbsApi/Fetcher/PeopleFetcher.php +++ b/src/Service/PbsApi/Fetcher/PeopleFetcher.php @@ -11,16 +11,8 @@ use Doctrine\ORM\EntityManagerInterface; use Doctrine\ORM\Id\AssignedGenerator; -class PeopleFetcher +class PeopleFetcher extends AbstractFetcher { - /** - * @var int - */ - private $batchSize = 100; - /** - * @var EntityManagerInterface - */ - private $em; /** * @var PersonRepository */ @@ -29,39 +21,19 @@ class PeopleFetcher * @var GroupRepository */ private $groupRepository; - /** - * @var PbsApiService - */ - private $pbsApiService; /** * @var PersonRoleMapper */ private $roleMapper; public function __construct(EntityManagerInterface $em, PbsApiService $pbsApiService, PersonRoleMapper $roleMapper) { - $this->em = $em; + parent::__construct($em, $pbsApiService); $this->personRepository = $this->em->getRepository(Person::class); $this->groupRepository = $this->em->getRepository(Group::class); - $this->pbsApiService = $pbsApiService; $this->roleMapper = $roleMapper; } - public function fetchAndPersistPeople(string $groupId, string $accessToken) - { - $i = 0; - foreach ($this->fetchPeople($groupId, $accessToken) as $person) { - $this->em->persist($person); - $i++; - - if (($i % $this->batchSize) === 0) { - $this->em->flush(); - $this->em->clear(); - } - } - $this->em->flush(); - } - - private function fetchPeople(string $groupId, string $accessToken): array + protected function fetch(string $groupId, string $accessToken): array { $peopleData = $this->pbsApiService->getApiData('/groups/'.$groupId.'/people?filters[role][kind]=with_deleted&range=layer', $accessToken); return $this->mapJsonToPeople($peopleData); @@ -108,8 +80,10 @@ private function mapJsonToPeople(array $json): array } } + // TODO this may also clear any roles of the same person in another Abteilung. + // We need to import and store the same person separately for each Abteilung they're in. + $person->clearPersonRoles(); foreach ($personJson['links']['roles'] ?? [] as $roleId) { - $person->clearPersonRoles(); $person->addPersonRole($this->roleMapper->mapFromJson($this->getLinked($linked, 'roles', $roleId), $person)); } @@ -118,10 +92,4 @@ private function mapJsonToPeople(array $json): array return $people; } - - private function getLinked(array $linked, string $rel, string $id) { - return array_values(array_filter($linked[$rel] ?? [], function($linkedEntity) use ($id) { - return $linkedEntity['id'] === $id; - }))[0] ?? null; - } } diff --git a/src/Service/PbsApi/Fetcher/PersonRoleMapper.php b/src/Service/PbsApi/Fetcher/PersonRoleMapper.php index eabda32..7d1f381 100644 --- a/src/Service/PbsApi/Fetcher/PersonRoleMapper.php +++ b/src/Service/PbsApi/Fetcher/PersonRoleMapper.php @@ -7,10 +7,8 @@ use App\Entity\PersonRole; use App\Entity\Role; use App\Repository\GroupRepository; -use App\Repository\PersonRepository; use App\Repository\PersonRoleRepository; use App\Repository\RoleRepository; -use App\Service\PbsApiService; use DateTimeImmutable; use Doctrine\ORM\EntityManagerInterface; use Doctrine\ORM\Id\AssignedGenerator; @@ -25,10 +23,6 @@ class PersonRoleMapper * @var PersonRoleRepository */ private $personRoleRepository; - /** - * @var PersonRepository - */ - private $personRepository; /** * @var GroupRepository */ @@ -37,18 +31,12 @@ class PersonRoleMapper * @var RoleRepository */ private $roleRepository; - /** - * @var PbsApiService - */ - private $pbsApiService; - public function __construct(EntityManagerInterface $em, PbsApiService $pbsApiService) { + public function __construct(EntityManagerInterface $em) { $this->em = $em; $this->personRoleRepository = $this->em->getRepository(PersonRole::class); - $this->personRepository = $this->em->getRepository(Person::class); $this->groupRepository = $this->em->getRepository(Group::class); $this->roleRepository = $this->em->getRepository(Role::class); - $this->pbsApiService = $pbsApiService; } /** diff --git a/src/Service/SyncService.php b/src/Service/SyncService.php index 4cdfc09..2daa96f 100644 --- a/src/Service/SyncService.php +++ b/src/Service/SyncService.php @@ -3,6 +3,7 @@ namespace App\Service; use App\Entity\Group; +use App\Service\PbsApi\Fetcher\CampsFetcher; use App\Service\PbsApi\Fetcher\GroupFetcher; use App\Service\PbsApi\Fetcher\PeopleFetcher; use Doctrine\ORM\EntityManagerInterface; @@ -25,16 +26,21 @@ class SyncService * @var PeopleFetcher */ private $peopleFetcher; + /** + * @var CampsFetcher + */ + private $campsFetcher; /** * @param PbsApiService $pbsApiService */ - public function __construct(EntityManagerInterface $em, PbsApiService $pbsApiService, GroupFetcher $groupMapper, PeopleFetcher $peopleFetcher) + public function __construct(EntityManagerInterface $em, PbsApiService $pbsApiService, GroupFetcher $groupMapper, PeopleFetcher $peopleFetcher, CampsFetcher $campsFetcher) { $this->em = $em; $this->pbsApiService = $pbsApiService; $this->groupFetcher = $groupMapper; $this->peopleFetcher = $peopleFetcher; + $this->campsFetcher = $campsFetcher; } /** @@ -45,7 +51,8 @@ public function __construct(EntityManagerInterface $em, PbsApiService $pbsApiSer public function startSync(int $groupId, $accessToken) { $this->groupFetcher->fetchAndPersistGroup($groupId, $accessToken); - $this->peopleFetcher->fetchAndPersistPeople($groupId, $accessToken); + $this->peopleFetcher->fetchAndPersist($groupId, $accessToken); + $this->campsFetcher->fetchAndPersist($groupId, $accessToken); // TODO run aggregations here, but only for the fetched group } From badc69170c80d11aeb347cad4c7cd4e4f73917ed Mon Sep 17 00:00:00 2001 From: Carlo Beltrame Date: Sun, 26 Sep 2021 03:02:45 +0200 Subject: [PATCH 08/12] Fetch courses from the normal hitobito API --- src/Service/PbsApi/Fetcher/CoursesFetcher.php | 75 +++++++++++++++++++ src/Service/SyncService.php | 10 ++- 2 files changed, 83 insertions(+), 2 deletions(-) create mode 100644 src/Service/PbsApi/Fetcher/CoursesFetcher.php diff --git a/src/Service/PbsApi/Fetcher/CoursesFetcher.php b/src/Service/PbsApi/Fetcher/CoursesFetcher.php new file mode 100644 index 0000000..3232ad7 --- /dev/null +++ b/src/Service/PbsApi/Fetcher/CoursesFetcher.php @@ -0,0 +1,75 @@ +courseRepository = $this->em->getRepository(Course::class); + $this->eventTypeRepository = $this->em->getRepository(EventType::class); + $this->eventDateMapper = $eventDateMapper; + } + + protected function fetch(string $groupId, string $accessToken): array + { + $startDate = date('d-m-Y', strtotime('-10 years')); + $endDate = date('d-m-Y', strtotime('+5 years')); + $courseData = $this->pbsApiService->getApiData('/groups/'.$groupId.'/events?type=Event::Course&start_date='.$startDate.'&end_date='.$endDate, $accessToken); + return $this->mapJsonToCourses($courseData); + } + + private function mapJsonToCourses(array $json): array + { + $coursesJson = $json['events'] ?? []; + $linked = $json['linked'] ?? []; + + $courses = []; + foreach ($coursesJson as $courseJson) { + /** @var Course $course */ + $course = $this->courseRepository->findOneBy(['id' => $courseJson['id']]); + if (!$course) { + $course = new Course(); + $course->setId($courseJson['id']); + $metadata = $this->em->getClassMetaData(get_class($course)); + $metadata->setIdGenerator(new AssignedGenerator()); + } + $course->setName($courseJson['name']); + + /** @var EventType $eventType */ + $eventType = $this->eventTypeRepository->findOneBy(['id' => $courseJson['links']['kind']]); + $course->setEventType($eventType); + + $course->clearEventDates(); + foreach ($courseJson['links']['dates'] ?? [] as $dateId) { + $course->addEventDate($this->eventDateMapper->mapFromJson($this->getLinked($linked, 'event_dates', $dateId), $course)); + } + + $courses[] = $course; + } + + return $courses; + } +} diff --git a/src/Service/SyncService.php b/src/Service/SyncService.php index 2daa96f..703dee7 100644 --- a/src/Service/SyncService.php +++ b/src/Service/SyncService.php @@ -2,8 +2,8 @@ namespace App\Service; -use App\Entity\Group; use App\Service\PbsApi\Fetcher\CampsFetcher; +use App\Service\PbsApi\Fetcher\CoursesFetcher; use App\Service\PbsApi\Fetcher\GroupFetcher; use App\Service\PbsApi\Fetcher\PeopleFetcher; use Doctrine\ORM\EntityManagerInterface; @@ -30,17 +30,22 @@ class SyncService * @var CampsFetcher */ private $campsFetcher; + /** + * @var CoursesFetcher + */ + private $coursesFetcher; /** * @param PbsApiService $pbsApiService */ - public function __construct(EntityManagerInterface $em, PbsApiService $pbsApiService, GroupFetcher $groupMapper, PeopleFetcher $peopleFetcher, CampsFetcher $campsFetcher) + public function __construct(EntityManagerInterface $em, PbsApiService $pbsApiService, GroupFetcher $groupMapper, PeopleFetcher $peopleFetcher, CampsFetcher $campsFetcher, CoursesFetcher $coursesFetcher) { $this->em = $em; $this->pbsApiService = $pbsApiService; $this->groupFetcher = $groupMapper; $this->peopleFetcher = $peopleFetcher; $this->campsFetcher = $campsFetcher; + $this->coursesFetcher = $coursesFetcher; } /** @@ -53,6 +58,7 @@ public function startSync(int $groupId, $accessToken) $this->groupFetcher->fetchAndPersistGroup($groupId, $accessToken); $this->peopleFetcher->fetchAndPersist($groupId, $accessToken); $this->campsFetcher->fetchAndPersist($groupId, $accessToken); + $this->coursesFetcher->fetchAndPersist($groupId, $accessToken); // TODO run aggregations here, but only for the fetched group } From fa1902494832f231d527ecded1b011b56137f1f5 Mon Sep 17 00:00:00 2001 From: Carlo Beltrame Date: Sun, 26 Sep 2021 17:12:05 +0200 Subject: [PATCH 09/12] Fetch participations in camps and courses from the normal hitobito API, and clear existing data before syncing --- src/Entity/Event.php | 18 ++++- src/Entity/EventDate.php | 2 +- src/Entity/EventTypeQualificationType.php | 2 +- src/Entity/Group.php | 2 +- src/Entity/PersonEvent.php | 4 +- src/Entity/PersonRole.php | 4 +- src/Migrations/Version20210925185300.php | 22 +++++ .../PbsApi/Fetcher/AbstractFetcher.php | 2 + src/Service/PbsApi/Fetcher/CampsFetcher.php | 48 ++++++++++- src/Service/PbsApi/Fetcher/CoursesFetcher.php | 49 +++++++++-- src/Service/PbsApi/Fetcher/GroupFetcher.php | 11 +++ src/Service/PbsApi/Fetcher/PeopleFetcher.php | 27 ++++--- .../PbsApi/Fetcher/PersonEventsFetcher.php | 81 +++++++++++++++++++ src/Service/SyncService.php | 5 ++ 14 files changed, 251 insertions(+), 26 deletions(-) create mode 100644 src/Service/PbsApi/Fetcher/PersonEventsFetcher.php diff --git a/src/Entity/Event.php b/src/Entity/Event.php index 356947f..bf71cc9 100644 --- a/src/Entity/Event.php +++ b/src/Entity/Event.php @@ -36,7 +36,7 @@ abstract class Event private $groups; /** - * @ORM\OneToMany(targetEntity="PersonEvent", mappedBy="event", cascade={"persist", "remove"}) + * @ORM\OneToMany(targetEntity="PersonEvent", mappedBy="event", cascade={"persist", "remove"}, orphanRemoval=true) */ private $persons; @@ -52,6 +52,7 @@ public function __construct() { $this->groups = new ArrayCollection(); $this->eventDates = new ArrayCollection(); + $this->persons = new ArrayCollection(); } /** @@ -144,4 +145,19 @@ public function clearEventDates(): self { return $this; } + + public function addPerson(PersonEvent $personEvent): self { + if (!$this->persons->contains($personEvent)) { + $this->persons[] = $personEvent; + $personEvent->setEvent($this); + } + + return $this; + } + + public function clearPersons(): self { + $this->persons = new ArrayCollection(); + + return $this; + } } diff --git a/src/Entity/EventDate.php b/src/Entity/EventDate.php index bedc239..31b72bd 100644 --- a/src/Entity/EventDate.php +++ b/src/Entity/EventDate.php @@ -24,7 +24,7 @@ class EventDate /** * @ORM\ManyToOne(targetEntity="Event") - * @ORM\JoinColumn(name="event_id", referencedColumnName="id") + * @ORM\JoinColumn(name="event_id", referencedColumnName="id", onDelete="CASCADE") */ private $event; diff --git a/src/Entity/EventTypeQualificationType.php b/src/Entity/EventTypeQualificationType.php index 0cbe9e3..b112207 100644 --- a/src/Entity/EventTypeQualificationType.php +++ b/src/Entity/EventTypeQualificationType.php @@ -25,7 +25,7 @@ class EventTypeQualificationType private $eventType; /** - * @ORM\ManyToOne(targetEntity="QualificationTYpe") + * @ORM\ManyToOne(targetEntity="QualificationType") * @ORM\JoinColumn(name="qualification_type_id", referencedColumnName="id") */ private $qualificationType; diff --git a/src/Entity/Group.php b/src/Entity/Group.php index 1434d41..a5f906a 100644 --- a/src/Entity/Group.php +++ b/src/Entity/Group.php @@ -31,7 +31,7 @@ class Group /** * @ORM\ManyToOne(targetEntity="Group", inversedBy="children") - * @ORM\JoinColumn(name="parent_group_id", referencedColumnName="id") + * @ORM\JoinColumn(name="parent_group_id", referencedColumnName="id", onDelete="CASCADE") */ private $parentGroup; diff --git a/src/Entity/PersonEvent.php b/src/Entity/PersonEvent.php index f251f7f..0da3379 100644 --- a/src/Entity/PersonEvent.php +++ b/src/Entity/PersonEvent.php @@ -22,13 +22,13 @@ class PersonEvent /** * @ORM\ManyToOne(targetEntity="Event", inversedBy="persons") - * @ORM\JoinColumn(nullable=false) + * @ORM\JoinColumn(nullable=false, onDelete="CASCADE") */ private $event; /** * @ORM\ManyToOne(targetEntity="Person", inversedBy="events") - * @ORM\JoinColumn(nullable=false) + * @ORM\JoinColumn(nullable=true) */ private $person; diff --git a/src/Entity/PersonRole.php b/src/Entity/PersonRole.php index 319a8e2..8354336 100644 --- a/src/Entity/PersonRole.php +++ b/src/Entity/PersonRole.php @@ -24,13 +24,13 @@ class PersonRole /** * @ORM\ManyToOne(targetEntity="Group", inversedBy="personRoles") - * @ORM\JoinColumn(name="group_id", referencedColumnName="id") + * @ORM\JoinColumn(name="group_id", referencedColumnName="id", onDelete="CASCADE") */ private $group; /** * @ORM\ManyToOne(targetEntity="Person") - * @ORM\JoinColumn(name="person_id", referencedColumnName="id") + * @ORM\JoinColumn(name="person_id", referencedColumnName="id", onDelete="CASCADE") */ private $person; diff --git a/src/Migrations/Version20210925185300.php b/src/Migrations/Version20210925185300.php index f627d50..4bb5cb3 100644 --- a/src/Migrations/Version20210925185300.php +++ b/src/Migrations/Version20210925185300.php @@ -20,10 +20,32 @@ public function getDescription() : string public function up(Schema $schema) : void { $this->addSql('ALTER TABLE midata_person ALTER nickname DROP NOT NULL'); + $this->addSql('ALTER TABLE midata_person_event ALTER person_id DROP NOT NULL'); + $this->addSql('ALTER TABLE midata_event_date DROP CONSTRAINT FK_E13DC41771F7E88B'); + $this->addSql('ALTER TABLE midata_event_date ADD CONSTRAINT FK_E13DC41771F7E88B FOREIGN KEY (event_id) REFERENCES midata_event (id) ON DELETE CASCADE NOT DEFERRABLE INITIALLY IMMEDIATE'); + $this->addSql('ALTER TABLE midata_person_event DROP CONSTRAINT FK_51E69FDE71F7E88B'); + $this->addSql('ALTER TABLE midata_person_event ADD CONSTRAINT FK_51E69FDE71F7E88B FOREIGN KEY (event_id) REFERENCES midata_event (id) ON DELETE CASCADE NOT DEFERRABLE INITIALLY IMMEDIATE'); + $this->addSql('ALTER TABLE midata_person_role DROP CONSTRAINT FK_AFDC017BFE54D947'); + $this->addSql('ALTER TABLE midata_person_role ADD CONSTRAINT FK_AFDC017BFE54D947 FOREIGN KEY (group_id) REFERENCES midata_group (id) ON DELETE CASCADE NOT DEFERRABLE INITIALLY IMMEDIATE'); + $this->addSql('ALTER TABLE midata_person_role DROP CONSTRAINT FK_AFDC017B217BBB47'); + $this->addSql('ALTER TABLE midata_person_role ADD CONSTRAINT FK_AFDC017B217BBB47 FOREIGN KEY (person_id) REFERENCES midata_person (id) ON DELETE CASCADE NOT DEFERRABLE INITIALLY IMMEDIATE'); + $this->addSql('ALTER TABLE midata_group DROP CONSTRAINT FK_2644E31861997596'); + $this->addSql('ALTER TABLE midata_group ADD CONSTRAINT FK_2644E31861997596 FOREIGN KEY (parent_group_id) REFERENCES midata_group (id) ON DELETE CASCADE NOT DEFERRABLE INITIALLY IMMEDIATE'); } public function down(Schema $schema) : void { + $this->addSql('ALTER TABLE midata_group DROP CONSTRAINT fk_2644e31861997596'); + $this->addSql('ALTER TABLE midata_group ADD CONSTRAINT fk_2644e31861997596 FOREIGN KEY (parent_group_id) REFERENCES midata_group (id) NOT DEFERRABLE INITIALLY IMMEDIATE'); + $this->addSql('ALTER TABLE midata_person_role DROP CONSTRAINT fk_afdc017bfe54d947'); + $this->addSql('ALTER TABLE midata_person_role ADD CONSTRAINT fk_afdc017bfe54d947 FOREIGN KEY (group_id) REFERENCES midata_group (id) NOT DEFERRABLE INITIALLY IMMEDIATE'); + $this->addSql('ALTER TABLE midata_person_role DROP CONSTRAINT fk_afdc017b217bbb47'); + $this->addSql('ALTER TABLE midata_person_role ADD CONSTRAINT fk_afdc017b217bbb47 FOREIGN KEY (person_id) REFERENCES midata_person (id) NOT DEFERRABLE INITIALLY IMMEDIATE'); + $this->addSql('ALTER TABLE midata_event_date DROP CONSTRAINT fk_e13dc41771f7e88b'); + $this->addSql('ALTER TABLE midata_event_date ADD CONSTRAINT fk_e13dc41771f7e88b FOREIGN KEY (event_id) REFERENCES midata_event (id) NOT DEFERRABLE INITIALLY IMMEDIATE'); + $this->addSql('ALTER TABLE midata_person_event DROP CONSTRAINT fk_51e69fde71f7e88b'); + $this->addSql('ALTER TABLE midata_person_event ADD CONSTRAINT fk_51e69fde71f7e88b FOREIGN KEY (event_id) REFERENCES midata_event (id) NOT DEFERRABLE INITIALLY IMMEDIATE'); + $this->addSql('ALTER TABLE midata_person_event ALTER person_id SET NOT NULL'); $this->addSql('ALTER TABLE midata_person ALTER nickname SET NOT NULL'); } } diff --git a/src/Service/PbsApi/Fetcher/AbstractFetcher.php b/src/Service/PbsApi/Fetcher/AbstractFetcher.php index a9cf8e8..36fa88e 100644 --- a/src/Service/PbsApi/Fetcher/AbstractFetcher.php +++ b/src/Service/PbsApi/Fetcher/AbstractFetcher.php @@ -48,6 +48,8 @@ public function fetchAndPersist(string $groupId, string $accessToken) protected abstract function fetch(string $groupId, string $accessToken): array; + public abstract function clean(string $groupId); + protected function getLinked(array $linked, string $rel, string $id) { return array_values(array_filter($linked[$rel] ?? [], function($linkedEntity) use ($id) { return $linkedEntity['id'] === $id; diff --git a/src/Service/PbsApi/Fetcher/CampsFetcher.php b/src/Service/PbsApi/Fetcher/CampsFetcher.php index 03b0499..8dee6f3 100644 --- a/src/Service/PbsApi/Fetcher/CampsFetcher.php +++ b/src/Service/PbsApi/Fetcher/CampsFetcher.php @@ -3,8 +3,11 @@ namespace App\Service\PbsApi\Fetcher; use App\Entity\Camp; +use App\Entity\EventDate; use App\Entity\YouthSportType; use App\Repository\CampRepository; +use App\Repository\EventDateRepository; +use App\Repository\PersonRepository; use App\Repository\YouthSportTypeRepository; use App\Service\PbsApiService; use Doctrine\ORM\EntityManagerInterface; @@ -24,12 +27,22 @@ class CampsFetcher extends AbstractFetcher * @var EventDateMapper */ private $eventDateMapper; + /** + * @var PersonRepository + */ + private $personRepository; + /** + * @var EventDateRepository + */ + private $eventDateRepository; - public function __construct(EntityManagerInterface $em, PbsApiService $pbsApiService, EventDateMapper $eventDateMapper) { + public function __construct(EntityManagerInterface $em, PbsApiService $pbsApiService, EventDateMapper $eventDateMapper, PersonRepository $personRepository, EventDateRepository $eventDateRepository) { parent::__construct($em, $pbsApiService); $this->campRepository = $this->em->getRepository(Camp::class); $this->youthSportTypeRepository = $this->em->getRepository(YouthSportType::class); $this->eventDateMapper = $eventDateMapper; + $this->personRepository = $personRepository; + $this->eventDateRepository = $eventDateRepository; } protected function fetch(string $groupId, string $accessToken): array @@ -37,10 +50,10 @@ protected function fetch(string $groupId, string $accessToken): array $startDate = date('d-m-Y', strtotime('-10 years')); $endDate = date('d-m-Y', strtotime('+5 years')); $campData = $this->pbsApiService->getApiData('/groups/'.$groupId.'/events?type=Event::Camp&start_date='.$startDate.'&end_date='.$endDate, $accessToken); - return $this->mapJsonToCamps($campData); + return $this->mapJsonToCamps($campData, $groupId, $accessToken); } - private function mapJsonToCamps(array $json): array + private function mapJsonToCamps(array $json, string $groupId, string $accessToken): array { $campsJson = $json['events'] ?? []; $linked = $json['linked'] ?? []; @@ -62,14 +75,41 @@ private function mapJsonToCamps(array $json): array $youthYouthType = $this->youthSportTypeRepository->findOneBy(['type' => $campJson['j_s_kind'] ?? 'j_s_kind_none']); $camp->setYouthSportType($youthYouthType); - $camp->clearEventDates(); foreach ($campJson['links']['dates'] ?? [] as $dateId) { $camp->addEventDate($this->eventDateMapper->mapFromJson($this->getLinked($linked, 'event_dates', $dateId), $camp)); } + $personEventsFetcher = new PersonEventsFetcher($this->em, $this->pbsApiService, $this->personRepository, $camp); + $personEvents = $personEventsFetcher->fetch($groupId, $accessToken); + foreach ($personEvents as $personEvent) { + $camp->addPerson($personEvent); + } + $camps[] = $camp; } return $camps; } + + public function clean(string $groupId) { + $this->eventDateRepository + ->createQueryBuilder('ed') + ->delete(EventDate::class, 'ed') + // TODO add layer_id during import + //->where('ed.layer_id = :layer_id') + //->setParameter('layer_id', $groupId) + ->getQuery() + ->execute(); + + $this->campRepository + ->createQueryBuilder('c') + ->delete(Camp::class, 'c') + // TODO add layer_id during import + //->where('c.layer_id = :layer_id') + //->setParameter('layer_id', $groupId) + ->getQuery() + ->execute(); + + $this->em->flush(); + } } diff --git a/src/Service/PbsApi/Fetcher/CoursesFetcher.php b/src/Service/PbsApi/Fetcher/CoursesFetcher.php index 3232ad7..c3dadc4 100644 --- a/src/Service/PbsApi/Fetcher/CoursesFetcher.php +++ b/src/Service/PbsApi/Fetcher/CoursesFetcher.php @@ -3,10 +3,12 @@ namespace App\Service\PbsApi\Fetcher; use App\Entity\Course; +use App\Entity\EventDate; use App\Entity\EventType; -use App\Entity\YouthSportType; use App\Repository\CourseRepository; +use App\Repository\EventDateRepository; use App\Repository\EventTypeRepository; +use App\Repository\PersonRepository; use App\Service\PbsApiService; use Doctrine\ORM\EntityManagerInterface; use Doctrine\ORM\Id\AssignedGenerator; @@ -25,12 +27,22 @@ class CoursesFetcher extends AbstractFetcher * @var EventDateMapper */ private $eventDateMapper; + /** + * @var PersonRepository + */ + private $personRepository; + /** + * @var EventDateRepository + */ + private $eventDateRepository; - public function __construct(EntityManagerInterface $em, PbsApiService $pbsApiService, EventDateMapper $eventDateMapper) { + public function __construct(EntityManagerInterface $em, PbsApiService $pbsApiService, EventDateMapper $eventDateMapper, PersonRepository $personRepository, EventDateRepository $eventDateRepository) { parent::__construct($em, $pbsApiService); $this->courseRepository = $this->em->getRepository(Course::class); $this->eventTypeRepository = $this->em->getRepository(EventType::class); $this->eventDateMapper = $eventDateMapper; + $this->personRepository = $personRepository; + $this->eventDateRepository = $eventDateRepository; } protected function fetch(string $groupId, string $accessToken): array @@ -38,10 +50,10 @@ protected function fetch(string $groupId, string $accessToken): array $startDate = date('d-m-Y', strtotime('-10 years')); $endDate = date('d-m-Y', strtotime('+5 years')); $courseData = $this->pbsApiService->getApiData('/groups/'.$groupId.'/events?type=Event::Course&start_date='.$startDate.'&end_date='.$endDate, $accessToken); - return $this->mapJsonToCourses($courseData); + return $this->mapJsonToCourses($courseData, $groupId, $accessToken); } - private function mapJsonToCourses(array $json): array + private function mapJsonToCourses(array $json, string $groupId, string $accessToken): array { $coursesJson = $json['events'] ?? []; $linked = $json['linked'] ?? []; @@ -62,14 +74,41 @@ private function mapJsonToCourses(array $json): array $eventType = $this->eventTypeRepository->findOneBy(['id' => $courseJson['links']['kind']]); $course->setEventType($eventType); - $course->clearEventDates(); foreach ($courseJson['links']['dates'] ?? [] as $dateId) { $course->addEventDate($this->eventDateMapper->mapFromJson($this->getLinked($linked, 'event_dates', $dateId), $course)); } + $personEventsFetcher = new PersonEventsFetcher($this->em, $this->pbsApiService, $this->personRepository, $course); + $personEvents = $personEventsFetcher->fetch($groupId, $accessToken); + foreach ($personEvents as $personEvent) { + $course->addPerson($personEvent); + } + $courses[] = $course; } return $courses; } + + public function clean(string $groupId) { + $this->eventDateRepository + ->createQueryBuilder('ed') + ->delete(EventDate::class, 'ed') + // TODO add layer_id during import + //->where('ed.layer_id = :layer_id') + //->setParameter('layer_id', $groupId) + ->getQuery() + ->execute(); + + $this->courseRepository + ->createQueryBuilder('c') + ->delete(Course::class, 'c') + // TODO add layer_id during import + //->where('c.layer_id = :layer_id') + //->setParameter('layer_id', $groupId) + ->getQuery() + ->execute(); + + $this->em->flush(); + } } diff --git a/src/Service/PbsApi/Fetcher/GroupFetcher.php b/src/Service/PbsApi/Fetcher/GroupFetcher.php index 8518a61..2271bba 100644 --- a/src/Service/PbsApi/Fetcher/GroupFetcher.php +++ b/src/Service/PbsApi/Fetcher/GroupFetcher.php @@ -88,4 +88,15 @@ protected function fetch(string $groupId, string $accessToken): array { // Not implemented because not needed return []; } + + public function clean(string $groupId) { + $this->groupRepository + ->createQueryBuilder('g') + ->delete(Group::class, 'g') + ->where('g.id = :id') + ->setParameter('id', $groupId) + ->getQuery() + ->execute(); + $this->em->flush(); + } } diff --git a/src/Service/PbsApi/Fetcher/PeopleFetcher.php b/src/Service/PbsApi/Fetcher/PeopleFetcher.php index bc85c0b..cb54780 100644 --- a/src/Service/PbsApi/Fetcher/PeopleFetcher.php +++ b/src/Service/PbsApi/Fetcher/PeopleFetcher.php @@ -54,25 +54,25 @@ private function mapJsonToPeople(array $json): array $metadata->setIdGenerator(new AssignedGenerator()); } $person->setNickname($personJson['nickname']); - $person->setGender($personJson['gender']); + $person->setGender($personJson['gender'] ?? null); $person->setAddress($personJson['address']); $person->setCountry($personJson['country']); $person->setZip(intval($personJson['zip_code'])); - if ($personJson['birthday']) { + if ($personJson['birthday'] ?? false) { $person->setBirthday(new DateTimeImmutable($personJson['birthday'])); } - $person->setPbsNumber($personJson['pbs_number']); - if ($personJson['entry_date']) { + $person->setPbsNumber($personJson['pbs_number'] ?? null); + if ($personJson['entry_date'] ?? false) { $person->setEntryDate(new DateTimeImmutable($personJson['entry_date'])); } - if ($personJson['leaving_date']) { + if ($personJson['leaving_date'] ?? false) { $person->setLeavingDate(new DateTimeImmutable($personJson['leaving_date'])); } else { $person->setLeavingDate(null); } $person->setTown($personJson['town']); - if ($personJson['primary_group_id']) { + if ($personJson['primary_group_id'] ?? false) { /** @var Group $group */ $group = $this->groupRepository->find($personJson['primary_group_id']); if ($group) { @@ -80,9 +80,6 @@ private function mapJsonToPeople(array $json): array } } - // TODO this may also clear any roles of the same person in another Abteilung. - // We need to import and store the same person separately for each Abteilung they're in. - $person->clearPersonRoles(); foreach ($personJson['links']['roles'] ?? [] as $roleId) { $person->addPersonRole($this->roleMapper->mapFromJson($this->getLinked($linked, 'roles', $roleId), $person)); } @@ -92,4 +89,16 @@ private function mapJsonToPeople(array $json): array return $people; } + + public function clean(string $groupId) { + $this->personRepository + ->createQueryBuilder('p') + ->delete(Person::class, 'p') + // TODO add layer_id during import + //->where('p.layer_id = :layer_id') + //->setParameter('layer_id', $groupId) + ->getQuery() + ->execute(); + $this->em->flush(); + } } diff --git a/src/Service/PbsApi/Fetcher/PersonEventsFetcher.php b/src/Service/PbsApi/Fetcher/PersonEventsFetcher.php new file mode 100644 index 0000000..e238a57 --- /dev/null +++ b/src/Service/PbsApi/Fetcher/PersonEventsFetcher.php @@ -0,0 +1,81 @@ +personEventRepository = $this->em->getRepository(PersonEvent::class); + $this->event = $event; + $this->personRepository = $personRepository; + } + + public function fetch(string $groupId, string $accessToken): array + { + $personEvents = $this->pbsApiService->getApiData('/groups/'.$groupId.'/events/'.$this->event->getId().'/participations', $accessToken); + return $this->mapJsonToPersonEvents($personEvents, $this->event); + } + + private function mapJsonToPersonEvents(array $json, Event $event): array + { + $personEventsJson = $json['event_participations'] ?? []; + $linked = $json['linked'] ?? []; + + $personEvents = []; + foreach ($personEventsJson as $personEventJson) { + $personEvent = $this->personEventRepository->findOneBy(['id' => $personEventJson['id']]); + if (!$personEvent) { + $personEvent = new PersonEvent(); + $personEvent->setId($personEventJson['id']); + $metadata = $this->em->getClassMetaData(get_class($personEvent)); + $metadata->setIdGenerator(new AssignedGenerator()); + } + $personEvent->setQualified($personEventJson['qualified'] ?? null); + + $personEvent->setEvent($event); + + /** @var Person $person */ + $person = $this->personRepository->findOneBy(['id' => $personEventJson['links']['person']]); + $personEvent->setPerson($person); + + $personEvents[] = $personEvent; + } + + return $personEvents; + } + + public function clean(string $groupId) { + $this->personEventRepository + ->createQueryBuilder('pe') + ->delete(PersonEvent::class, 'pe') + // TODO add layer_id during import + //->where('pe.layer_id = :layer_id') + //->setParameter('layer_id', $groupId) + ->getQuery() + ->execute(); + $this->em->flush(); + } +} diff --git a/src/Service/SyncService.php b/src/Service/SyncService.php index 703dee7..36a9224 100644 --- a/src/Service/SyncService.php +++ b/src/Service/SyncService.php @@ -55,6 +55,11 @@ public function __construct(EntityManagerInterface $em, PbsApiService $pbsApiSer */ public function startSync(int $groupId, $accessToken) { + $this->coursesFetcher->clean($groupId); + $this->campsFetcher->clean($groupId); + $this->peopleFetcher->clean($groupId); + $this->groupFetcher->clean($groupId); + $this->groupFetcher->fetchAndPersistGroup($groupId, $accessToken); $this->peopleFetcher->fetchAndPersist($groupId, $accessToken); $this->campsFetcher->fetchAndPersist($groupId, $accessToken); From 17c940ce55a0d1f2daa99f35162c73c0aeb9f0e1 Mon Sep 17 00:00:00 2001 From: Carlo Beltrame Date: Tue, 28 Sep 2021 21:25:48 +0200 Subject: [PATCH 10/12] Properly separate data of different Abteilungen Each sensitive data point is marked with the id of the sync group (Abteilung for now, maybe canton in the future). Also, the id column in the tables is no longer misused to store the id from MiData, because we potentially need to store the same entity twice for different sync groups (e.g. when the same person is in multiple Abteilungen). --- src/Entity/Event.php | 39 ++++++ src/Entity/EventDate.php | 39 ++++++ src/Entity/Group.php | 39 ++++++ src/Entity/Person.php | 39 ++++++ src/Entity/PersonEvent.php | 41 +++++- src/Entity/PersonRole.php | 39 ++++++ src/Migrations/Version20210925185300.php | 51 -------- src/Migrations/Version20210928123831.php | 36 ++++++ src/Migrations/Version20210928134409.php | 119 ++++++++++++++++++ .../PbsApi/Fetcher/AbstractFetcher.php | 6 +- src/Service/PbsApi/Fetcher/CampsFetcher.php | 36 +++--- src/Service/PbsApi/Fetcher/CoursesFetcher.php | 36 +++--- .../PbsApi/Fetcher/EventDateMapper.php | 11 +- src/Service/PbsApi/Fetcher/GroupFetcher.php | 39 +++--- src/Service/PbsApi/Fetcher/PeopleFetcher.php | 25 ++-- .../PbsApi/Fetcher/PersonEventsFetcher.php | 25 ++-- .../PbsApi/Fetcher/PersonRoleMapper.php | 10 +- src/Service/SyncService.php | 8 +- 18 files changed, 479 insertions(+), 159 deletions(-) delete mode 100644 src/Migrations/Version20210925185300.php create mode 100644 src/Migrations/Version20210928123831.php create mode 100644 src/Migrations/Version20210928134409.php diff --git a/src/Entity/Event.php b/src/Entity/Event.php index bf71cc9..c9e09a6 100644 --- a/src/Entity/Event.php +++ b/src/Entity/Event.php @@ -10,6 +10,8 @@ * @ORM\Entity * @ORM\Table(name="midata_event", indexes={ * @ORM\Index(columns={"name"}) + * }, uniqueConstraints={ + * @ORM\UniqueConstraint(columns={"midata_id", "sync_group_id"}) * }) * @ORM\InheritanceType("SINGLE_TABLE") * @ORM\DiscriminatorColumn(name="type", type="string") @@ -24,6 +26,17 @@ abstract class Event */ private $id; + /** + * @ORM\Column(type="integer") + */ + private $midataId; + + /** + * @ORM\ManyToOne(targetEntity="Group") + * @ORM\JoinColumn(name="sync_group_id", referencedColumnName="id", nullable=false, onDelete="CASCADE") + */ + private $syncGroup; + /** * @ORM\Column(type="string", length=255) */ @@ -71,6 +84,32 @@ public function getId() return $this->id; } + /** + * @return mixed + */ + public function getMidataId() + { + return $this->midataId; + } + + /** + * @param int $midataId + */ + public function setMidataId(int $midataId) + { + $this->midataId = $midataId; + } + + public function getSyncGroup(): Group + { + return $this->syncGroup; + } + + public function setSyncGroup($syncGroup) + { + $this->syncGroup = $syncGroup; + } + /** * @return null|string */ diff --git a/src/Entity/EventDate.php b/src/Entity/EventDate.php index 31b72bd..abf805b 100644 --- a/src/Entity/EventDate.php +++ b/src/Entity/EventDate.php @@ -9,6 +9,8 @@ * @ORM\Table(name="midata_event_date", indexes={ * @ORM\Index(columns={"start_at"}), * @ORM\Index(columns={"end_at"}) + * }, uniqueConstraints={ + * @ORM\UniqueConstraint(columns={"midata_id", "sync_group_id"}) * }) * @ORM\Entity(repositoryClass="App\Repository\EventDateRepository") * @ORM\HasLifecycleCallbacks() @@ -22,6 +24,17 @@ class EventDate */ private $id; + /** + * @ORM\Column(type="integer") + */ + private $midataId; + + /** + * @ORM\ManyToOne(targetEntity="Group") + * @ORM\JoinColumn(name="sync_group_id", referencedColumnName="id", nullable=false, onDelete="CASCADE") + */ + private $syncGroup; + /** * @ORM\ManyToOne(targetEntity="Event") * @ORM\JoinColumn(name="event_id", referencedColumnName="id", onDelete="CASCADE") @@ -54,6 +67,32 @@ public function getId() return $this->id; } + /** + * @return mixed + */ + public function getMidataId() + { + return $this->midataId; + } + + /** + * @param int $midataId + */ + public function setMidataId(int $midataId) + { + $this->midataId = $midataId; + } + + public function getSyncGroup(): Group + { + return $this->syncGroup; + } + + public function setSyncGroup($syncGroup) + { + $this->syncGroup = $syncGroup; + } + /** * @return DateTimeImmutable|null */ diff --git a/src/Entity/Group.php b/src/Entity/Group.php index a5f906a..0d32f74 100644 --- a/src/Entity/Group.php +++ b/src/Entity/Group.php @@ -11,6 +11,8 @@ * @ORM\Index(columns={"name"}), * @ORM\Index(columns={"created_at"}), * @ORM\Index(columns={"deleted_at"}) + * }, uniqueConstraints={ + * @ORM\UniqueConstraint(columns={"midata_id", "sync_group_id"}) * }) * @ORM\Entity(repositoryClass="App\Repository\GroupRepository") * @ORM\HasLifecycleCallbacks() @@ -24,6 +26,17 @@ class Group */ private $id; + /** + * @ORM\Column(type="integer") + */ + private $midataId; + + /** + * @ORM\ManyToOne(targetEntity="Group") + * @ORM\JoinColumn(name="sync_group_id", referencedColumnName="id", nullable=true, onDelete="CASCADE") + */ + private $syncGroup; + /** * @ORM\OneToMany(targetEntity="Group", mappedBy="parentGroup", cascade={"persist"}) */ @@ -99,6 +112,32 @@ public function getId() return $this->id; } + /** + * @return mixed + */ + public function getMidataId() + { + return $this->midataId; + } + + /** + * @param int $midataId + */ + public function setMidataId(int $midataId) + { + $this->midataId = $midataId; + } + + public function getSyncGroup(): Group + { + return $this->syncGroup; + } + + public function setSyncGroup($syncGroup) + { + $this->syncGroup = $syncGroup; + } + /** * @return null|string */ diff --git a/src/Entity/Person.php b/src/Entity/Person.php index 5c077ae..7a4a3f5 100644 --- a/src/Entity/Person.php +++ b/src/Entity/Person.php @@ -14,6 +14,8 @@ * @ORM\Index(columns={"birthday"}), * @ORM\Index(columns={"entry_date"}), * @ORM\Index(columns={"leaving_date"}) + * }, uniqueConstraints={ + * @ORM\UniqueConstraint(columns={"midata_id", "sync_group_id"}) * }) * @ORM\Entity(repositoryClass="App\Repository\PersonRepository") * @ORM\HasLifecycleCallbacks() @@ -31,6 +33,17 @@ class Person */ private $id; + /** + * @ORM\Column(type="integer") + */ + private $midataId; + + /** + * @ORM\ManyToOne(targetEntity="Group") + * @ORM\JoinColumn(name="sync_group_id", referencedColumnName="id", nullable=false, onDelete="CASCADE") + */ + private $syncGroup; + /** * @ORM\Column(type="string", length=255, nullable=true) */ @@ -129,6 +142,32 @@ public function getId() return $this->id; } + /** + * @return mixed + */ + public function getMidataId() + { + return $this->midataId; + } + + /** + * @param int $midataId + */ + public function setMidataId(int $midataId) + { + $this->midataId = $midataId; + } + + public function getSyncGroup(): Group + { + return $this->syncGroup; + } + + public function setSyncGroup($syncGroup) + { + $this->syncGroup = $syncGroup; + } + /** * @return null|string */ diff --git a/src/Entity/PersonEvent.php b/src/Entity/PersonEvent.php index 0da3379..3994bda 100644 --- a/src/Entity/PersonEvent.php +++ b/src/Entity/PersonEvent.php @@ -7,7 +7,9 @@ use Doctrine\ORM\Mapping as ORM; /** - * @ORM\Table(name="midata_person_event") + * @ORM\Table(name="midata_person_event", uniqueConstraints={ + * @ORM\UniqueConstraint(columns={"midata_id", "sync_group_id"}) + * }) * @ORM\Entity(repositoryClass="App\Repository\PersonEventRepository") * @ORM\HasLifecycleCallbacks() */ @@ -20,6 +22,17 @@ class PersonEvent */ private $id; + /** + * @ORM\Column(type="integer") + */ + private $midataId; + + /** + * @ORM\ManyToOne(targetEntity="Group") + * @ORM\JoinColumn(name="sync_group_id", referencedColumnName="id", nullable=false, onDelete="CASCADE") + */ + private $syncGroup; + /** * @ORM\ManyToOne(targetEntity="Event", inversedBy="persons") * @ORM\JoinColumn(nullable=false, onDelete="CASCADE") @@ -70,6 +83,32 @@ public function getId() return $this->id; } + /** + * @return mixed + */ + public function getMidataId() + { + return $this->midataId; + } + + /** + * @param int $midataId + */ + public function setMidataId(int $midataId) + { + $this->midataId = $midataId; + } + + public function getSyncGroup(): Group + { + return $this->syncGroup; + } + + public function setSyncGroup($syncGroup) + { + $this->syncGroup = $syncGroup; + } + /** * @param Person|null $person */ diff --git a/src/Entity/PersonRole.php b/src/Entity/PersonRole.php index 8354336..d61a1cb 100644 --- a/src/Entity/PersonRole.php +++ b/src/Entity/PersonRole.php @@ -9,6 +9,8 @@ * @ORM\Table(name="midata_person_role", indexes={ * @ORM\Index(columns={"created_at"}), * @ORM\Index(columns={"deleted_at"}) + * }, uniqueConstraints={ + * @ORM\UniqueConstraint(columns={"midata_id", "sync_group_id"}) * }) * @ORM\Entity(repositoryClass="App\Repository\PersonRoleRepository") * @ORM\HasLifecycleCallbacks() @@ -22,6 +24,17 @@ class PersonRole */ private $id; + /** + * @ORM\Column(type="integer") + */ + private $midataId; + + /** + * @ORM\ManyToOne(targetEntity="Group") + * @ORM\JoinColumn(name="sync_group_id", referencedColumnName="id", nullable=false, onDelete="CASCADE") + */ + private $syncGroup; + /** * @ORM\ManyToOne(targetEntity="Group", inversedBy="personRoles") * @ORM\JoinColumn(name="group_id", referencedColumnName="id", onDelete="CASCADE") @@ -71,6 +84,32 @@ public function getId() return $this->id; } + /** + * @return mixed + */ + public function getMidataId() + { + return $this->midataId; + } + + /** + * @param int $midataId + */ + public function setMidataId(int $midataId) + { + $this->midataId = $midataId; + } + + public function getSyncGroup(): Group + { + return $this->syncGroup; + } + + public function setSyncGroup($syncGroup) + { + $this->syncGroup = $syncGroup; + } + /** * @return null|string */ diff --git a/src/Migrations/Version20210925185300.php b/src/Migrations/Version20210925185300.php deleted file mode 100644 index 4bb5cb3..0000000 --- a/src/Migrations/Version20210925185300.php +++ /dev/null @@ -1,51 +0,0 @@ -addSql('ALTER TABLE midata_person ALTER nickname DROP NOT NULL'); - $this->addSql('ALTER TABLE midata_person_event ALTER person_id DROP NOT NULL'); - $this->addSql('ALTER TABLE midata_event_date DROP CONSTRAINT FK_E13DC41771F7E88B'); - $this->addSql('ALTER TABLE midata_event_date ADD CONSTRAINT FK_E13DC41771F7E88B FOREIGN KEY (event_id) REFERENCES midata_event (id) ON DELETE CASCADE NOT DEFERRABLE INITIALLY IMMEDIATE'); - $this->addSql('ALTER TABLE midata_person_event DROP CONSTRAINT FK_51E69FDE71F7E88B'); - $this->addSql('ALTER TABLE midata_person_event ADD CONSTRAINT FK_51E69FDE71F7E88B FOREIGN KEY (event_id) REFERENCES midata_event (id) ON DELETE CASCADE NOT DEFERRABLE INITIALLY IMMEDIATE'); - $this->addSql('ALTER TABLE midata_person_role DROP CONSTRAINT FK_AFDC017BFE54D947'); - $this->addSql('ALTER TABLE midata_person_role ADD CONSTRAINT FK_AFDC017BFE54D947 FOREIGN KEY (group_id) REFERENCES midata_group (id) ON DELETE CASCADE NOT DEFERRABLE INITIALLY IMMEDIATE'); - $this->addSql('ALTER TABLE midata_person_role DROP CONSTRAINT FK_AFDC017B217BBB47'); - $this->addSql('ALTER TABLE midata_person_role ADD CONSTRAINT FK_AFDC017B217BBB47 FOREIGN KEY (person_id) REFERENCES midata_person (id) ON DELETE CASCADE NOT DEFERRABLE INITIALLY IMMEDIATE'); - $this->addSql('ALTER TABLE midata_group DROP CONSTRAINT FK_2644E31861997596'); - $this->addSql('ALTER TABLE midata_group ADD CONSTRAINT FK_2644E31861997596 FOREIGN KEY (parent_group_id) REFERENCES midata_group (id) ON DELETE CASCADE NOT DEFERRABLE INITIALLY IMMEDIATE'); - } - - public function down(Schema $schema) : void - { - $this->addSql('ALTER TABLE midata_group DROP CONSTRAINT fk_2644e31861997596'); - $this->addSql('ALTER TABLE midata_group ADD CONSTRAINT fk_2644e31861997596 FOREIGN KEY (parent_group_id) REFERENCES midata_group (id) NOT DEFERRABLE INITIALLY IMMEDIATE'); - $this->addSql('ALTER TABLE midata_person_role DROP CONSTRAINT fk_afdc017bfe54d947'); - $this->addSql('ALTER TABLE midata_person_role ADD CONSTRAINT fk_afdc017bfe54d947 FOREIGN KEY (group_id) REFERENCES midata_group (id) NOT DEFERRABLE INITIALLY IMMEDIATE'); - $this->addSql('ALTER TABLE midata_person_role DROP CONSTRAINT fk_afdc017b217bbb47'); - $this->addSql('ALTER TABLE midata_person_role ADD CONSTRAINT fk_afdc017b217bbb47 FOREIGN KEY (person_id) REFERENCES midata_person (id) NOT DEFERRABLE INITIALLY IMMEDIATE'); - $this->addSql('ALTER TABLE midata_event_date DROP CONSTRAINT fk_e13dc41771f7e88b'); - $this->addSql('ALTER TABLE midata_event_date ADD CONSTRAINT fk_e13dc41771f7e88b FOREIGN KEY (event_id) REFERENCES midata_event (id) NOT DEFERRABLE INITIALLY IMMEDIATE'); - $this->addSql('ALTER TABLE midata_person_event DROP CONSTRAINT fk_51e69fde71f7e88b'); - $this->addSql('ALTER TABLE midata_person_event ADD CONSTRAINT fk_51e69fde71f7e88b FOREIGN KEY (event_id) REFERENCES midata_event (id) NOT DEFERRABLE INITIALLY IMMEDIATE'); - $this->addSql('ALTER TABLE midata_person_event ALTER person_id SET NOT NULL'); - $this->addSql('ALTER TABLE midata_person ALTER nickname SET NOT NULL'); - } -} diff --git a/src/Migrations/Version20210928123831.php b/src/Migrations/Version20210928123831.php new file mode 100644 index 0000000..0c9da32 --- /dev/null +++ b/src/Migrations/Version20210928123831.php @@ -0,0 +1,36 @@ +addSql('DELETE FROM midata_event_date'); + $this->addSql('DELETE FROM midata_person_event'); + $this->addSql('DELETE FROM midata_person_role'); + $this->addSql('DELETE FROM midata_event'); + $this->addSql('DELETE FROM midata_person'); + $this->addSql('DELETE FROM midata_group'); + } + + public function down(Schema $schema) : void + { + } +} diff --git a/src/Migrations/Version20210928134409.php b/src/Migrations/Version20210928134409.php new file mode 100644 index 0000000..ed17fb5 --- /dev/null +++ b/src/Migrations/Version20210928134409.php @@ -0,0 +1,119 @@ +addSql('ALTER TABLE midata_event ADD sync_group_id INT NOT NULL'); + $this->addSql('ALTER TABLE midata_event ADD midata_id INT NOT NULL'); + $this->addSql('ALTER TABLE midata_event ADD CONSTRAINT FK_702AAD7ABF466769 FOREIGN KEY (sync_group_id) REFERENCES midata_group (id) ON DELETE CASCADE NOT DEFERRABLE INITIALLY IMMEDIATE'); + $this->addSql('CREATE INDEX IDX_702AAD7ABF466769 ON midata_event (sync_group_id)'); + $this->addSql('CREATE UNIQUE INDEX UNIQ_702AAD7A2E18B78FBF466769 ON midata_event (midata_id, sync_group_id)'); + $this->addSql('ALTER TABLE midata_event_date DROP CONSTRAINT FK_E13DC41771F7E88B'); + $this->addSql('ALTER TABLE midata_event_date ADD sync_group_id INT NOT NULL'); + $this->addSql('ALTER TABLE midata_event_date ADD midata_id INT NOT NULL'); + $this->addSql('ALTER TABLE midata_event_date ADD CONSTRAINT FK_E13DC417BF466769 FOREIGN KEY (sync_group_id) REFERENCES midata_group (id) ON DELETE CASCADE NOT DEFERRABLE INITIALLY IMMEDIATE'); + $this->addSql('ALTER TABLE midata_event_date ADD CONSTRAINT FK_E13DC41771F7E88B FOREIGN KEY (event_id) REFERENCES midata_event (id) ON DELETE CASCADE NOT DEFERRABLE INITIALLY IMMEDIATE'); + $this->addSql('CREATE INDEX IDX_E13DC417BF466769 ON midata_event_date (sync_group_id)'); + $this->addSql('CREATE UNIQUE INDEX UNIQ_E13DC4172E18B78FBF466769 ON midata_event_date (midata_id, sync_group_id)'); + $this->addSql('ALTER TABLE midata_group DROP CONSTRAINT FK_2644E31861997596'); + $this->addSql('ALTER TABLE midata_group ADD sync_group_id INT NULL'); + $this->addSql('ALTER TABLE midata_group ADD midata_id INT NOT NULL'); + $this->addSql('ALTER TABLE midata_group ADD CONSTRAINT FK_2644E318BF466769 FOREIGN KEY (sync_group_id) REFERENCES midata_group (id) ON DELETE CASCADE NOT DEFERRABLE INITIALLY IMMEDIATE'); + $this->addSql('ALTER TABLE midata_group ADD CONSTRAINT FK_2644E31861997596 FOREIGN KEY (parent_group_id) REFERENCES midata_group (id) ON DELETE CASCADE NOT DEFERRABLE INITIALLY IMMEDIATE'); + $this->addSql('CREATE INDEX IDX_2644E318BF466769 ON midata_group (sync_group_id)'); + $this->addSql('CREATE UNIQUE INDEX UNIQ_2644E3182E18B78FBF466769 ON midata_group (midata_id, sync_group_id)'); + $this->addSql('ALTER TABLE midata_person ADD sync_group_id INT NOT NULL'); + $this->addSql('ALTER TABLE midata_person ADD midata_id INT NOT NULL'); + $this->addSql('ALTER TABLE midata_person ALTER nickname DROP NOT NULL'); + $this->addSql('ALTER TABLE midata_person ADD CONSTRAINT FK_CCF5FBB8BF466769 FOREIGN KEY (sync_group_id) REFERENCES midata_group (id) ON DELETE CASCADE NOT DEFERRABLE INITIALLY IMMEDIATE'); + $this->addSql('CREATE INDEX IDX_CCF5FBB8BF466769 ON midata_person (sync_group_id)'); + $this->addSql('CREATE UNIQUE INDEX UNIQ_CCF5FBB82E18B78FBF466769 ON midata_person (midata_id, sync_group_id)'); + $this->addSql('ALTER TABLE midata_person_event DROP CONSTRAINT FK_51E69FDE71F7E88B'); + $this->addSql('ALTER TABLE midata_person_event ADD sync_group_id INT NOT NULL'); + $this->addSql('ALTER TABLE midata_person_event ADD midata_id INT NOT NULL'); + $this->addSql('ALTER TABLE midata_person_event ALTER person_id DROP NOT NULL'); + $this->addSql('ALTER TABLE midata_person_event ADD CONSTRAINT FK_51E69FDEBF466769 FOREIGN KEY (sync_group_id) REFERENCES midata_group (id) ON DELETE CASCADE NOT DEFERRABLE INITIALLY IMMEDIATE'); + $this->addSql('ALTER TABLE midata_person_event ADD CONSTRAINT FK_51E69FDE71F7E88B FOREIGN KEY (event_id) REFERENCES midata_event (id) ON DELETE CASCADE NOT DEFERRABLE INITIALLY IMMEDIATE'); + $this->addSql('CREATE INDEX IDX_51E69FDEBF466769 ON midata_person_event (sync_group_id)'); + $this->addSql('CREATE UNIQUE INDEX UNIQ_51E69FDE2E18B78FBF466769 ON midata_person_event (midata_id, sync_group_id)'); + $this->addSql('ALTER TABLE midata_person_role DROP CONSTRAINT FK_AFDC017BFE54D947'); + $this->addSql('ALTER TABLE midata_person_role DROP CONSTRAINT FK_AFDC017B217BBB47'); + $this->addSql('ALTER TABLE midata_person_role ADD sync_group_id INT NOT NULL'); + $this->addSql('ALTER TABLE midata_person_role ADD midata_id INT NOT NULL'); + $this->addSql('ALTER TABLE midata_person_role ADD CONSTRAINT FK_AFDC017BBF466769 FOREIGN KEY (sync_group_id) REFERENCES midata_group (id) ON DELETE CASCADE NOT DEFERRABLE INITIALLY IMMEDIATE'); + $this->addSql('ALTER TABLE midata_person_role ADD CONSTRAINT FK_AFDC017BFE54D947 FOREIGN KEY (group_id) REFERENCES midata_group (id) ON DELETE CASCADE NOT DEFERRABLE INITIALLY IMMEDIATE'); + $this->addSql('ALTER TABLE midata_person_role ADD CONSTRAINT FK_AFDC017B217BBB47 FOREIGN KEY (person_id) REFERENCES midata_person (id) ON DELETE CASCADE NOT DEFERRABLE INITIALLY IMMEDIATE'); + $this->addSql('CREATE INDEX IDX_AFDC017BBF466769 ON midata_person_role (sync_group_id)'); + $this->addSql('CREATE UNIQUE INDEX UNIQ_AFDC017B2E18B78FBF466769 ON midata_person_role (midata_id, sync_group_id)'); + } + + public function down(Schema $schema) : void + { + // this down() migration is auto-generated, please modify it to your needs + $this->addSql('ALTER TABLE midata_event_date DROP CONSTRAINT FK_E13DC417BF466769'); + $this->addSql('ALTER TABLE midata_event_date DROP CONSTRAINT fk_e13dc41771f7e88b'); + $this->addSql('DROP INDEX IDX_E13DC417BF466769'); + $this->addSql('DROP INDEX UNIQ_E13DC4172E18B78FBF466769'); + $this->addSql('ALTER TABLE midata_event_date DROP sync_group_id'); + $this->addSql('ALTER TABLE midata_event_date DROP midata_id'); + $this->addSql('ALTER TABLE midata_event_date ADD CONSTRAINT fk_e13dc41771f7e88b FOREIGN KEY (event_id) REFERENCES midata_event (id) NOT DEFERRABLE INITIALLY IMMEDIATE'); + $this->addSql('ALTER TABLE midata_event DROP CONSTRAINT FK_702AAD7ABF466769'); + $this->addSql('DROP INDEX IDX_702AAD7ABF466769'); + $this->addSql('DROP INDEX UNIQ_702AAD7A2E18B78FBF466769'); + $this->addSql('ALTER TABLE midata_event DROP sync_group_id'); + $this->addSql('ALTER TABLE midata_event DROP midata_id'); + $this->addSql('ALTER TABLE midata_person_event DROP CONSTRAINT FK_51E69FDEBF466769'); + $this->addSql('ALTER TABLE midata_person_event DROP CONSTRAINT fk_51e69fde71f7e88b'); + $this->addSql('DROP INDEX IDX_51E69FDEBF466769'); + $this->addSql('DROP INDEX UNIQ_51E69FDE2E18B78FBF466769'); + $this->addSql('ALTER TABLE midata_person_event DROP sync_group_id'); + $this->addSql('ALTER TABLE midata_person_event DROP midata_id'); + $this->addSql('ALTER TABLE midata_person_event ALTER person_id SET NOT NULL'); + $this->addSql('ALTER TABLE midata_person_event ADD CONSTRAINT fk_51e69fde71f7e88b FOREIGN KEY (event_id) REFERENCES midata_event (id) NOT DEFERRABLE INITIALLY IMMEDIATE'); + $this->addSql('ALTER TABLE midata_person_role DROP CONSTRAINT FK_AFDC017BBF466769'); + $this->addSql('ALTER TABLE midata_person_role DROP CONSTRAINT fk_afdc017bfe54d947'); + $this->addSql('ALTER TABLE midata_person_role DROP CONSTRAINT fk_afdc017b217bbb47'); + $this->addSql('DROP INDEX IDX_AFDC017BBF466769'); + $this->addSql('DROP INDEX UNIQ_AFDC017B2E18B78FBF466769'); + $this->addSql('ALTER TABLE midata_person_role DROP sync_group_id'); + $this->addSql('ALTER TABLE midata_person_role DROP midata_id'); + $this->addSql('ALTER TABLE midata_person_role ADD CONSTRAINT fk_afdc017bfe54d947 FOREIGN KEY (group_id) REFERENCES midata_group (id) NOT DEFERRABLE INITIALLY IMMEDIATE'); + $this->addSql('ALTER TABLE midata_person_role ADD CONSTRAINT fk_afdc017b217bbb47 FOREIGN KEY (person_id) REFERENCES midata_person (id) NOT DEFERRABLE INITIALLY IMMEDIATE'); + $this->addSql('ALTER TABLE midata_group DROP CONSTRAINT FK_2644E318BF466769'); + $this->addSql('ALTER TABLE midata_group DROP CONSTRAINT fk_2644e31861997596'); + $this->addSql('DROP INDEX IDX_2644E318BF466769'); + $this->addSql('DROP INDEX UNIQ_2644E3182E18B78FBF466769'); + $this->addSql('ALTER TABLE midata_group DROP sync_group_id'); + $this->addSql('ALTER TABLE midata_group DROP midata_id'); + $this->addSql('ALTER TABLE midata_group ADD CONSTRAINT fk_2644e31861997596 FOREIGN KEY (parent_group_id) REFERENCES midata_group (id) NOT DEFERRABLE INITIALLY IMMEDIATE'); + $this->addSql('ALTER TABLE midata_person DROP CONSTRAINT FK_CCF5FBB8BF466769'); + $this->addSql('DROP INDEX IDX_CCF5FBB8BF466769'); + $this->addSql('DROP INDEX UNIQ_CCF5FBB82E18B78FBF466769'); + $this->addSql('ALTER TABLE midata_person DROP sync_group_id'); + $this->addSql('ALTER TABLE midata_person DROP midata_id'); + $this->addSql('ALTER TABLE midata_person ALTER nickname SET NOT NULL'); + } +} diff --git a/src/Service/PbsApi/Fetcher/AbstractFetcher.php b/src/Service/PbsApi/Fetcher/AbstractFetcher.php index 36fa88e..b53eb60 100644 --- a/src/Service/PbsApi/Fetcher/AbstractFetcher.php +++ b/src/Service/PbsApi/Fetcher/AbstractFetcher.php @@ -31,10 +31,10 @@ public function __construct(EntityManagerInterface $em, PbsApiService $pbsApiSer $this->pbsApiService = $pbsApiService; } - public function fetchAndPersist(string $groupId, string $accessToken) + public function fetchAndPersist(Group $syncGroup, string $accessToken) { $i = 0; - foreach ($this->fetch($groupId, $accessToken) as $entity) { + foreach ($this->fetch($syncGroup, $accessToken) as $entity) { $this->em->persist($entity); $i++; @@ -46,7 +46,7 @@ public function fetchAndPersist(string $groupId, string $accessToken) $this->em->flush(); } - protected abstract function fetch(string $groupId, string $accessToken): array; + protected abstract function fetch(Group $syncGroup, string $accessToken): array; public abstract function clean(string $groupId); diff --git a/src/Service/PbsApi/Fetcher/CampsFetcher.php b/src/Service/PbsApi/Fetcher/CampsFetcher.php index 8dee6f3..b4b671f 100644 --- a/src/Service/PbsApi/Fetcher/CampsFetcher.php +++ b/src/Service/PbsApi/Fetcher/CampsFetcher.php @@ -4,6 +4,7 @@ use App\Entity\Camp; use App\Entity\EventDate; +use App\Entity\Group; use App\Entity\YouthSportType; use App\Repository\CampRepository; use App\Repository\EventDateRepository; @@ -11,7 +12,6 @@ use App\Repository\YouthSportTypeRepository; use App\Service\PbsApiService; use Doctrine\ORM\EntityManagerInterface; -use Doctrine\ORM\Id\AssignedGenerator; class CampsFetcher extends AbstractFetcher { @@ -45,27 +45,27 @@ public function __construct(EntityManagerInterface $em, PbsApiService $pbsApiSer $this->eventDateRepository = $eventDateRepository; } - protected function fetch(string $groupId, string $accessToken): array + protected function fetch(Group $syncGroup, string $accessToken): array { + $groupId = $syncGroup->getMidataId(); $startDate = date('d-m-Y', strtotime('-10 years')); $endDate = date('d-m-Y', strtotime('+5 years')); $campData = $this->pbsApiService->getApiData('/groups/'.$groupId.'/events?type=Event::Camp&start_date='.$startDate.'&end_date='.$endDate, $accessToken); - return $this->mapJsonToCamps($campData, $groupId, $accessToken); + return $this->mapJsonToCamps($campData, $syncGroup, $accessToken); } - private function mapJsonToCamps(array $json, string $groupId, string $accessToken): array + private function mapJsonToCamps(array $json, Group $syncGroup, string $accessToken): array { $campsJson = $json['events'] ?? []; $linked = $json['linked'] ?? []; $camps = []; foreach ($campsJson as $campJson) { - $camp = $this->campRepository->findOneBy(['id' => $campJson['id']]); + $camp = $this->campRepository->findOneBy(['midataId' => $campJson['id'], 'syncGroup' => $syncGroup]); if (!$camp) { $camp = new Camp(); - $camp->setId($campJson['id']); - $metadata = $this->em->getClassMetaData(get_class($camp)); - $metadata->setIdGenerator(new AssignedGenerator()); + $camp->setMidataId($campJson['id']); + $camp->setSyncGroup($syncGroup); } $camp->setName($campJson['name']); $camp->setLocation($campJson['location']); @@ -76,11 +76,11 @@ private function mapJsonToCamps(array $json, string $groupId, string $accessToke $camp->setYouthSportType($youthYouthType); foreach ($campJson['links']['dates'] ?? [] as $dateId) { - $camp->addEventDate($this->eventDateMapper->mapFromJson($this->getLinked($linked, 'event_dates', $dateId), $camp)); + $camp->addEventDate($this->eventDateMapper->mapFromJson($this->getLinked($linked, 'event_dates', $dateId), $camp, $syncGroup)); } $personEventsFetcher = new PersonEventsFetcher($this->em, $this->pbsApiService, $this->personRepository, $camp); - $personEvents = $personEventsFetcher->fetch($groupId, $accessToken); + $personEvents = $personEventsFetcher->fetch($syncGroup, $accessToken); foreach ($personEvents as $personEvent) { $camp->addPerson($personEvent); } @@ -92,21 +92,17 @@ private function mapJsonToCamps(array $json, string $groupId, string $accessToke } public function clean(string $groupId) { - $this->eventDateRepository - ->createQueryBuilder('ed') + $this->em->createQueryBuilder() ->delete(EventDate::class, 'ed') - // TODO add layer_id during import - //->where('ed.layer_id = :layer_id') - //->setParameter('layer_id', $groupId) + ->where('ed.syncGroup = :sync_group_id') + ->setParameter('sync_group_id', $groupId) ->getQuery() ->execute(); - $this->campRepository - ->createQueryBuilder('c') + $this->em->createQueryBuilder() ->delete(Camp::class, 'c') - // TODO add layer_id during import - //->where('c.layer_id = :layer_id') - //->setParameter('layer_id', $groupId) + ->where('c.syncGroup = :sync_group_id') + ->setParameter('sync_group_id', $groupId) ->getQuery() ->execute(); diff --git a/src/Service/PbsApi/Fetcher/CoursesFetcher.php b/src/Service/PbsApi/Fetcher/CoursesFetcher.php index c3dadc4..2c79102 100644 --- a/src/Service/PbsApi/Fetcher/CoursesFetcher.php +++ b/src/Service/PbsApi/Fetcher/CoursesFetcher.php @@ -5,13 +5,13 @@ use App\Entity\Course; use App\Entity\EventDate; use App\Entity\EventType; +use App\Entity\Group; use App\Repository\CourseRepository; use App\Repository\EventDateRepository; use App\Repository\EventTypeRepository; use App\Repository\PersonRepository; use App\Service\PbsApiService; use Doctrine\ORM\EntityManagerInterface; -use Doctrine\ORM\Id\AssignedGenerator; class CoursesFetcher extends AbstractFetcher { @@ -45,15 +45,16 @@ public function __construct(EntityManagerInterface $em, PbsApiService $pbsApiSer $this->eventDateRepository = $eventDateRepository; } - protected function fetch(string $groupId, string $accessToken): array + protected function fetch(Group $syncGroup, string $accessToken): array { + $groupId = $syncGroup->getMidataId(); $startDate = date('d-m-Y', strtotime('-10 years')); $endDate = date('d-m-Y', strtotime('+5 years')); $courseData = $this->pbsApiService->getApiData('/groups/'.$groupId.'/events?type=Event::Course&start_date='.$startDate.'&end_date='.$endDate, $accessToken); - return $this->mapJsonToCourses($courseData, $groupId, $accessToken); + return $this->mapJsonToCourses($courseData, $syncGroup, $accessToken); } - private function mapJsonToCourses(array $json, string $groupId, string $accessToken): array + private function mapJsonToCourses(array $json, Group $syncGroup, string $accessToken): array { $coursesJson = $json['events'] ?? []; $linked = $json['linked'] ?? []; @@ -61,12 +62,11 @@ private function mapJsonToCourses(array $json, string $groupId, string $accessTo $courses = []; foreach ($coursesJson as $courseJson) { /** @var Course $course */ - $course = $this->courseRepository->findOneBy(['id' => $courseJson['id']]); + $course = $this->courseRepository->findOneBy(['midataId' => $courseJson['id'], 'syncGroup' => $syncGroup]); if (!$course) { $course = new Course(); - $course->setId($courseJson['id']); - $metadata = $this->em->getClassMetaData(get_class($course)); - $metadata->setIdGenerator(new AssignedGenerator()); + $course->setMidataId($courseJson['id']); + $course->setSyncGroup($syncGroup); } $course->setName($courseJson['name']); @@ -75,11 +75,11 @@ private function mapJsonToCourses(array $json, string $groupId, string $accessTo $course->setEventType($eventType); foreach ($courseJson['links']['dates'] ?? [] as $dateId) { - $course->addEventDate($this->eventDateMapper->mapFromJson($this->getLinked($linked, 'event_dates', $dateId), $course)); + $course->addEventDate($this->eventDateMapper->mapFromJson($this->getLinked($linked, 'event_dates', $dateId), $course, $syncGroup)); } $personEventsFetcher = new PersonEventsFetcher($this->em, $this->pbsApiService, $this->personRepository, $course); - $personEvents = $personEventsFetcher->fetch($groupId, $accessToken); + $personEvents = $personEventsFetcher->fetch($syncGroup, $accessToken); foreach ($personEvents as $personEvent) { $course->addPerson($personEvent); } @@ -91,21 +91,17 @@ private function mapJsonToCourses(array $json, string $groupId, string $accessTo } public function clean(string $groupId) { - $this->eventDateRepository - ->createQueryBuilder('ed') + $this->em->createQueryBuilder() ->delete(EventDate::class, 'ed') - // TODO add layer_id during import - //->where('ed.layer_id = :layer_id') - //->setParameter('layer_id', $groupId) + ->where('ed.syncGroup = :sync_group_id') + ->setParameter('sync_group_id', $groupId) ->getQuery() ->execute(); - $this->courseRepository - ->createQueryBuilder('c') + $this->em->createQueryBuilder() ->delete(Course::class, 'c') - // TODO add layer_id during import - //->where('c.layer_id = :layer_id') - //->setParameter('layer_id', $groupId) + ->where('c.syncGroup = :sync_group_id') + ->setParameter('sync_group_id', $groupId) ->getQuery() ->execute(); diff --git a/src/Service/PbsApi/Fetcher/EventDateMapper.php b/src/Service/PbsApi/Fetcher/EventDateMapper.php index a645741..cff2d58 100644 --- a/src/Service/PbsApi/Fetcher/EventDateMapper.php +++ b/src/Service/PbsApi/Fetcher/EventDateMapper.php @@ -4,11 +4,11 @@ use App\Entity\Event; use App\Entity\EventDate; +use App\Entity\Group; use App\Entity\PersonRole; use App\Repository\EventDateRepository; use DateTimeImmutable; use Doctrine\ORM\EntityManagerInterface; -use Doctrine\ORM\Id\AssignedGenerator; class EventDateMapper { @@ -31,14 +31,13 @@ public function __construct(EntityManagerInterface $em) { * @return PersonRole|null * @throws \Exception */ - public function mapFromJson(array $dateJson, Event $event): ?EventDate + public function mapFromJson(array $dateJson, Event $event, Group $syncGroup): ?EventDate { - $eventDate = $this->eventDateRepository->findOneBy(['id' => $dateJson['id']]); + $eventDate = $this->eventDateRepository->findOneBy(['midataId' => $dateJson['id'], 'syncGroup' => $syncGroup]); if (!$eventDate) { $eventDate = new EventDate(); - $eventDate->setId($dateJson['id']); - $metadata = $this->em->getClassMetaData(get_class($eventDate)); - $metadata->setIdGenerator(new AssignedGenerator()); + $eventDate->setMidataId($dateJson['id']); + $eventDate->setSyncGroup($syncGroup); } $eventDate->setEvent($event); diff --git a/src/Service/PbsApi/Fetcher/GroupFetcher.php b/src/Service/PbsApi/Fetcher/GroupFetcher.php index 2271bba..fa035b5 100644 --- a/src/Service/PbsApi/Fetcher/GroupFetcher.php +++ b/src/Service/PbsApi/Fetcher/GroupFetcher.php @@ -9,7 +9,6 @@ use App\Service\PbsApiService; use DateTimeImmutable; use Doctrine\ORM\EntityManagerInterface; -use Doctrine\ORM\Id\AssignedGenerator; class GroupFetcher extends AbstractFetcher { @@ -28,30 +27,29 @@ public function __construct(EntityManagerInterface $em, PbsApiService $pbsApiSer $this->groupRepository = $this->em->getRepository(Group::class); } - public function fetchAndPersistGroup(string $id, string $accessToken) + public function fetchAndPersistGroup(string $id, string $accessToken): Group { - $this->em->persist($this->fetchGroup($id, $accessToken)); + $syncGroup = $this->fetchGroup($id, $accessToken); + $this->em->persist($syncGroup); $this->em->flush(); + return $syncGroup; } - protected function fetchGroup(string $id, string $accessToken): Group + protected function fetchGroup(string $groupId, string $accessToken, ?Group $syncGroup = null): Group { - $groupData = $this->pbsApiService->getApiData('/groups/'.$id, $accessToken); - return $this->mapJsonToGroup($groupData, $accessToken); + $groupData = $this->pbsApiService->getApiData('/groups/'.$groupId, $accessToken); + return $this->mapJsonToGroup($groupData, $accessToken, $syncGroup); } - private function mapJsonToGroup(array $json, string $accessToken): Group + private function mapJsonToGroup(array $json, string $accessToken, ?Group $syncGroup = null): Group { $groupJson = $json['groups'][0] ?? []; $linked = $json['linked'] ?? []; - $group = $this->groupRepository->findOneBy(['id' => $groupJson['id']]); - if (!$group) { - $group = new Group(); - $group->setId($groupJson['id']); - $metadata = $this->em->getClassMetaData(Group::class); - $metadata->setIdGenerator(new AssignedGenerator()); - } + $group = null; + $group = new Group(); + $group->setMidataId($groupJson['id']); + $group->setSyncGroup($syncGroup); $group->setName($groupJson['name'] ?? null); $cantonId = $groupJson['links']['hierarchies'][1] ?? null; @@ -66,7 +64,7 @@ private function mapJsonToGroup(array $json, string $accessToken): Group } /** @var GroupType $gt */ - $gt = $this->groupTypeRepository->findOneBy(['groupType' => $groupJson['type']]); + $gt = $this->groupTypeRepository->findOneBy(['groupType' => $groupJson['group_type_class']]); $group->setGroupType($gt); if ($groupJson['links']['parent'] ?? false) { @@ -78,23 +76,22 @@ private function mapJsonToGroup(array $json, string $accessToken): Group } foreach ($groupJson['links']['children'] ?? [] as $child) { - $group->addChild($this->fetchGroup($child, $accessToken)); + $group->addChild($this->fetchGroup($child, $accessToken, $syncGroup ?? $group)); } return $group; } - protected function fetch(string $groupId, string $accessToken): array { + protected function fetch(Group $syncGroup, string $accessToken): array { // Not implemented because not needed return []; } public function clean(string $groupId) { - $this->groupRepository - ->createQueryBuilder('g') + $this->em->createQueryBuilder() ->delete(Group::class, 'g') - ->where('g.id = :id') - ->setParameter('id', $groupId) + ->where('g.midataId = :group_id') + ->setParameter('group_id', $groupId) ->getQuery() ->execute(); $this->em->flush(); diff --git a/src/Service/PbsApi/Fetcher/PeopleFetcher.php b/src/Service/PbsApi/Fetcher/PeopleFetcher.php index cb54780..6404573 100644 --- a/src/Service/PbsApi/Fetcher/PeopleFetcher.php +++ b/src/Service/PbsApi/Fetcher/PeopleFetcher.php @@ -9,7 +9,6 @@ use App\Service\PbsApiService; use DateTimeImmutable; use Doctrine\ORM\EntityManagerInterface; -use Doctrine\ORM\Id\AssignedGenerator; class PeopleFetcher extends AbstractFetcher { @@ -33,25 +32,25 @@ public function __construct(EntityManagerInterface $em, PbsApiService $pbsApiSer $this->roleMapper = $roleMapper; } - protected function fetch(string $groupId, string $accessToken): array + protected function fetch(Group $syncGroup, string $accessToken): array { + $groupId = $syncGroup->getMidataId(); $peopleData = $this->pbsApiService->getApiData('/groups/'.$groupId.'/people?filters[role][kind]=with_deleted&range=layer', $accessToken); - return $this->mapJsonToPeople($peopleData); + return $this->mapJsonToPeople($peopleData, $syncGroup); } - private function mapJsonToPeople(array $json): array + private function mapJsonToPeople(array $json, Group $syncGroup): array { $peopleJson = $json['people'] ?? []; $linked = $json['linked'] ?? []; $people = []; foreach ($peopleJson as $personJson) { - $person = $this->personRepository->findOneBy(['id' => $personJson['id']]); + $person = $this->personRepository->findOneBy(['midataId' => $personJson['id'], 'syncGroup' => $syncGroup]); if (!$person) { $person = new Person(); - $person->setId($personJson['id']); - $metadata = $this->em->getClassMetaData(get_class($person)); - $metadata->setIdGenerator(new AssignedGenerator()); + $person->setMidataId($personJson['id']); + $person->setSyncGroup($syncGroup); } $person->setNickname($personJson['nickname']); $person->setGender($personJson['gender'] ?? null); @@ -81,7 +80,7 @@ private function mapJsonToPeople(array $json): array } foreach ($personJson['links']['roles'] ?? [] as $roleId) { - $person->addPersonRole($this->roleMapper->mapFromJson($this->getLinked($linked, 'roles', $roleId), $person)); + $person->addPersonRole($this->roleMapper->mapFromJson($this->getLinked($linked, 'roles', $roleId), $person, $syncGroup)); } $people[] = $person; @@ -91,12 +90,10 @@ private function mapJsonToPeople(array $json): array } public function clean(string $groupId) { - $this->personRepository - ->createQueryBuilder('p') + $this->em->createQueryBuilder() ->delete(Person::class, 'p') - // TODO add layer_id during import - //->where('p.layer_id = :layer_id') - //->setParameter('layer_id', $groupId) + ->where('p.syncGroup = :sync_group') + ->setParameter('sync_group', $groupId) ->getQuery() ->execute(); $this->em->flush(); diff --git a/src/Service/PbsApi/Fetcher/PersonEventsFetcher.php b/src/Service/PbsApi/Fetcher/PersonEventsFetcher.php index e238a57..5ddcdf1 100644 --- a/src/Service/PbsApi/Fetcher/PersonEventsFetcher.php +++ b/src/Service/PbsApi/Fetcher/PersonEventsFetcher.php @@ -3,13 +3,13 @@ namespace App\Service\PbsApi\Fetcher; use App\Entity\Event; +use App\Entity\Group; use App\Entity\Person; use App\Entity\PersonEvent; use App\Repository\PersonEventRepository; use App\Repository\PersonRepository; use App\Service\PbsApiService; use Doctrine\ORM\EntityManagerInterface; -use Doctrine\ORM\Id\AssignedGenerator; class PersonEventsFetcher extends AbstractFetcher { @@ -33,32 +33,32 @@ public function __construct(EntityManagerInterface $em, PbsApiService $pbsApiSer $this->personRepository = $personRepository; } - public function fetch(string $groupId, string $accessToken): array + public function fetch(Group $syncGroup, string $accessToken): array { - $personEvents = $this->pbsApiService->getApiData('/groups/'.$groupId.'/events/'.$this->event->getId().'/participations', $accessToken); - return $this->mapJsonToPersonEvents($personEvents, $this->event); + $groupId = $syncGroup->getMidataId(); + $personEvents = $this->pbsApiService->getApiData('/groups/'.$groupId.'/events/'.$this->event->getMidataId().'/participations', $accessToken); + return $this->mapJsonToPersonEvents($personEvents, $this->event, $syncGroup); } - private function mapJsonToPersonEvents(array $json, Event $event): array + private function mapJsonToPersonEvents(array $json, Event $event, Group $syncGroup): array { $personEventsJson = $json['event_participations'] ?? []; $linked = $json['linked'] ?? []; $personEvents = []; foreach ($personEventsJson as $personEventJson) { - $personEvent = $this->personEventRepository->findOneBy(['id' => $personEventJson['id']]); + $personEvent = $this->personEventRepository->findOneBy(['midataId' => $personEventJson['id'], 'syncGroup' => $syncGroup]); if (!$personEvent) { $personEvent = new PersonEvent(); - $personEvent->setId($personEventJson['id']); - $metadata = $this->em->getClassMetaData(get_class($personEvent)); - $metadata->setIdGenerator(new AssignedGenerator()); + $personEvent->setMidataId($personEventJson['id']); + $personEvent->setSyncGroup($syncGroup); } $personEvent->setQualified($personEventJson['qualified'] ?? null); $personEvent->setEvent($event); /** @var Person $person */ - $person = $this->personRepository->findOneBy(['id' => $personEventJson['links']['person']]); + $person = $this->personRepository->findOneBy(['midataId' => $personEventJson['links']['person'], 'syncGroup' => $syncGroup]); $personEvent->setPerson($person); $personEvents[] = $personEvent; @@ -71,9 +71,8 @@ public function clean(string $groupId) { $this->personEventRepository ->createQueryBuilder('pe') ->delete(PersonEvent::class, 'pe') - // TODO add layer_id during import - //->where('pe.layer_id = :layer_id') - //->setParameter('layer_id', $groupId) + ->where('pe.syncGroup = :sync_group_id') + ->setParameter('sync_group_id', $groupId) ->getQuery() ->execute(); $this->em->flush(); diff --git a/src/Service/PbsApi/Fetcher/PersonRoleMapper.php b/src/Service/PbsApi/Fetcher/PersonRoleMapper.php index 7d1f381..2f08c80 100644 --- a/src/Service/PbsApi/Fetcher/PersonRoleMapper.php +++ b/src/Service/PbsApi/Fetcher/PersonRoleMapper.php @@ -11,7 +11,6 @@ use App\Repository\RoleRepository; use DateTimeImmutable; use Doctrine\ORM\EntityManagerInterface; -use Doctrine\ORM\Id\AssignedGenerator; class PersonRoleMapper { @@ -44,14 +43,13 @@ public function __construct(EntityManagerInterface $em) { * @return PersonRole|null * @throws \Exception */ - public function mapFromJson(array $roleJson, Person $person): ?PersonRole + public function mapFromJson(array $roleJson, Person $person, Group $syncGroup): ?PersonRole { - $personRole = $this->personRoleRepository->findOneBy(['id' => $roleJson['id']]); + $personRole = $this->personRoleRepository->findOneBy(['midataId' => $roleJson['id'], 'syncGroup' => $syncGroup]); if (!$personRole) { $personRole = new PersonRole(); - $personRole->setId($roleJson['id']); - $metadata = $this->em->getClassMetaData(get_class($personRole)); - $metadata->setIdGenerator(new AssignedGenerator()); + $personRole->setMidataId($roleJson['id']); + $personRole->setSyncGroup($syncGroup); } $personRole->setPerson($person); diff --git a/src/Service/SyncService.php b/src/Service/SyncService.php index 36a9224..130925f 100644 --- a/src/Service/SyncService.php +++ b/src/Service/SyncService.php @@ -60,10 +60,10 @@ public function startSync(int $groupId, $accessToken) $this->peopleFetcher->clean($groupId); $this->groupFetcher->clean($groupId); - $this->groupFetcher->fetchAndPersistGroup($groupId, $accessToken); - $this->peopleFetcher->fetchAndPersist($groupId, $accessToken); - $this->campsFetcher->fetchAndPersist($groupId, $accessToken); - $this->coursesFetcher->fetchAndPersist($groupId, $accessToken); + $syncGroup = $this->groupFetcher->fetchAndPersistGroup($groupId, $accessToken); + $this->peopleFetcher->fetchAndPersist($syncGroup, $accessToken); + $this->campsFetcher->fetchAndPersist($syncGroup, $accessToken); + $this->coursesFetcher->fetchAndPersist($syncGroup, $accessToken); // TODO run aggregations here, but only for the fetched group } From f0aa5a2ac0f449a0cec004d65319e10a1aaa822c Mon Sep 17 00:00:00 2001 From: Carlo Beltrame Date: Tue, 26 Oct 2021 21:18:30 +0200 Subject: [PATCH 11/12] Implement opt-out --- config/routing_api.yml | 4 ++++ src/Controller/Api/SyncController.php | 18 ++++++++++++++++++ src/Service/SyncService.php | 20 ++++++++++++++++---- 3 files changed, 38 insertions(+), 4 deletions(-) diff --git a/config/routing_api.yml b/config/routing_api.yml index f6fbe32..fab57ae 100644 --- a/config/routing_api.yml +++ b/config/routing_api.yml @@ -29,6 +29,10 @@ sync_start: path: /groups/{groupId}/sync methods: POST controller: App\Controller\Api\SyncController::startSync +opt_out: + path: /groups/{groupId}/optOut + methods: POST + controller: App\Controller\Api\SyncController::optOut # Filter data routes filter_data: diff --git a/src/Controller/Api/SyncController.php b/src/Controller/Api/SyncController.php index 6eb8e9f..aeec98e 100644 --- a/src/Controller/Api/SyncController.php +++ b/src/Controller/Api/SyncController.php @@ -74,4 +74,22 @@ public function startSync( $this->syncService->startSync($groupId, $accessToken); return $this->json($message, JsonResponse::HTTP_CREATED); } + + /** + * @param Request $request + * @param int $groupId + * @param SerializerInterface $serializer + * @param ValidatorInterface $validator + * @return JsonResponse + * @ParamConverter(name="group", options={"mapping":{"groupId":"id"}}) + */ + public function optOut( + Request $request, + int $groupId, + SerializerInterface $serializer, + ValidatorInterface $validator + ) { + $this->syncService->clearAllData($groupId); + return $this->json(null); + } } diff --git a/src/Service/SyncService.php b/src/Service/SyncService.php index 130925f..6e0425e 100644 --- a/src/Service/SyncService.php +++ b/src/Service/SyncService.php @@ -55,11 +55,10 @@ public function __construct(EntityManagerInterface $em, PbsApiService $pbsApiSer */ public function startSync(int $groupId, $accessToken) { - $this->coursesFetcher->clean($groupId); - $this->campsFetcher->clean($groupId); - $this->peopleFetcher->clean($groupId); - $this->groupFetcher->clean($groupId); + // First, clear all data we have on this Abteilung + $this->clearAllData($groupId); + // Then, fetch and persist the most up-to-date data $syncGroup = $this->groupFetcher->fetchAndPersistGroup($groupId, $accessToken); $this->peopleFetcher->fetchAndPersist($syncGroup, $accessToken); $this->campsFetcher->fetchAndPersist($syncGroup, $accessToken); @@ -67,4 +66,17 @@ public function startSync(int $groupId, $accessToken) // TODO run aggregations here, but only for the fetched group } + + /** + * @param int $groupId + * @param $accessToken + * @return void + */ + public function clearAllData(int $groupId) + { + $this->coursesFetcher->clean($groupId); + $this->campsFetcher->clean($groupId); + $this->peopleFetcher->clean($groupId); + $this->groupFetcher->clean($groupId); + } } From dba9484f244741fe6c370ea27e21901a93166731 Mon Sep 17 00:00:00 2001 From: Carlo Beltrame Date: Tue, 26 Oct 2021 22:15:39 +0200 Subject: [PATCH 12/12] WIP run aggregations for the single synced main group --- config/services.yaml | 2 +- src/Controller/Api/SyncController.php | 1 + .../Aggregator/DemographicCampAggregator.php | 156 +++++++++--------- .../DemographicEnteredLeftAggregator.php | 91 +++++----- .../Aggregator/DemographicGroupAggregator.php | 43 +++-- .../DepartmentDemographicAggregator.php | 59 ++++--- .../Aggregator/GeoLocationAggregator.php | 54 +++--- .../Aggregator/LeaderOverviewAggregator.php | 96 ++++++----- src/Service/Aggregator/WidgetAggregator.php | 2 +- src/Service/PbsApi/Fetcher/CampsFetcher.php | 3 +- src/Service/PbsApi/Fetcher/CoursesFetcher.php | 3 +- src/Service/SyncService.php | 27 ++- 12 files changed, 275 insertions(+), 262 deletions(-) diff --git a/config/services.yaml b/config/services.yaml index 319b079..6a2273b 100644 --- a/config/services.yaml +++ b/config/services.yaml @@ -96,7 +96,7 @@ services: $pbsCallbackUrl: '%env(PBS_CALLBACK_URL)%' $specialAccessEmails: '%env(SPECIAL_ACCESS)%' - App\Service\SyncService: + App\Service\SyncService: ~ App\Service\Aggregator\DemographicGroupAggregator: tags: diff --git a/src/Controller/Api/SyncController.php b/src/Controller/Api/SyncController.php index aeec98e..22fc655 100644 --- a/src/Controller/Api/SyncController.php +++ b/src/Controller/Api/SyncController.php @@ -90,6 +90,7 @@ public function optOut( ValidatorInterface $validator ) { $this->syncService->clearAllData($groupId); + $this->syncService->clearAllAggregatedData($groupId); return $this->json(null); } } diff --git a/src/Service/Aggregator/DemographicCampAggregator.php b/src/Service/Aggregator/DemographicCampAggregator.php index 955ba57..ec210f0 100644 --- a/src/Service/Aggregator/DemographicCampAggregator.php +++ b/src/Service/Aggregator/DemographicCampAggregator.php @@ -93,9 +93,10 @@ public function getName() * @throws DBALException * @throws Exception */ - public function aggregate(DateTime $startDate = null) + public function aggregate(string $groupId, DateTime $startDate = null) { - $mainGroups = $this->groupRepository->findAllParentGroups(); + /** @var Group $mainGroup */ + $mainGroup = $this->groupRepository->find($groupId); $maxDate = new DateTime(); $minDate = $startDate !== null ? $startDate : new DateTime(self::AGGREGATION_START_DATE); @@ -113,95 +114,92 @@ public function aggregate(DateTime $startDate = null) $prevMonth = clone $startPointDate; $prevMonth->modify('first day of last month'); - /** @var Group $mainGroup */ - foreach ($mainGroups as $mainGroup) { - $this->deleteLastPeriod($this->widgetDemographicCampRepository, $mainGroup->getId()); + $this->deleteLastPeriod($this->widgetDemographicCampRepository, $mainGroup->getId()); - $existingData = $this->getAllDataPointDates( - $this->widgetDemographicCampRepository, - $mainGroup->getId() - ); - if ($this->isDataExistsForDate($startPointDate->format('Y-m-d 00:00:00'), $existingData)) { - continue; - } + $existingData = $this->getAllDataPointDates( + $this->widgetDemographicCampRepository, + $mainGroup->getId() + ); + if ($this->isDataExistsForDate($startPointDate->format('Y-m-d 00:00:00'), $existingData)) { + continue; + } - $subGroupIds = $this->groupRepository->findAllRelevantSubGroupIdsByParentGroupId($mainGroup->getId()); - $allGroupIds = array_merge($subGroupIds, [$mainGroup->getId()]); - $eventDates = $this->eventDateRepository->getAllForPeriodAndSubgroups( - $prevMonth->format('Y-m-d 00:00:00'), - $startPointDate->format( - 'Y-m-d 00:00:00' - ), - $allGroupIds - ); + $subGroupIds = $this->groupRepository->findAllRelevantSubGroupIdsByParentGroupId($mainGroup->getId()); + $allGroupIds = array_merge($subGroupIds, [$mainGroup->getId()]); + $eventDates = $this->eventDateRepository->getAllForPeriodAndSubgroups( + $prevMonth->format('Y-m-d 00:00:00'), + $startPointDate->format( + 'Y-m-d 00:00:00' + ), + $allGroupIds + ); - if (!$eventDates) { + if (!$eventDates) { + continue; + } + + /** @var EventDate $eventDate */ + foreach ($eventDates as $eventDate) { + if (!$eventDate->getEvent() instanceof Camp) { + continue; + } + /** @var Camp $camp */ + $camp = $eventDate->getEvent(); + if ($camp->getState() && $camp->getState() === 'canceled') { continue; } - /** @var EventDate $eventDate */ - foreach ($eventDates as $eventDate) { - if (!$eventDate->getEvent() instanceof Camp) { - continue; - } - /** @var Camp $camp */ - $camp = $eventDate->getEvent(); - if ($camp->getState() && $camp->getState() === 'canceled') { - continue; - } - - $widgetDemographicCamp = $this->widgetDemographicCampRepository->findOneBy([ - 'dataPointDate' => new DateTimeImmutable($startPointDate->format('Y-m-d')), - 'campName' => $eventDate->getEvent()->getName(), - 'startDate' => new DateTimeImmutable($eventDate->getStartAt()->format('Y-m-d')) - ]); - - if (!$widgetDemographicCamp) { - $widgetDemographicCamp = new WidgetDemographicCamp(); - $widgetDemographicCamp->setStartDate($eventDate->getStartAt()); - $widgetDemographicCamp->setCreatedAt(new DateTimeImmutable()); - $widgetDemographicCamp->setDataPointDate( - new DateTimeImmutable($startPointDate->format('Y-m-d')) - ); - $widgetDemographicCamp->setCampName($eventDate->getEvent()->getName()); - $this->em->persist($widgetDemographicCamp); - $this->em->flush(); - } + $widgetDemographicCamp = $this->widgetDemographicCampRepository->findOneBy([ + 'dataPointDate' => new DateTimeImmutable($startPointDate->format('Y-m-d')), + 'campName' => $eventDate->getEvent()->getName(), + 'startDate' => new DateTimeImmutable($eventDate->getStartAt()->format('Y-m-d')) + ]); - $eventStartDate = $eventDate->getStartAt(); - $memberParticipantIds = $this->personRoleRepository->getMemberParticipants( - $allGroupIds, - $eventDate->getEvent()->getId(), - $eventStartDate->format('Y-m-d') + if (!$widgetDemographicCamp) { + $widgetDemographicCamp = new WidgetDemographicCamp(); + $widgetDemographicCamp->setStartDate($eventDate->getStartAt()); + $widgetDemographicCamp->setCreatedAt(new DateTimeImmutable()); + $widgetDemographicCamp->setDataPointDate( + new DateTimeImmutable($startPointDate->format('Y-m-d')) ); - $leaderParticipantIds = $this->personRoleRepository->getLeaderParticipants( - $allGroupIds, - $eventDate->getEvent()->getId(), - $eventStartDate->format('Y-m-d') - ); - $memberData = $this->processPersonIds($memberParticipantIds, $eventStartDate); - $leaderData = $this->processPersonIds($leaderParticipantIds, $eventStartDate); + $widgetDemographicCamp->setCampName($eventDate->getEvent()->getName()); + $this->em->persist($widgetDemographicCamp); + $this->em->flush(); + } - foreach (WidgetAggregator::$typePriority as $groupType) { - $membersCounts = array_key_exists($groupType, $memberData) ? $memberData[$groupType] : null; - $leadersCounts = array_key_exists($groupType, $leaderData) ? $leaderData[$groupType] : null; + $eventStartDate = $eventDate->getStartAt(); + $memberParticipantIds = $this->personRoleRepository->getMemberParticipants( + $allGroupIds, + $eventDate->getEvent()->getId(), + $eventStartDate->format('Y-m-d') + ); + $leaderParticipantIds = $this->personRoleRepository->getLeaderParticipants( + $allGroupIds, + $eventDate->getEvent()->getId(), + $eventStartDate->format('Y-m-d') + ); + $memberData = $this->processPersonIds($memberParticipantIds, $eventStartDate); + $leaderData = $this->processPersonIds($leaderParticipantIds, $eventStartDate); - $this->demographicCampGroupRepository->deleteAllByCampGroupAndGroupType($widgetDemographicCamp->getId(), $mainGroup->getId(), $groupType); + foreach (WidgetAggregator::$typePriority as $groupType) { + $membersCounts = array_key_exists($groupType, $memberData) ? $memberData[$groupType] : null; + $leadersCounts = array_key_exists($groupType, $leaderData) ? $leaderData[$groupType] : null; - $demographicCampGroup = new DemographicCampGroup(); - $demographicCampGroup->setMCount($membersCounts ? $membersCounts['m'] : 0); - $demographicCampGroup->setFCount($membersCounts ? $membersCounts['w'] : 0); - $demographicCampGroup->setUCount($membersCounts ? $membersCounts['u'] : 0); - $demographicCampGroup->setMCountLeader($leadersCounts ? $leadersCounts['m'] : 0); - $demographicCampGroup->setFCountLeader($leadersCounts ? $leadersCounts['w'] : 0); - $demographicCampGroup->setUCountLeader($leadersCounts ? $leadersCounts['u'] : 0); - $demographicCampGroup->setDemographicCamp($widgetDemographicCamp); - $demographicCampGroup->setGroupType($groupType); - $demographicCampGroup->setGroup($mainGroup); - $this->em->persist($demographicCampGroup); - } - $this->em->persist($widgetDemographicCamp); + $this->demographicCampGroupRepository->deleteAllByCampGroupAndGroupType($widgetDemographicCamp->getId(), $mainGroup->getId(), $groupType); + + $demographicCampGroup = new DemographicCampGroup(); + $demographicCampGroup->setMCount($membersCounts ? $membersCounts['m'] : 0); + $demographicCampGroup->setFCount($membersCounts ? $membersCounts['w'] : 0); + $demographicCampGroup->setUCount($membersCounts ? $membersCounts['u'] : 0); + $demographicCampGroup->setMCountLeader($leadersCounts ? $leadersCounts['m'] : 0); + $demographicCampGroup->setFCountLeader($leadersCounts ? $leadersCounts['w'] : 0); + $demographicCampGroup->setUCountLeader($leadersCounts ? $leadersCounts['u'] : 0); + $demographicCampGroup->setDemographicCamp($widgetDemographicCamp); + $demographicCampGroup->setGroupType($groupType); + $demographicCampGroup->setGroup($mainGroup); + $this->em->persist($demographicCampGroup); } + $this->em->persist($widgetDemographicCamp); } } $this->em->flush(); diff --git a/src/Service/Aggregator/DemographicEnteredLeftAggregator.php b/src/Service/Aggregator/DemographicEnteredLeftAggregator.php index 4911a24..d60160c 100644 --- a/src/Service/Aggregator/DemographicEnteredLeftAggregator.php +++ b/src/Service/Aggregator/DemographicEnteredLeftAggregator.php @@ -80,9 +80,10 @@ public function getName() * @param DateTime|null $startDate * @throws DBALException */ - public function aggregate(DateTime $startDate = null) + public function aggregate(string $groupId, DateTime $startDate = null) { - $mainGroups = $this->groupRepository->findAllParentGroups(); + /** @var Group $mainGroup */ + $mainGroup = $this->groupRepository->find($groupId); $minDate = $startDate !== null ? $startDate : new DateTime(self::AGGREGATION_START_DATE); $maxDate = new DateTime(); @@ -99,55 +100,53 @@ public function aggregate(DateTime $startDate = null) $startPointDate = clone $maxDate; } - /** @var Group $mainGroup */ - foreach ($mainGroups as $mainGroup) { - $this->deleteLastPeriod($this->widgetDemographicEnteredLeftRepository, $mainGroup->getId()); + $this->deleteLastPeriod($this->widgetDemographicEnteredLeftRepository, $mainGroup->getId()); - $existingData = $this->getAllDataPointDates( - $this->widgetDemographicEnteredLeftRepository, - $mainGroup->getId() - ); - if ($this->isDataExistsForDate($startPointDate->format('Y-m-d 00:00:00'), $existingData)) { - continue; - } + $existingData = $this->getAllDataPointDates( + $this->widgetDemographicEnteredLeftRepository, + $mainGroup->getId() + ); + if ($this->isDataExistsForDate($startPointDate->format('Y-m-d 00:00:00'), $existingData)) { + continue; + } - $mainGroup = $this->groupRepository->findOneBy(['id' => $mainGroup->getId()]); - $allSubGroupIds = $this->groupRepository->findAllRelevantSubGroupIdsByParentGroupId( - $mainGroup->getId() - ); - $allGroupIds = array_merge($allSubGroupIds, [$mainGroup->getId()]); + $mainGroup = $this->groupRepository->findOneBy(['id' => $mainGroup->getId()]); + $allSubGroupIds = $this->groupRepository->findAllRelevantSubGroupIdsByParentGroupId( + $mainGroup->getId() + ); + $allGroupIds = array_merge($allSubGroupIds, [$mainGroup->getId()]); + + $newPeople = $this->personRoleRepository->findAllNewPeopleByIdsAndGroup( + $allGroupIds, + $prevPointDate->format('Y-m-d'), + $startPointDate->format('Y-m-d') + ); + $processedNewByGroupTypePeopleTypeAndGender = $this->processNewPersonsForGroup( + $newPeople, + $prevPointDate->format('Y-m-d'), + $startPointDate->format('Y-m-d'), + $allGroupIds + ); - $newPeople = $this->personRoleRepository->findAllNewPeopleByIdsAndGroup( - $allGroupIds, - $prevPointDate->format('Y-m-d'), - $startPointDate->format('Y-m-d') - ); - $processedNewByGroupTypePeopleTypeAndGender = $this->processNewPersonsForGroup( - $newPeople, - $prevPointDate->format('Y-m-d'), - $startPointDate->format('Y-m-d'), - $allGroupIds - ); + $leftPeople = $this->personRoleRepository->findAllLeftPeopleByIdsAndGroup( + $allGroupIds, + $prevPointDate->format('Y-m-d'), + $startPointDate->format('Y-m-d') + ); + $processedLeftByGroupTypePeopleTypeAndGender = $this->processLeftPersonsForGroup( + $leftPeople, + $allGroupIds, + $prevPointDate, + $startPointDate + ); - $leftPeople = $this->personRoleRepository->findAllLeftPeopleByIdsAndGroup( - $allGroupIds, - $prevPointDate->format('Y-m-d'), - $startPointDate->format('Y-m-d') - ); - $processedLeftByGroupTypePeopleTypeAndGender = $this->processLeftPersonsForGroup( - $leftPeople, - $allGroupIds, - $prevPointDate, - $startPointDate - ); + $this->processDataForGroup( + $processedNewByGroupTypePeopleTypeAndGender, + $processedLeftByGroupTypePeopleTypeAndGender, + $mainGroup, + $startPointDate->format('Y-m-d') + ); - $this->processDataForGroup( - $processedNewByGroupTypePeopleTypeAndGender, - $processedLeftByGroupTypePeopleTypeAndGender, - $mainGroup, - $startPointDate->format('Y-m-d') - ); - } $this->em->flush(); $this->em->clear(); } diff --git a/src/Service/Aggregator/DemographicGroupAggregator.php b/src/Service/Aggregator/DemographicGroupAggregator.php index ec7cba8..95d7cbc 100644 --- a/src/Service/Aggregator/DemographicGroupAggregator.php +++ b/src/Service/Aggregator/DemographicGroupAggregator.php @@ -63,9 +63,10 @@ public function getName() * @param DateTime|null $startDate * @throws Exception */ - public function aggregate(DateTime $startDate = null) + public function aggregate(string $groupId, DateTime $startDate = null) { - $mainGroups = $this->groupRepository->findAllParentGroups(); + /** @var Group $mainGroup */ + $mainGroup = $this->groupRepository->find($groupId); $minDate = $startDate !== null ? $startDate : new DateTime(self::AGGREGATION_START_DATE); $maxDate = new DateTime(); @@ -79,31 +80,29 @@ public function aggregate(DateTime $startDate = null) $startPointDate = clone $maxDate; } - /** @var Group $mainGroup */ - foreach ($mainGroups as $mainGroup) { - $this->deleteLastPeriod($this->widgetDemographicGroupRepository, $mainGroup->getId()); + $this->deleteLastPeriod($this->widgetDemographicGroupRepository, $mainGroup->getId()); - $existingData = $this->getAllDataPointDates( - $this->widgetDemographicGroupRepository, - $mainGroup->getId() - ); - if ($this->isDataExistsForDate($startPointDate->format('Y-m-d 00:00:00'), $existingData)) { - continue; - } + $existingData = $this->getAllDataPointDates( + $this->widgetDemographicGroupRepository, + $mainGroup->getId() + ); + if ($this->isDataExistsForDate($startPointDate->format('Y-m-d 00:00:00'), $existingData)) { + continue; + } - $mainGroup = $this->groupRepository->findOneBy(['id' => $mainGroup->getId()]); - $subGroupIds = $this->groupRepository->findAllRelevantSubGroupIdsByParentGroupId($mainGroup->getId()); - $ids = array_merge($subGroupIds, [$mainGroup->getId()]); + $mainGroup = $this->groupRepository->findOneBy(['id' => $mainGroup->getId()]); + $subGroupIds = $this->groupRepository->findAllRelevantSubGroupIdsByParentGroupId($mainGroup->getId()); + $ids = array_merge($subGroupIds, [$mainGroup->getId()]); - $results = $this->personRoleRepository->findAllWithRoleCountInGroup( - $startPointDate->format('Y-m-d'), - $ids - ); + $results = $this->personRoleRepository->findAllWithRoleCountInGroup( + $startPointDate->format('Y-m-d'), + $ids + ); - $data = $this->processPersonIdsAndRoles($results, $ids, $startPointDate->format('Y-m-d')); + $data = $this->processPersonIdsAndRoles($results, $ids, $startPointDate->format('Y-m-d')); + + $this->createWidgetsFromData($data, $mainGroup, $startPointDate); - $this->createWidgetsFromData($data, $mainGroup, $startPointDate); - } $this->em->flush(); $this->em->clear(); } diff --git a/src/Service/Aggregator/DepartmentDemographicAggregator.php b/src/Service/Aggregator/DepartmentDemographicAggregator.php index 2dfb3b5..31ae759 100644 --- a/src/Service/Aggregator/DepartmentDemographicAggregator.php +++ b/src/Service/Aggregator/DepartmentDemographicAggregator.php @@ -69,9 +69,10 @@ public function getName() * @param DateTime|null $startDate * @throws Exception */ - public function aggregate(DateTime $startDate = null) + public function aggregate(string $groupId, DateTime $startDate = null) { - $mainGroups = $this->groupRepository->findAllParentGroups(); + /** @var Group $mainGroup */ + $mainGroup = $this->groupRepository->find($groupId); $minDate = $startDate !== null ? $startDate : new DateTime(self::AGGREGATION_START_DATE); $maxDate = new DateTime(); @@ -85,42 +86,38 @@ public function aggregate(DateTime $startDate = null) $startPointDate = clone $maxDate; } - foreach ($mainGroups as $mainGroup) { - $this->deleteLastPeriod($this->widgetDemographicDepartmentRepository, $mainGroup->getId()); + $this->deleteLastPeriod($this->widgetDemographicDepartmentRepository, $mainGroup->getId()); - $existingData = $this->getAllDataPointDates( - $this->widgetDemographicDepartmentRepository, - $mainGroup->getId() - ); - if ($this->isDataExistsForDate($startPointDate->format('Y-m-d 00:00:00'), $existingData)) { - continue; - } + $existingData = $this->getAllDataPointDates( + $this->widgetDemographicDepartmentRepository, + $mainGroup->getId() + ); + if ($this->isDataExistsForDate($startPointDate->format('Y-m-d 00:00:00'), $existingData)) { + continue; + } - $mainGroup = $this->groupRepository->findOneBy(['id' => $mainGroup->getId()]); - $subGroupIds = $this->groupRepository->findAllRelevantSubGroupIdsByParentGroupId($mainGroup->getId()); - $allIds = array_merge($subGroupIds, [$mainGroup->getId()]); + $mainGroup = $this->groupRepository->findOneBy(['id' => $mainGroup->getId()]); + $subGroupIds = $this->groupRepository->findAllRelevantSubGroupIdsByParentGroupId($mainGroup->getId()); + $allIds = array_merge($subGroupIds, [$mainGroup->getId()]); - $birthYears = $this->personRoleRepository->findBirthYearsForDepartment( + $birthYears = $this->personRoleRepository->findBirthYearsForDepartment( + $startPointDate->format('Y-m-d'), + $allIds + ); + if (!$birthYears) { + continue; + } + + foreach ($birthYears as $year) { + $results = $this->personRoleRepository->findAllByYearWithRoleCountInGroup( $startPointDate->format('Y-m-d'), + $year, $allIds ); - if (!$birthYears) { - continue; - } - - foreach ($birthYears as $year) { - $results = $this->personRoleRepository->findAllByYearWithRoleCountInGroup( - $startPointDate->format('Y-m-d'), - $year, - $allIds - ); - $data = $this->processPersonIdsAndRoles($results, $allIds, $startPointDate->format('Y-m-d')); - $this->createWidgetsFromData($data, $year, $mainGroup, $startPointDate); - } - - $this->em->flush(); - $this->em->clear(); + $data = $this->processPersonIdsAndRoles($results, $allIds, $startPointDate->format('Y-m-d')); + $this->createWidgetsFromData($data, $year, $mainGroup, $startPointDate); } + $this->em->flush(); $this->em->clear(); } diff --git a/src/Service/Aggregator/GeoLocationAggregator.php b/src/Service/Aggregator/GeoLocationAggregator.php index 3906d79..77ec061 100644 --- a/src/Service/Aggregator/GeoLocationAggregator.php +++ b/src/Service/Aggregator/GeoLocationAggregator.php @@ -70,9 +70,10 @@ public function getName(): string * @throws \Doctrine\DBAL\DBALException * @throws \Exception */ - public function aggregate(DateTime $startDate = null): void + public function aggregate(string $groupId, DateTime $startDate = null) { - $mainGroups = $this->groupRepository->findAllParentGroups(); + /** @var Group $mainGroup */ + $mainGroup = $this->groupRepository->find($groupId); $minDate = $startDate !== null ? $startDate : new DateTime(self::AGGREGATION_START_DATE); $maxDate = new DateTime(); @@ -86,34 +87,31 @@ public function aggregate(DateTime $startDate = null): void $startPointDate = clone $maxDate; } - /** @var Group $mainGroup */ - foreach ($mainGroups as $mainGroup) { - $this->deleteLastPeriod($this->geoLocationRepository, $mainGroup->getId()); - - $existingData = $this->getAllDataPointDates( - $this->geoLocationRepository, - $mainGroup->getId() - ); - if ($this->isDataExistsForDate($startPointDate->format('Y-m-d 00:00:00'), $existingData)) { - continue; - } - - $mainGroup = $this->groupRepository->findOneBy(['id' => $mainGroup->getId()]); - $subGroupIds = $this->groupRepository->findAllRelevantSubGroupIdsByParentGroupId($mainGroup->getId()); - $groupIds = array_merge($subGroupIds, [$mainGroup->getId()]); - - $personGroups = $this->personRoleRepository->findAllByDate( - $startPointDate->format('Y-m-d'), - $groupIds, - array_merge(parent::$memberRoleTypes, parent::$leadersRoleTypes), - parent::$leadersRoleTypes, - parent::$memberRoleTypes, - parent::$roleTypePriority - ); - - $this->createWidgetsFromData($personGroups, $mainGroup, $startPointDate); + $this->deleteLastPeriod($this->geoLocationRepository, $mainGroup->getId()); + + $existingData = $this->getAllDataPointDates( + $this->geoLocationRepository, + $mainGroup->getId() + ); + if ($this->isDataExistsForDate($startPointDate->format('Y-m-d 00:00:00'), $existingData)) { + continue; } + $mainGroup = $this->groupRepository->findOneBy(['id' => $mainGroup->getId()]); + $subGroupIds = $this->groupRepository->findAllRelevantSubGroupIdsByParentGroupId($mainGroup->getId()); + $groupIds = array_merge($subGroupIds, [$mainGroup->getId()]); + + $personGroups = $this->personRoleRepository->findAllByDate( + $startPointDate->format('Y-m-d'), + $groupIds, + array_merge(parent::$memberRoleTypes, parent::$leadersRoleTypes), + parent::$leadersRoleTypes, + parent::$memberRoleTypes, + parent::$roleTypePriority + ); + + $this->createWidgetsFromData($personGroups, $mainGroup, $startPointDate); + $this->em->flush(); $this->em->clear(); } diff --git a/src/Service/Aggregator/LeaderOverviewAggregator.php b/src/Service/Aggregator/LeaderOverviewAggregator.php index cbe5fb4..45bb72e 100644 --- a/src/Service/Aggregator/LeaderOverviewAggregator.php +++ b/src/Service/Aggregator/LeaderOverviewAggregator.php @@ -84,9 +84,10 @@ public function getName() * @throws DBALException * @throws Exception */ - public function aggregate(DateTime $startDate = null) + public function aggregate(string $groupId, DateTime $startDate = null) { - $mainGroups = $this->groupRepository->findAllParentGroups(); + /** @var Group $mainGroup */ + $mainGroup = $this->groupRepository->find($groupId); $minDate = $startDate !== null ? $startDate : new DateTime(self::AGGREGATION_START_DATE); $maxDate = new DateTime(); @@ -100,59 +101,54 @@ public function aggregate(DateTime $startDate = null) $startPointDate = clone $maxDate; } - /** @var Group $mainGroup */ - foreach ($mainGroups as $mainGroup) { - $this->deleteLastPeriod($this->widgetLeaderOverviewRepository, $mainGroup->getId()); + $this->deleteLastPeriod($this->widgetLeaderOverviewRepository, $mainGroup->getId()); - $existingData = $this->getAllDataPointDates( - $this->widgetLeaderOverviewRepository, - $mainGroup->getId() - ); - if ($this->isDataExistsForDate($startPointDate->format('Y-m-d 00:00:00'), $existingData)) { - continue; - } + $existingData = $this->getAllDataPointDates( + $this->widgetLeaderOverviewRepository, + $mainGroup->getId() + ); + if ($this->isDataExistsForDate($startPointDate->format('Y-m-d 00:00:00'), $existingData)) { + continue; + } - $mainGroup = $this->groupRepository->findOneBy(['id' => $mainGroup->getId()]); - $subGroups = $this->groupRepository->findAllRelevantSubGroupsByParentGroupId($mainGroup->getId()); - $subGroupsByType = $this->groupSubGroupsByGroupType($subGroups); - foreach ($subGroupsByType as $groupsAndType) { - $groupType = $groupsAndType['group_type']; - $groupIds = $groupsAndType['groups']; - $mCount = $this->personRoleRepository->findMemberCountForPeriodByGenderGroupTypeAndGroupIds( - $startPointDate->format('Y-m-d'), - $groupIds, - 'm' - ); - $fCount = $this->personRoleRepository->findMemberCountForPeriodByGenderGroupTypeAndGroupIds( - $startPointDate->format('Y-m-d'), - $groupIds, - 'w' - ); - $uCount = $this->personRoleRepository->findMemberCountForPeriodByGenderGroupTypeAndGroupIds( - $startPointDate->format('Y-m-d'), - $groupIds - ); - $widget = new WidgetLeaderOverview(); - $widget->setGroup($mainGroup); - $widget->setGroupType($groupType); - $widget->setMCount($mCount[0]); - $widget->setFCount($fCount[0]); - $widget->setUCount($uCount[0]); - $widget->setCreatedAt(new DateTimeImmutable()); - $widget->setDataPointDate(new DateTimeImmutable($startPointDate->format('Y-m-d'))); + $mainGroup = $this->groupRepository->findOneBy(['id' => $mainGroup->getId()]); + $subGroups = $this->groupRepository->findAllRelevantSubGroupsByParentGroupId($mainGroup->getId()); + $subGroupsByType = $this->groupSubGroupsByGroupType($subGroups); + foreach ($subGroupsByType as $groupsAndType) { + $groupType = $groupsAndType['group_type']; + $groupIds = $groupsAndType['groups']; + $mCount = $this->personRoleRepository->findMemberCountForPeriodByGenderGroupTypeAndGroupIds( + $startPointDate->format('Y-m-d'), + $groupIds, + 'm' + ); + $fCount = $this->personRoleRepository->findMemberCountForPeriodByGenderGroupTypeAndGroupIds( + $startPointDate->format('Y-m-d'), + $groupIds, + 'w' + ); + $uCount = $this->personRoleRepository->findMemberCountForPeriodByGenderGroupTypeAndGroupIds( + $startPointDate->format('Y-m-d'), + $groupIds + ); + $widget = new WidgetLeaderOverview(); + $widget->setGroup($mainGroup); + $widget->setGroupType($groupType); + $widget->setMCount($mCount[0]); + $widget->setFCount($fCount[0]); + $widget->setUCount($uCount[0]); + $widget->setCreatedAt(new DateTimeImmutable()); + $widget->setDataPointDate(new DateTimeImmutable($startPointDate->format('Y-m-d'))); - $this->aggregateLeadersData($mainGroup, $startPointDate, $groupType, $groupIds, $widget); - $this->em->persist($widget); - } + $this->aggregateLeadersData($mainGroup, $startPointDate, $groupType, $groupIds, $widget); + $this->em->persist($widget); + } - $allSubGroupIds = []; - foreach ($subGroups as $group) { - $allSubGroupIds[] = $group['id']; - } - $this->aggregateDataForMainGroup($mainGroup, $startPointDate, $allSubGroupIds); - $this->em->flush(); - $this->em->clear(); + $allSubGroupIds = []; + foreach ($subGroups as $group) { + $allSubGroupIds[] = $group['id']; } + $this->aggregateDataForMainGroup($mainGroup, $startPointDate, $allSubGroupIds); $this->em->flush(); $this->em->clear(); } diff --git a/src/Service/Aggregator/WidgetAggregator.php b/src/Service/Aggregator/WidgetAggregator.php index 278e428..76c16e2 100644 --- a/src/Service/Aggregator/WidgetAggregator.php +++ b/src/Service/Aggregator/WidgetAggregator.php @@ -196,7 +196,7 @@ public function __construct(GroupRepository $groupRepository) abstract public function getName(); - abstract public function aggregate(DateTime $startDate = null); + abstract public function aggregate(string $groupId, DateTime $startDate = null); public function groupSubGroupsByGroupType(array $subGroups) { diff --git a/src/Service/PbsApi/Fetcher/CampsFetcher.php b/src/Service/PbsApi/Fetcher/CampsFetcher.php index b4b671f..130b2c0 100644 --- a/src/Service/PbsApi/Fetcher/CampsFetcher.php +++ b/src/Service/PbsApi/Fetcher/CampsFetcher.php @@ -10,6 +10,7 @@ use App\Repository\EventDateRepository; use App\Repository\PersonRepository; use App\Repository\YouthSportTypeRepository; +use App\Service\Aggregator\WidgetAggregator; use App\Service\PbsApiService; use Doctrine\ORM\EntityManagerInterface; @@ -48,7 +49,7 @@ public function __construct(EntityManagerInterface $em, PbsApiService $pbsApiSer protected function fetch(Group $syncGroup, string $accessToken): array { $groupId = $syncGroup->getMidataId(); - $startDate = date('d-m-Y', strtotime('-10 years')); + $startDate = date('d-m-Y', strtotime(WidgetAggregator::AGGREGATION_START_DATE)); $endDate = date('d-m-Y', strtotime('+5 years')); $campData = $this->pbsApiService->getApiData('/groups/'.$groupId.'/events?type=Event::Camp&start_date='.$startDate.'&end_date='.$endDate, $accessToken); return $this->mapJsonToCamps($campData, $syncGroup, $accessToken); diff --git a/src/Service/PbsApi/Fetcher/CoursesFetcher.php b/src/Service/PbsApi/Fetcher/CoursesFetcher.php index 2c79102..c7ac8c1 100644 --- a/src/Service/PbsApi/Fetcher/CoursesFetcher.php +++ b/src/Service/PbsApi/Fetcher/CoursesFetcher.php @@ -10,6 +10,7 @@ use App\Repository\EventDateRepository; use App\Repository\EventTypeRepository; use App\Repository\PersonRepository; +use App\Service\Aggregator\WidgetAggregator; use App\Service\PbsApiService; use Doctrine\ORM\EntityManagerInterface; @@ -48,7 +49,7 @@ public function __construct(EntityManagerInterface $em, PbsApiService $pbsApiSer protected function fetch(Group $syncGroup, string $accessToken): array { $groupId = $syncGroup->getMidataId(); - $startDate = date('d-m-Y', strtotime('-10 years')); + $startDate = date('d-m-Y', strtotime(WidgetAggregator::AGGREGATION_START_DATE)); $endDate = date('d-m-Y', strtotime('+5 years')); $courseData = $this->pbsApiService->getApiData('/groups/'.$groupId.'/events?type=Event::Course&start_date='.$startDate.'&end_date='.$endDate, $accessToken); return $this->mapJsonToCourses($courseData, $syncGroup, $accessToken); diff --git a/src/Service/SyncService.php b/src/Service/SyncService.php index 6e0425e..9a7accd 100644 --- a/src/Service/SyncService.php +++ b/src/Service/SyncService.php @@ -2,6 +2,8 @@ namespace App\Service; +use App\Service\Aggregator\AggregatorRegistry; +use App\Service\Aggregator\WidgetAggregator; use App\Service\PbsApi\Fetcher\CampsFetcher; use App\Service\PbsApi\Fetcher\CoursesFetcher; use App\Service\PbsApi\Fetcher\GroupFetcher; @@ -35,10 +37,15 @@ class SyncService */ private $coursesFetcher; + /** + * @var AggregatorRegistry + */ + protected $aggregatorRegistry; + /** * @param PbsApiService $pbsApiService */ - public function __construct(EntityManagerInterface $em, PbsApiService $pbsApiService, GroupFetcher $groupMapper, PeopleFetcher $peopleFetcher, CampsFetcher $campsFetcher, CoursesFetcher $coursesFetcher) + public function __construct(EntityManagerInterface $em, PbsApiService $pbsApiService, GroupFetcher $groupMapper, PeopleFetcher $peopleFetcher, CampsFetcher $campsFetcher, CoursesFetcher $coursesFetcher, AggregatorRegistry $aggregatorRegistry) { $this->em = $em; $this->pbsApiService = $pbsApiService; @@ -46,6 +53,7 @@ public function __construct(EntityManagerInterface $em, PbsApiService $pbsApiSer $this->peopleFetcher = $peopleFetcher; $this->campsFetcher = $campsFetcher; $this->coursesFetcher = $coursesFetcher; + $this->aggregatorRegistry = $aggregatorRegistry; } /** @@ -64,7 +72,12 @@ public function startSync(int $groupId, $accessToken) $this->campsFetcher->fetchAndPersist($syncGroup, $accessToken); $this->coursesFetcher->fetchAndPersist($syncGroup, $accessToken); - // TODO run aggregations here, but only for the fetched group + // Finally, run the aggregations for this group + // This runs only the necessary delta since the last aggregation, except after a fresh opt-in + /** @var WidgetAggregator $aggregator */ + foreach ($this->aggregatorRegistry->getAggregators() as $aggregator) { + $aggregator->aggregate($syncGroup->getId()); + } } /** @@ -79,4 +92,14 @@ public function clearAllData(int $groupId) $this->peopleFetcher->clean($groupId); $this->groupFetcher->clean($groupId); } + + /** + * @param int $groupId + * @param $accessToken + * @return void + */ + public function clearAllAggregatedData(int $groupId) + { + // TODO implement this + } }