Fetch group geolocations from hitobito's existing JSON:API

config/services.yaml

@@ -79,7 +79,7 @@ services:
     App\Command\FetchDataCommand:
         arguments:
             $importDirectory: '%import_data_dir%'
-
+            $url: '%env(PBS_DATA_URL)%'
 
     # PBS Services
     App\Service\PbsApiService:

src/Command/FetchDataCommand.php

@@ -32,13 +32,16 @@ class FetchDataCommand extends StatisticsCommand
     /**
      * FetchDataCommand constructor.
      * @param string $importDirectory
+     * @param string $url
      * @param PbsApiService $pbsApiService
      */
     public function __construct(
         string $importDirectory,
+        string $url,
         PbsApiService $pbsApiService
     ) {
         $this->targetDir = $importDirectory;
+        $this->url = $url;
         $this->pbsApiService = $pbsApiService;
         parent::__construct();
     }
@@ -66,6 +69,9 @@ public function execute(InputInterface $input, OutputInterface $output)
             $this->processPaginatedTable($paginatedTable);
         }
 
+        $this->io->section('Fetching geolocations');
+        $this->processGeolocations();
+
         $this->io->success('Finished fetching all table data successfully');
 
         $this->io->title('Fetch Command Stats');
@@ -198,6 +204,72 @@ private function processPaginatedTable(string $tableName): void
         $this->processTableStats($filePath, $tableName, $timeElapsed, $totalItemCount, $page - 1);
     }
 
+    /**
+     * @return void
+     * @throws Exception
+     */
+    private function processGeolocations(): void
+    {
+        $start = microtime(true);
+
+        $filePath = $this->targetDir . '/geolocations.json';
+        $groupIds = $this->getAbteilungenToFetch();
+        $this->io->text('Will fetch ' . count($groupIds) . ' Abteilungen');
+        foreach ($groupIds as $groupId) {
+            $identifier = 'groups/' . $groupId . '.json';
+            $this->io->text('Fetching ' . $identifier);
+            $result = $this->pbsApiService->getApiData($identifier);
+
+            if ($result->getStatusCode() !== 200) {
+                $this->io->error([
+                    'API call for ' . $identifier . ' failed!',
+                    'HTTP status code: ' . $result->getStatusCode()
+                ]);
+                throw new Exception(
+                    'Got http status code ' . $result->getStatusCode() . ' from API. Stopped fetching data.'
+                );
+            }
+
+            $content = $result->getContent();
+            if (isset($content['groups'][0]['links']['geolocations'])) {
+                foreach ($content['groups'][0]['links']['geolocations'] as $geolocationId) {
+                    $geolocations = array_filter($content['linked']['geolocations'], function ($geolocation) use ($geolocationId) {
+                        return $geolocation['id'] === $geolocationId;
+                    });
+
+                    if (count($geolocations) === 0) {
+                        continue;
+                    }
+
+                    $geolocation = array_values($geolocations)[0];
+                    $geolocation['group_id'] = $groupId;
+                    $this->appendJsonToFile($filePath, $geolocation);
+                }
+            }
+        }
+
+        $timeElapsed = microtime(true) - $start;
+        $this->processTableStats($filePath, 'geolocations', $timeElapsed, count($groupIds));
+    }
+
+    /**
+     * Returns the ids of all Abteilung groups that we are allowed to fetch
+     * @return array
+     */
+    private function getAbteilungenToFetch(): array
+    {
+        $fileName = $this->targetDir . '/groups.json';
+        $groups = json_decode(file_get_contents($fileName), true);
+
+        $abteilungen = array_filter($groups, function ($group) {
+            return 'Group::Abteilung' === ($group['type'] ?? '');
+        });
+
+        return array_map(function ($abteilung) {
+            return $abteilung['id'];
+        }, $abteilungen);
+    }
+
     /**
      * @param $filePath
      * @param $data

src/Command/ImportFromJsonCommand.php

@@ -3,6 +3,7 @@
 namespace App\Command;
 
 use App\Entity\EventGroup;
+use App\Entity\GeoLocation;
 use App\Entity\YouthSportType;
 use App\Entity\Camp;
 use App\Entity\CampState;
@@ -108,6 +109,7 @@ protected function execute(InputInterface $input, OutputInterface $output): int
             $this->importParticipations($output);
             $this->importQualifications($output);
             $this->importRoles($output);
+            $this->importGeoLocations($output);
 
             $this->em->getConnection()->commit();
 
@@ -133,6 +135,8 @@ private function cleaningUpEntities(OutputInterface $output)
 
         $connection->executeQuery("DELETE FROM midata_event_group");
 
+        $connection->executeQuery("DELETE FROM midata_geo_location");
+
         $output->writeln("cleaned some entities from the db");
     }
 
@@ -758,6 +762,45 @@ private function importRoles(OutputInterface $output)
         $output->writeln([sprintf('%s rows imported from roles.json', $i)]);
     }
 
+    /**
+     * @param OutputInterface $output
+     * @throws Exception
+     */
+    private function importGeoLocations(OutputInterface $output)
+    {
+        $start = microtime(true);
+        $geoLocations = JsonMachine::fromFile(sprintf('%s/geolocations.json', $this->params->get('import_data_dir')));
+        $i = 0;
+
+        foreach ($geoLocations as $gl) {
+            $longitude = floatval($gl['long']);
+            $latitude = floatval($gl['lat']);
+            // Comparing float to 0. should be safe, even though comparing floats directly normally isn't
+            if (0. === $longitude || 0. === $latitude) {
+                // A parsed value of 0. implies that the string from the JSON was not a float
+                // because hitobito validates that the coordinates lie in Switzerland
+                continue;
+            }
+
+            /** @var Group $abteilung */
+            $abteilung = $this->em->getRepository(Group::class)->find($gl['group_id']);
+            if (!$abteilung) {
+                continue;
+            }
+
+            $geoLocation = new GeoLocation();
+            $geoLocation->setAbteilung($abteilung);
+            $geoLocation->setLongitude($longitude);
+            $geoLocation->setLatitude($latitude);
+            $this->em->persist($geoLocation);
+            $this->em->flush();
+            $i++;
+        }
+        $timeElapsed = microtime(true) - $start;
+        $this->stats[] = ['geolocations.json', $timeElapsed, $i];
+        $output->writeln([sprintf('%s rows imported from geolocations.json', $i)]);
+    }
+
     public function getStats(): CommandStatistics
     {
         $totalItems = 0;

src/Entity/GeoLocation.php

@@ -0,0 +1,77 @@
+<?php
+
+namespace App\Entity;
+
+use App\Repository\GeoLocationRepository;
+use Doctrine\ORM\Mapping as ORM;
+
+/**
+ * @ORM\Table(name="midata_geo_location")
+ * @ORM\Entity(repositoryClass=GeoLocationRepository::class)
+ */
+class GeoLocation
+{
+    /**
+     * @ORM\Id
+     * @ORM\GeneratedValue
+     * @ORM\Column(type="integer")
+     */
+    private $id;
+
+    /**
+     * @ORM\Column(type="float")
+     */
+    private $longitude;
+
+    /**
+     * @ORM\Column(type="float")
+     */
+    private $latitude;
+
+    /**
+     * @ORM\ManyToOne(targetEntity=Group::class, inversedBy="geoLocations")
+     * @ORM\JoinColumn(nullable=false)
+     */
+    private $abteilung;
+
+    public function getId(): ?int
+    {
+        return $this->id;
+    }
+
+    public function getLongitude(): ?float
+    {
+        return $this->longitude;
+    }
+
+    public function setLongitude(float $longitude): self
+    {
+        $this->longitude = $longitude;
+
+        return $this;
+    }
+
+    public function getLatitude(): ?float
+    {
+        return $this->latitude;
+    }
+
+    public function setLatitude(float $latitude): self
+    {
+        $this->latitude = $latitude;
+
+        return $this;
+    }
+
+    public function getAbteilung(): ?Group
+    {
+        return $this->abteilung;
+    }
+
+    public function setAbteilung(?Group $abteilung): self
+    {
+        $this->abteilung = $abteilung;
+
+        return $this;
+    }
+}

src/Entity/Group.php

@@ -4,6 +4,7 @@
 
 use DateTimeImmutable;
 use Doctrine\Common\Collections\ArrayCollection;
+use Doctrine\Common\Collections\Collection;
 use Doctrine\ORM\Mapping as ORM;
 
 /**
@@ -77,9 +78,15 @@ class Group
      */
     private $personRoles;
 
+    /**
+     * @ORM\OneToMany(targetEntity=GeoLocation::class, mappedBy="abteilung", orphanRemoval=true)
+     */
+    private $geoLocations;
+
     public function __construct()
     {
         $this->events = new ArrayCollection();
+        $this->geoLocations = new ArrayCollection();
     }
 
     /**
@@ -214,4 +221,34 @@ public function __toString()
     {
         return (string)$this->id;
     }
+
+    /**
+     * @return Collection|GeoLocation[]
+     */
+    public function getGeoLocations(): Collection
+    {
+        return $this->geoLocations;
+    }
+
+    public function addGeoLocation(GeoLocation $geoLocation): self
+    {
+        if (!$this->geoLocations->contains($geoLocation)) {
+            $this->geoLocations[] = $geoLocation;
+            $geoLocation->setAbteilung($this);
+        }
+
+        return $this;
+    }
+
+    public function removeGeoLocation(GeoLocation $geoLocation): self
+    {
+        if ($this->geoLocations->removeElement($geoLocation)) {
+            // set the owning side to null (unless already changed)
+            if ($geoLocation->getAbteilung() === $this) {
+                $geoLocation->setAbteilung(null);
+            }
+        }
+
+        return $this;
+    }
 }

src/Migrations/Version20210805192246.php

@@ -0,0 +1,36 @@
+<?php
+
+declare(strict_types=1);
+
+namespace DoctrineMigrations;
+
+use Doctrine\DBAL\Schema\Schema;
+use Doctrine\Migrations\AbstractMigration;
+
+/**
+ * Auto-generated Migration: Please modify to your needs!
+ */
+final class Version20210805192246 extends AbstractMigration
+{
+    public function getDescription() : string
+    {
+        return '';
+    }
+
+    public function up(Schema $schema) : void
+    {
+        // this up() migration is auto-generated, please modify it to your needs
+        $this->addSql('CREATE SEQUENCE midata_geo_location_id_seq INCREMENT BY 1 MINVALUE 1 START 1');
+        $this->addSql('CREATE TABLE midata_geo_location (id INT NOT NULL, abteilung_id INT NOT NULL, longitude DOUBLE PRECISION NOT NULL, latitude DOUBLE PRECISION NOT NULL, PRIMARY KEY(id))');
+        $this->addSql('CREATE INDEX IDX_B027FE6AE862333F ON midata_geo_location (abteilung_id)');
+        $this->addSql('ALTER TABLE midata_geo_location ADD CONSTRAINT FK_B027FE6AE862333F FOREIGN KEY (abteilung_id) REFERENCES midata_group (id) NOT DEFERRABLE INITIALLY IMMEDIATE');
+    }
+
+    public function down(Schema $schema) : void
+    {
+        // this down() migration is auto-generated, please modify it to your needs
+        $this->addSql('CREATE SCHEMA public');
+        $this->addSql('DROP SEQUENCE midata_geo_location_id_seq CASCADE');
+        $this->addSql('DROP TABLE midata_geo_location');
+    }
+}

src/Repository/GeoLocationRepository.php

@@ -0,0 +1,21 @@
+<?php
+
+namespace App\Repository;
+
+use App\Entity\GeoLocation;
+use Doctrine\Bundle\DoctrineBundle\Repository\ServiceEntityRepository;
+use Doctrine\Persistence\ManagerRegistry;
+
+/**
+ * @method GeoLocation|null find($id, $lockMode = null, $lockVersion = null)
+ * @method GeoLocation|null findOneBy(array $criteria, array $orderBy = null)
+ * @method GeoLocation[]    findAll()
+ * @method GeoLocation[]    findBy(array $criteria, array $orderBy = null, $limit = null, $offset = null)
+ */
+class GeoLocationRepository extends ServiceEntityRepository
+{
+    public function __construct(ManagerRegistry $registry)
+    {
+        parent::__construct($registry, GeoLocation::class);
+    }
+}

src/Service/PbsApiService.php

@@ -42,4 +42,20 @@ public function getTableData(string $tableName, int $page = null, int $itemsPerP
         $additionalHeaders = ['X-Token' => $this->apiKey];
         return $this->guzzleWrapper->getJson($endpoint, null, $additionalHeaders);
     }
+
+    /**
+     * @param string $tableName
+     * @param int|null $page
+     * @param int|null $itemsPerPage
+     * @return Http\CurlResponse
+     */
+    public function getApiData(string $endpoint, int $page = null, int $itemsPerPage = null)
+    {
+        $endpoint = $this->url . '/' . $endpoint;
+        if ($page !== null && $itemsPerPage !== null) {
+            $endpoint .= '?page=' . $page . '&size=' . $itemsPerPage;
+        }
+        $additionalHeaders = ['X-Token' => $this->apiKey];
+        return $this->guzzleWrapper->getJson($endpoint, null, $additionalHeaders);
+    }
 }