Merge pull request #194 from leepeuker/improve-trakt-import

Improve logging and refactor trakt import
leepeuker · Dec 28, 2022 · 03f3ffd · 03f3ffd
2 parents 125cc9b + 03a09e9
commit 03f3ffd
Show file tree

Hide file tree

Showing 7 changed files with 179 additions and 73 deletions.
diff --git a/README.md b/README.md
@@ -245,7 +245,7 @@ Example cli import (import history and ratings for user with id 1 and overwrite
 - `--history`
   Import trakt watch history (plays)
 - `--overwrite`
-  Use if you want to overwrite the local state with the trakt state (deletes and overwrites local data)
+  Use if you want to overwrite the local data with the data coming from trakt
 - `--ignore-cache`
   Use if you want to force import everything regardless if there was a change since the last import
 

diff --git a/db/migrations/sqlite/20221228172042_AddPrimaryIndexToMovieUserWatchDatesTable.php b/db/migrations/sqlite/20221228172042_AddPrimaryIndexToMovieUserWatchDatesTable.php
@@ -0,0 +1,45 @@
+<?php declare(strict_types=1);
+
+use Phinx\Migration\AbstractMigration;
+
+final class AddPrimaryIndexToMovieUserWatchDatesTable extends AbstractMigration
+{
+    public function down() : void
+    {
+        $this->execute(
+            <<<SQL
+            CREATE TABLE `tmp_movie_user_watch_dates` (
+                `movie_id` INTEGER NOT NULL,
+                `user_id` INTEGER NOT NULL,
+                `watched_at` TEXT DEFAULT NULL,
+                `plays` INTEGER DEFAULT 1,
+                FOREIGN KEY (`movie_id`) REFERENCES movie (`id`) ON DELETE CASCADE,
+                FOREIGN KEY (`user_id`) REFERENCES user (`id`) ON DELETE CASCADE
+            )
+            SQL,
+        );
+        $this->execute('INSERT INTO `tmp_movie_user_watch_dates` (movie_id, user_id, watched_at, plays) SELECT * FROM movie_user_watch_dates');
+        $this->execute('DROP TABLE `movie_user_watch_dates`');
+        $this->execute('ALTER TABLE `tmp_movie_user_watch_dates` RENAME TO `movie_user_watch_dates`');
+    }
+
+    public function up() : void
+    {
+        $this->execute(
+            <<<SQL
+            CREATE TABLE `tmp_movie_user_watch_dates` (
+                `movie_id` INTEGER NOT NULL,
+                `user_id` INTEGER NOT NULL,
+                `watched_at` TEXT DEFAULT NULL,
+                `plays` INTEGER DEFAULT 1,
+                FOREIGN KEY (`movie_id`) REFERENCES movie (`id`) ON DELETE CASCADE,
+                FOREIGN KEY (`user_id`) REFERENCES user (`id`) ON DELETE CASCADE,
+                PRIMARY KEY (`movie_id`, `user_id`, `watched_at`)
+            )
+            SQL,
+        );
+        $this->execute('REPLACE INTO `tmp_movie_user_watch_dates` (movie_id, user_id, watched_at, plays) SELECT * FROM movie_user_watch_dates');
+        $this->execute('DROP TABLE `movie_user_watch_dates`');
+        $this->execute('ALTER TABLE `tmp_movie_user_watch_dates` RENAME TO `movie_user_watch_dates`');
+    }
+}
diff --git a/src/Service/Letterboxd/ImportHistory.php b/src/Service/Letterboxd/ImportHistory.php
@@ -7,7 +7,7 @@
 use Movary\Domain\Movie\MovieEntity;
 use Movary\JobQueue\JobEntity;
 use Movary\Service\Letterboxd\ValueObject\CsvLineHistory;
-use Movary\Service\Trakt\PlaysPerDateDtoList;
+use Movary\Service\Trakt\WatchDateToPlaysMap;
 use Psr\Log\LoggerInterface;
 use RuntimeException;
 
@@ -31,7 +31,7 @@ public function execute(int $userId, string $historyCsvPath) : void
         $watchDatesCsv->setHeaderOffset(0);
         $watchDateRecords = $watchDatesCsv->getRecords();
 
-        /** @var array<int, PlaysPerDateDtoList> $watchDatesToImport */
+        /** @var array<int, WatchDateToPlaysMap> $watchDatesToImport */
         $watchDatesToImport = [];
 
         foreach ($watchDateRecords as $watchDateRecord) {
@@ -45,7 +45,7 @@ public function execute(int $userId, string $historyCsvPath) : void
             }
 
             if (empty($watchDatesToImport[$movie->getId()]) === true) {
-                $watchDatesToImport[$movie->getId()] = PlaysPerDateDtoList::create();
+                $watchDatesToImport[$movie->getId()] = WatchDateToPlaysMap::create();
             }
 
             $watchDatesToImport[$movie->getId()]->incrementPlaysForDate($csvLineHistory->getDate());

diff --git a/src/Service/Trakt/ImportWatchedMovies.php b/src/Service/Trakt/ImportWatchedMovies.php
@@ -9,7 +9,6 @@
 use Movary\Domain\Movie\MovieEntity;
 use Movary\Domain\User\UserApi;
 use Movary\JobQueue\JobEntity;
-use Movary\Service\Tmdb\SyncMovie;
 use Movary\Service\Trakt\Exception\TraktClientIdNotSet;
 use Movary\Service\Trakt\Exception\TraktUserNameNotSet;
 use Movary\ValueObject\Date;
@@ -24,8 +23,8 @@ public function __construct(
         private readonly Api\Trakt\Cache\User\Movie\Watched\Service $traktApiCacheUserMovieWatchedService,
         private readonly LoggerInterface $logger,
         private readonly PlaysPerDateFetcher $playsPerDateFetcher,
-        private readonly SyncMovie $tmdbMovieSync,
         private readonly UserApi $userApi,
+        private readonly MovieImporter $movieImporter,
     ) {
     }
 
@@ -41,14 +40,16 @@ public function execute(int $userId, bool $overwriteExistingData = false, bool $
             throw new TraktUserNameNotSet();
         }
 
-        $watchedMovies = $this->traktApi->fetchUserMoviesWatched($traktClientId, $traktUserName);
+        $traktWatchedMovies = $this->traktApi->fetchUserMoviesWatched($traktClientId, $traktUserName);
 
-        foreach ($watchedMovies as $watchedMovie) {
+        foreach ($this->traktApi->fetchUserMoviesWatched($traktClientId, $traktUserName) as $watchedMovie) {
             $traktId = $watchedMovie->getMovie()->getTraktId();
 
-            $movie = $this->findOrCreateMovieLocally($watchedMovie->getMovie());
+            $movie = $this->movieImporter->importMovie($watchedMovie->getMovie());
 
             if ($ignoreCache === false && $this->isWatchedCacheUpToDate($userId, $watchedMovie) === true) {
+                $this->logger->debug('Trakt history import: Skipped "' . $movie->getTitle() . '" because trakt cache is up to date');
+
                 continue;
             }
 
@@ -57,17 +58,7 @@ public function execute(int $userId, bool $overwriteExistingData = false, bool $
             $this->traktApiCacheUserMovieWatchedService->setOne($userId, $traktId, $watchedMovie->getLastUpdated());
         }
 
-        foreach ($this->traktApi->fetchUniqueCachedTraktIds($userId) as $traktId) {
-            if ($watchedMovies->containsTraktId($traktId) === false) {
-                if ($overwriteExistingData === true) {
-                    $this->movieApi->deleteHistoryForUserByTraktId($userId, $traktId);
-
-                    $this->logger->info('Removed watch dates for movie with trakt id: ' . $traktId);
-                }
-
-                $this->traktApi->removeWatchCacheByTraktId($userId, $traktId);
-            }
-        }
+        $this->removeWatchesNoLongerExistingInTrakt($userId, $traktWatchedMovies, $overwriteExistingData);
     }
 
     public function executeJob(JobEntity $job) : void
@@ -80,33 +71,6 @@ public function executeJob(JobEntity $job) : void
         $this->execute($userId);
     }
 
-    private function findOrCreateMovieLocally(Api\Trakt\ValueObject\TraktMovie $watchedMovie) : MovieEntity
-    {
-        $traktId = $watchedMovie->getTraktId();
-        $tmdbId = $watchedMovie->getTmdbId();
-
-        $movie = $this->movieApi->findByTraktId($traktId);
-
-        if ($movie !== null) {
-            return $movie;
-        }
-
-        $movie = $this->movieApi->findByTmdbId($tmdbId);
-
-        if ($movie !== null) {
-            $this->movieApi->updateTraktId($movie->getId(), $traktId);
-
-            return $this->movieApi->fetchByTraktId($traktId);
-        }
-
-        $movie = $this->tmdbMovieSync->syncMovie($tmdbId);
-        $this->movieApi->updateTraktId($movie->getId(), $traktId);
-
-        $this->logger->info('Added movie: ' . $movie->getTitle());
-
-        return $this->movieApi->fetchByTraktId($traktId);
-    }
-
     private function importMovieHistory(
         string $traktClientId,
         string $traktUserName,
@@ -115,39 +79,42 @@ private function importMovieHistory(
         MovieEntity $movie,
         bool $overwriteExistingData,
     ) : void {
-        $traktHistoryEntries = $this->playsPerDateFetcher->fetchTraktPlaysPerDate($traktClientId, $traktUserName, $traktId);
+        $latestTraktWatchDateToPlaysMap = $this->playsPerDateFetcher->fetchTraktPlaysPerDate($traktClientId, $traktUserName, $traktId);
 
-        foreach ($this->movieApi->fetchHistoryByMovieId($movie->getId(), $userId) as $localHistoryEntry) {
-            $localHistoryEntryDate = Date::createFromString($localHistoryEntry['watched_at']);
+        $skipWatchDates = WatchDateToPlaysMap::create();
 
-            if ($traktHistoryEntries->containsDate($localHistoryEntryDate) === false) {
-                if ($overwriteExistingData === false) {
-                    continue;
-                }
-
-                $this->movieApi->deleteHistoryByIdAndDate($movie->getId(), $userId, $localHistoryEntryDate);
+        foreach ($this->movieApi->fetchHistoryByMovieId($movie->getId(), $userId) as $localHistoryEntry) {
+            $localWatchDate = Date::createFromString($localHistoryEntry['watched_at']);
 
+            if ($latestTraktWatchDateToPlaysMap->containsDate($localWatchDate) === false) {
                 continue;
             }
 
-            $localHistoryEntryPlays = $localHistoryEntry['plays'];
-            $traktHistoryEntryPlays = $traktHistoryEntries->getPlaysForDate($localHistoryEntryDate);
+            $localWatchDatePlays = $localHistoryEntry['plays'];
+            $latestTraktWatchDatePlays = $latestTraktWatchDateToPlaysMap->getPlaysForDate($localWatchDate);
 
-            if ($localHistoryEntryPlays < $traktHistoryEntryPlays || ($localHistoryEntryPlays > $traktHistoryEntryPlays && $overwriteExistingData === true)) {
-                $this->movieApi->replaceHistoryForMovieByDate($movie->getId(), $userId, $localHistoryEntryDate, $traktHistoryEntryPlays);
+            if ($localWatchDatePlays === $latestTraktWatchDatePlays) {
+                $this->logger->debug('Trakt history import: Skipped "' . $movie->getTitle() . '" watch date "' . $localWatchDate . '" plays update, already up to date');
 
-                $this->logger->info('Updated plays for "' . $movie->getTitle() . '" at ' . $localHistoryEntryDate . " from $localHistoryEntryPlays to $traktHistoryEntryPlays");
-            }
+                $skipWatchDates->add($localWatchDate, $localWatchDatePlays);
 
-            $traktHistoryEntries->removeDate($localHistoryEntryDate);
-        }
+                continue;
+            }
 
-        foreach ($traktHistoryEntries as $watchedAt => $plays) {
-            $localHistoryEntryDate = Date::createFromString($watchedAt);
+            if ($overwriteExistingData === false) {
+                $this->logger->debug('Trakt history import: Skipped "' . $movie->getTitle() . '" watch date "' . $localWatchDate . '" plays update, overwrite not set');
 
-            $this->movieApi->replaceHistoryForMovieByDate($movie->getId(), $userId, $localHistoryEntryDate, $plays);
+                $skipWatchDates->add($localWatchDate, $localWatchDatePlays);
+            }
+        }
 
-            $this->logger->info('Added plays for "' . $movie->getTitle() . '" at ' . $watchedAt . " with $plays");
+        foreach ($latestTraktWatchDateToPlaysMap->removeWatchDates($skipWatchDates) as $watchedAt => $plays) {
+            $this->replacePlaysForMovieWatchDate(
+                $movie,
+                $userId,
+                Date::createFromString($watchedAt),
+                $plays,
+            );
         }
     }
 
@@ -157,4 +124,31 @@ private function isWatchedCacheUpToDate(int $userId, Api\Trakt\ValueObject\User\
 
         return $cacheLastUpdated !== null && $watchedMovie->getLastUpdated()->isEqual($cacheLastUpdated) === true;
     }
+
+    private function removeWatchesNoLongerExistingInTrakt(int $userId, Api\Trakt\ValueObject\User\Movie\Watched\DtoList $traktWatchedMovies, bool $overwriteExistingData) : void
+    {
+        if ($overwriteExistingData === false) {
+            $this->logger->debug('Trakt history import: Skipping removing outdated watch dates, no overwrite set');
+
+            exit;
+        }
+
+        foreach ($this->traktApi->fetchUniqueCachedTraktIds($userId) as $cachedTraktId) {
+            if ($traktWatchedMovies->containsTraktId($cachedTraktId) === true) {
+                continue;
+            }
+
+            $this->traktApi->removeWatchCacheByTraktId($userId, $cachedTraktId);
+
+            $this->movieApi->deleteHistoryForUserByTraktId($userId, $cachedTraktId);
+            $this->logger->info('Trakt history import: Removed outdated watch dates for movie with trakt id: ' . $cachedTraktId);
+        }
+    }
+
+    private function replacePlaysForMovieWatchDate(MovieEntity $movie, int $userId, Date $watchedAt, int $plays) : void
+    {
+        $this->movieApi->replaceHistoryForMovieByDate($movie->getId(), $userId, $watchedAt, $plays);
+
+        $this->logger->info('Trakt history import: Imported "' . $movie->getTitle() . "\" watch date $watchedAt  with \"$plays\" plays");
+    }
 }
diff --git a/src/Service/Trakt/MovieImporter.php b/src/Service/Trakt/MovieImporter.php
@@ -0,0 +1,46 @@
+<?php declare(strict_types=1);
+
+namespace Movary\Service\Trakt;
+
+use Movary\Api\Trakt\ValueObject\TraktMovie;
+use Movary\Domain\Movie\MovieApi;
+use Movary\Domain\Movie\MovieEntity;
+use Movary\Service\Tmdb\SyncMovie;
+use Psr\Log\LoggerInterface;
+
+class MovieImporter
+{
+    public function __construct(
+        private readonly MovieApi $movieApi,
+        private readonly LoggerInterface $logger,
+        private readonly SyncMovie $tmdbMovieSync,
+    ) {
+    }
+
+    public function importMovie(TraktMovie $traktMovie) : MovieEntity
+    {
+        $traktId = $traktMovie->getTraktId();
+        $tmdbId = $traktMovie->getTmdbId();
+
+        $movie = $this->movieApi->findByTraktId($traktId);
+
+        if ($movie !== null) {
+            return $movie;
+        }
+
+        $movie = $this->movieApi->findByTmdbId($tmdbId);
+
+        if ($movie !== null) {
+            $this->movieApi->updateTraktId($movie->getId(), $traktId);
+
+            return $this->movieApi->fetchByTraktId($traktId);
+        }
+
+        $movie = $this->tmdbMovieSync->syncMovie($tmdbId);
+        $this->movieApi->updateTraktId($movie->getId(), $traktId);
+
+        $this->logger->info('Trakt history import: Added new movie: ' . $movie->getTitle());
+
+        return $this->movieApi->fetchByTraktId($traktId);
+    }
+}
diff --git a/src/Service/Trakt/PlaysPerDateFetcher.php b/src/Service/Trakt/PlaysPerDateFetcher.php
@@ -12,9 +12,9 @@ public function __construct(private readonly TraktApi $traktApi)
     {
     }
 
-    public function fetchTraktPlaysPerDate(string $traktClientId, string $username, TraktId $traktId) : PlaysPerDateDtoList
+    public function fetchTraktPlaysPerDate(string $traktClientId, string $username, TraktId $traktId) : WatchDateToPlaysMap
     {
-        $playsPerDates = PlaysPerDateDtoList::create();
+        $playsPerDates = WatchDateToPlaysMap::create();
 
         foreach ($this->traktApi->fetchUserMovieHistoryByMovieId($traktClientId, $username, $traktId) as $movieHistoryEntry) {
             $watchDate = Date::createFromDateTime($movieHistoryEntry->getWatchedAt());

diff --git a/src/Service/Trakt/PlaysPerDateDtoList.php → src/Service/Trakt/WatchDateToPlaysMap.php b/src/Service/Trakt/PlaysPerDateDtoList.php → src/Service/Trakt/WatchDateToPlaysMap.php
@@ -10,13 +10,22 @@
  * @method array<string, int> getIterator()
  * @psalm-suppress ImplementedReturnTypeMismatch
  */
-class PlaysPerDateDtoList extends AbstractList
+class WatchDateToPlaysMap extends AbstractList
 {
     public static function create() : self
     {
         return new self();
     }
 
+    public function add(Date $watchDate, int $plays) : void
+    {
+        if ($this->containsDate($watchDate) === true) {
+            throw new \RuntimeException('Cannot add date date, date already exists.');
+        }
+
+        $this->data[(string)$watchDate] = $plays;
+    }
+
     public function containsDate(Date $watchDate) : bool
     {
         return isset($this->data[(string)$watchDate]) === true;
@@ -42,8 +51,20 @@ public function incrementPlaysForDate(Date $watchDate) : void
         $this->data[(string)$watchDate]++;
     }
 
-    public function removeDate(Date $watchDate) : void
+    public function removeWatchDates(WatchDateToPlaysMap $filteredWatchDateToPlayCountMap) : self
     {
-        unset($this->data[(string)$watchDate]);
+        $filteredList = self::create();
+
+        foreach ($this as $watchDate => $plays) {
+            $watchDate = Date::createFromString($watchDate);
+
+            if ($filteredWatchDateToPlayCountMap->containsDate($watchDate) === true) {
+                continue;
+            }
+
+            $filteredList->add($watchDate, $plays);
+        }
+
+        return $filteredList;
     }
 }