From 84f03950ba2ce1c548a11c133cef6954aa92188e Mon Sep 17 00:00:00 2001 From: Lee Peuker Date: Wed, 28 Dec 2022 18:55:48 +0100 Subject: [PATCH 1/4] Improve logging and refactor trakt import --- ...PrimaryIndexToMovieUserWatchDatesTable.php | 45 +++++++ src/Service/Letterboxd/ImportHistory.php | 6 +- src/Service/Trakt/ImportWatchedMovies.php | 120 +++++++++--------- src/Service/Trakt/MovieImporter.php | 46 +++++++ src/Service/Trakt/PlaysPerDateFetcher.php | 4 +- ...ateDtoList.php => WatchDateToPlaysMap.php} | 27 +++- 6 files changed, 183 insertions(+), 65 deletions(-) create mode 100644 db/migrations/sqlite/20221228172042_AddPrimaryIndexToMovieUserWatchDatesTable.php create mode 100644 src/Service/Trakt/MovieImporter.php rename src/Service/Trakt/{PlaysPerDateDtoList.php => WatchDateToPlaysMap.php} (56%) diff --git a/db/migrations/sqlite/20221228172042_AddPrimaryIndexToMovieUserWatchDatesTable.php b/db/migrations/sqlite/20221228172042_AddPrimaryIndexToMovieUserWatchDatesTable.php new file mode 100644 index 00000000..dfcf2f90 --- /dev/null +++ b/db/migrations/sqlite/20221228172042_AddPrimaryIndexToMovieUserWatchDatesTable.php @@ -0,0 +1,45 @@ +execute( + <<execute('INSERT INTO `tmp_movie_user_watch_dates` (movie_id, user_id, watched_at, plays) SELECT * FROM movie_user_watch_dates'); + $this->execute('DROP TABLE `movie_user_watch_dates`'); + $this->execute('ALTER TABLE `tmp_movie_user_watch_dates` RENAME TO `movie_user_watch_dates`'); + } + + public function up() : void + { + $this->execute( + <<execute('REPLACE INTO `tmp_movie_user_watch_dates` (movie_id, user_id, watched_at, plays) SELECT * FROM movie_user_watch_dates'); + $this->execute('DROP TABLE `movie_user_watch_dates`'); + $this->execute('ALTER TABLE `tmp_movie_user_watch_dates` RENAME TO `movie_user_watch_dates`'); + } +} diff --git a/src/Service/Letterboxd/ImportHistory.php b/src/Service/Letterboxd/ImportHistory.php index be30de12..41b4086d 100644 --- a/src/Service/Letterboxd/ImportHistory.php +++ b/src/Service/Letterboxd/ImportHistory.php @@ -7,7 +7,7 @@ use Movary\Domain\Movie\MovieEntity; use Movary\JobQueue\JobEntity; use Movary\Service\Letterboxd\ValueObject\CsvLineHistory; -use Movary\Service\Trakt\PlaysPerDateDtoList; +use Movary\Service\Trakt\WatchDateToPlaysMap; use Psr\Log\LoggerInterface; use RuntimeException; @@ -31,7 +31,7 @@ public function execute(int $userId, string $historyCsvPath) : void $watchDatesCsv->setHeaderOffset(0); $watchDateRecords = $watchDatesCsv->getRecords(); - /** @var array $watchDatesToImport */ + /** @var array $watchDatesToImport */ $watchDatesToImport = []; foreach ($watchDateRecords as $watchDateRecord) { @@ -45,7 +45,7 @@ public function execute(int $userId, string $historyCsvPath) : void } if (empty($watchDatesToImport[$movie->getId()]) === true) { - $watchDatesToImport[$movie->getId()] = PlaysPerDateDtoList::create(); + $watchDatesToImport[$movie->getId()] = WatchDateToPlaysMap::create(); } $watchDatesToImport[$movie->getId()]->incrementPlaysForDate($csvLineHistory->getDate()); diff --git a/src/Service/Trakt/ImportWatchedMovies.php b/src/Service/Trakt/ImportWatchedMovies.php index 84b4359f..5bf8d4ed 100644 --- a/src/Service/Trakt/ImportWatchedMovies.php +++ b/src/Service/Trakt/ImportWatchedMovies.php @@ -9,7 +9,6 @@ use Movary\Domain\Movie\MovieEntity; use Movary\Domain\User\UserApi; use Movary\JobQueue\JobEntity; -use Movary\Service\Tmdb\SyncMovie; use Movary\Service\Trakt\Exception\TraktClientIdNotSet; use Movary\Service\Trakt\Exception\TraktUserNameNotSet; use Movary\ValueObject\Date; @@ -24,8 +23,8 @@ public function __construct( private readonly Api\Trakt\Cache\User\Movie\Watched\Service $traktApiCacheUserMovieWatchedService, private readonly LoggerInterface $logger, private readonly PlaysPerDateFetcher $playsPerDateFetcher, - private readonly SyncMovie $tmdbMovieSync, private readonly UserApi $userApi, + private readonly MovieImporter $movieImporter, ) { } @@ -41,14 +40,16 @@ public function execute(int $userId, bool $overwriteExistingData = false, bool $ throw new TraktUserNameNotSet(); } - $watchedMovies = $this->traktApi->fetchUserMoviesWatched($traktClientId, $traktUserName); + $traktWatchedMovies = $this->traktApi->fetchUserMoviesWatched($traktClientId, $traktUserName); - foreach ($watchedMovies as $watchedMovie) { + foreach ($this->traktApi->fetchUserMoviesWatched($traktClientId, $traktUserName) as $watchedMovie) { $traktId = $watchedMovie->getMovie()->getTraktId(); - $movie = $this->findOrCreateMovieLocally($watchedMovie->getMovie()); + $movie = $this->movieImporter->importMovie($watchedMovie->getMovie()); if ($ignoreCache === false && $this->isWatchedCacheUpToDate($userId, $watchedMovie) === true) { + $this->logger->debug('Trakt history import: Skipped "' . $movie->getTitle() . '" because trakt cache is up to date'); + continue; } @@ -57,17 +58,7 @@ public function execute(int $userId, bool $overwriteExistingData = false, bool $ $this->traktApiCacheUserMovieWatchedService->setOne($userId, $traktId, $watchedMovie->getLastUpdated()); } - foreach ($this->traktApi->fetchUniqueCachedTraktIds($userId) as $traktId) { - if ($watchedMovies->containsTraktId($traktId) === false) { - if ($overwriteExistingData === true) { - $this->movieApi->deleteHistoryForUserByTraktId($userId, $traktId); - - $this->logger->info('Removed watch dates for movie with trakt id: ' . $traktId); - } - - $this->traktApi->removeWatchCacheByTraktId($userId, $traktId); - } - } + $this->removeWatchesNoLongerExistingInTrakt($userId, $traktWatchedMovies, $overwriteExistingData); } public function executeJob(JobEntity $job) : void @@ -80,31 +71,15 @@ public function executeJob(JobEntity $job) : void $this->execute($userId); } - private function findOrCreateMovieLocally(Api\Trakt\ValueObject\TraktMovie $watchedMovie) : MovieEntity + private function deleteWatchDate(MovieEntity $movie, int $userId, Date $watchDate, bool $overwriteExistingData) : void { - $traktId = $watchedMovie->getTraktId(); - $tmdbId = $watchedMovie->getTmdbId(); - - $movie = $this->movieApi->findByTraktId($traktId); - - if ($movie !== null) { - return $movie; - } - - $movie = $this->movieApi->findByTmdbId($tmdbId); - - if ($movie !== null) { - $this->movieApi->updateTraktId($movie->getId(), $traktId); - - return $this->movieApi->fetchByTraktId($traktId); + if ($overwriteExistingData === false) { + return; } - $movie = $this->tmdbMovieSync->syncMovie($tmdbId); - $this->movieApi->updateTraktId($movie->getId(), $traktId); - - $this->logger->info('Added movie: ' . $movie->getTitle()); + $this->movieApi->deleteHistoryByIdAndDate($movie->getId(), $userId, $watchDate); - return $this->movieApi->fetchByTraktId($traktId); + $this->logger->info(sprintf('Trakt history import: Deleted watch dates not existing in trakt for movie %s at %s', $movie->getTitle(), $watchDate)); } private function importMovieHistory( @@ -115,39 +90,45 @@ private function importMovieHistory( MovieEntity $movie, bool $overwriteExistingData, ) : void { - $traktHistoryEntries = $this->playsPerDateFetcher->fetchTraktPlaysPerDate($traktClientId, $traktUserName, $traktId); + // + $latestTraktWatchDateToPlaysMap = $this->playsPerDateFetcher->fetchTraktPlaysPerDate($traktClientId, $traktUserName, $traktId); - foreach ($this->movieApi->fetchHistoryByMovieId($movie->getId(), $userId) as $localHistoryEntry) { - $localHistoryEntryDate = Date::createFromString($localHistoryEntry['watched_at']); + $skipWatchDates = WatchDateToPlaysMap::create(); - if ($traktHistoryEntries->containsDate($localHistoryEntryDate) === false) { - if ($overwriteExistingData === false) { - continue; - } + foreach ($this->movieApi->fetchHistoryByMovieId($movie->getId(), $userId) as $localHistoryEntry) { + $localWatchDate = Date::createFromString($localHistoryEntry['watched_at']); - $this->movieApi->deleteHistoryByIdAndDate($movie->getId(), $userId, $localHistoryEntryDate); + if ($latestTraktWatchDateToPlaysMap->containsDate($localWatchDate) === false) { + $this->deleteWatchDate($movie, $userId, $localWatchDate, $overwriteExistingData); continue; } - $localHistoryEntryPlays = $localHistoryEntry['plays']; - $traktHistoryEntryPlays = $traktHistoryEntries->getPlaysForDate($localHistoryEntryDate); + $localWatchDatePlays = $localHistoryEntry['plays']; + $latestTraktWatchDatePlays = $latestTraktWatchDateToPlaysMap->getPlaysForDate($localWatchDate); - if ($localHistoryEntryPlays < $traktHistoryEntryPlays || ($localHistoryEntryPlays > $traktHistoryEntryPlays && $overwriteExistingData === true)) { - $this->movieApi->replaceHistoryForMovieByDate($movie->getId(), $userId, $localHistoryEntryDate, $traktHistoryEntryPlays); + if ($localWatchDatePlays === $latestTraktWatchDatePlays) { + $this->logger->debug('Trakt history import: Skipped "' . $movie->getTitle() . '" watch date "' . $localWatchDate . '" plays update, already up to date'); - $this->logger->info('Updated plays for "' . $movie->getTitle() . '" at ' . $localHistoryEntryDate . " from $localHistoryEntryPlays to $traktHistoryEntryPlays"); - } + $skipWatchDates->add($localWatchDate, $localWatchDatePlays); - $traktHistoryEntries->removeDate($localHistoryEntryDate); - } + continue; + } - foreach ($traktHistoryEntries as $watchedAt => $plays) { - $localHistoryEntryDate = Date::createFromString($watchedAt); + if ($overwriteExistingData === false) { + $this->logger->debug('Trakt history import: Skipped "' . $movie->getTitle() . '" watch date "' . $localWatchDate . '" plays update, overwrite not set'); - $this->movieApi->replaceHistoryForMovieByDate($movie->getId(), $userId, $localHistoryEntryDate, $plays); + $skipWatchDates->add($localWatchDate, $localWatchDatePlays); + } + } - $this->logger->info('Added plays for "' . $movie->getTitle() . '" at ' . $watchedAt . " with $plays"); + foreach ($latestTraktWatchDateToPlaysMap->removeWatchDates($skipWatchDates) as $watchedAt => $plays) { + $this->replacePlaysForMovieWatchDate( + $movie, + $userId, + Date::createFromString($watchedAt), + $plays, + ); } } @@ -157,4 +138,29 @@ private function isWatchedCacheUpToDate(int $userId, Api\Trakt\ValueObject\User\ return $cacheLastUpdated !== null && $watchedMovie->getLastUpdated()->isEqual($cacheLastUpdated) === true; } + + private function removeWatchesNoLongerExistingInTrakt(int $userId, Api\Trakt\ValueObject\User\Movie\Watched\DtoList $traktWatchedMovies, bool $overwriteExistingData) : void + { + foreach ($this->traktApi->fetchUniqueCachedTraktIds($userId) as $cachedTraktId) { + if ($traktWatchedMovies->containsTraktId($cachedTraktId) === true) { + continue; + } + + $this->traktApi->removeWatchCacheByTraktId($userId, $cachedTraktId); + + if ($overwriteExistingData === false) { + continue; + } + + $this->movieApi->deleteHistoryForUserByTraktId($userId, $cachedTraktId); + $this->logger->info('Trakt history import: Removed outdated watch dates for movie with trakt id: ' . $cachedTraktId); + } + } + + private function replacePlaysForMovieWatchDate(MovieEntity $movie, int $userId, Date $watchedAt, int $plays) : void + { + $this->movieApi->replaceHistoryForMovieByDate($movie->getId(), $userId, $watchedAt, $plays); + + $this->logger->info('Trakt history import: Imported "' . $movie->getTitle() . "\" watch date $watchedAt with \"$plays\" plays"); + } } diff --git a/src/Service/Trakt/MovieImporter.php b/src/Service/Trakt/MovieImporter.php new file mode 100644 index 00000000..2c80171a --- /dev/null +++ b/src/Service/Trakt/MovieImporter.php @@ -0,0 +1,46 @@ +getTraktId(); + $tmdbId = $traktMovie->getTmdbId(); + + $movie = $this->movieApi->findByTraktId($traktId); + + if ($movie !== null) { + return $movie; + } + + $movie = $this->movieApi->findByTmdbId($tmdbId); + + if ($movie !== null) { + $this->movieApi->updateTraktId($movie->getId(), $traktId); + + return $this->movieApi->fetchByTraktId($traktId); + } + + $movie = $this->tmdbMovieSync->syncMovie($tmdbId); + $this->movieApi->updateTraktId($movie->getId(), $traktId); + + $this->logger->info('Trakt history import: Added new movie: ' . $movie->getTitle()); + + return $this->movieApi->fetchByTraktId($traktId); + } +} diff --git a/src/Service/Trakt/PlaysPerDateFetcher.php b/src/Service/Trakt/PlaysPerDateFetcher.php index 7584e5e3..ad5870a0 100644 --- a/src/Service/Trakt/PlaysPerDateFetcher.php +++ b/src/Service/Trakt/PlaysPerDateFetcher.php @@ -12,9 +12,9 @@ public function __construct(private readonly TraktApi $traktApi) { } - public function fetchTraktPlaysPerDate(string $traktClientId, string $username, TraktId $traktId) : PlaysPerDateDtoList + public function fetchTraktPlaysPerDate(string $traktClientId, string $username, TraktId $traktId) : WatchDateToPlaysMap { - $playsPerDates = PlaysPerDateDtoList::create(); + $playsPerDates = WatchDateToPlaysMap::create(); foreach ($this->traktApi->fetchUserMovieHistoryByMovieId($traktClientId, $username, $traktId) as $movieHistoryEntry) { $watchDate = Date::createFromDateTime($movieHistoryEntry->getWatchedAt()); diff --git a/src/Service/Trakt/PlaysPerDateDtoList.php b/src/Service/Trakt/WatchDateToPlaysMap.php similarity index 56% rename from src/Service/Trakt/PlaysPerDateDtoList.php rename to src/Service/Trakt/WatchDateToPlaysMap.php index 1ac9a114..ecbe02bd 100644 --- a/src/Service/Trakt/PlaysPerDateDtoList.php +++ b/src/Service/Trakt/WatchDateToPlaysMap.php @@ -10,13 +10,22 @@ * @method array getIterator() * @psalm-suppress ImplementedReturnTypeMismatch */ -class PlaysPerDateDtoList extends AbstractList +class WatchDateToPlaysMap extends AbstractList { public static function create() : self { return new self(); } + public function add(Date $watchDate, int $plays) : void + { + if ($this->containsDate($watchDate) === true) { + throw new \RuntimeException('Cannot add date date, date already exists.'); + } + + $this->data[(string)$watchDate] = $plays; + } + public function containsDate(Date $watchDate) : bool { return isset($this->data[(string)$watchDate]) === true; @@ -42,8 +51,20 @@ public function incrementPlaysForDate(Date $watchDate) : void $this->data[(string)$watchDate]++; } - public function removeDate(Date $watchDate) : void + public function removeWatchDates(WatchDateToPlaysMap $filteredWatchDateToPlayCountMap) : self { - unset($this->data[(string)$watchDate]); + $filteredList = self::create(); + + foreach ($this as $watchDate => $plays) { + $watchDate = Date::createFromString($watchDate); + + if ($filteredWatchDateToPlayCountMap->containsDate($watchDate) === true) { + continue; + } + + $filteredList->add($watchDate, $plays); + } + + return $filteredList; } } From ee5856bfc0b8bf034854c64a669724b1411fd0eb Mon Sep 17 00:00:00 2001 From: Lee Peuker Date: Wed, 28 Dec 2022 18:57:00 +0100 Subject: [PATCH 2/4] Cleanup --- src/Service/Trakt/ImportWatchedMovies.php | 1 - 1 file changed, 1 deletion(-) diff --git a/src/Service/Trakt/ImportWatchedMovies.php b/src/Service/Trakt/ImportWatchedMovies.php index 5bf8d4ed..5b104b04 100644 --- a/src/Service/Trakt/ImportWatchedMovies.php +++ b/src/Service/Trakt/ImportWatchedMovies.php @@ -90,7 +90,6 @@ private function importMovieHistory( MovieEntity $movie, bool $overwriteExistingData, ) : void { - // $latestTraktWatchDateToPlaysMap = $this->playsPerDateFetcher->fetchTraktPlaysPerDate($traktClientId, $traktUserName, $traktId); $skipWatchDates = WatchDateToPlaysMap::create(); From afad6d499cac3206c6d79e4663543a664b8503fe Mon Sep 17 00:00:00 2001 From: Lee Peuker Date: Wed, 28 Dec 2022 19:08:00 +0100 Subject: [PATCH 3/4] Remove locacl watch date deletion for missing entries in trakt on overwrite --- README.md | 2 +- src/Service/Trakt/ImportWatchedMovies.php | 13 ------------- 2 files changed, 1 insertion(+), 14 deletions(-) diff --git a/README.md b/README.md index 8c148dbc..6eb09206 100644 --- a/README.md +++ b/README.md @@ -245,7 +245,7 @@ Example cli import (import history and ratings for user with id 1 and overwrite - `--history` Import trakt watch history (plays) - `--overwrite` - Use if you want to overwrite the local state with the trakt state (deletes and overwrites local data) + Use if you want to overwrite the local data with the data coming from trakt - `--ignore-cache` Use if you want to force import everything regardless if there was a change since the last import diff --git a/src/Service/Trakt/ImportWatchedMovies.php b/src/Service/Trakt/ImportWatchedMovies.php index 5b104b04..48207a29 100644 --- a/src/Service/Trakt/ImportWatchedMovies.php +++ b/src/Service/Trakt/ImportWatchedMovies.php @@ -71,17 +71,6 @@ public function executeJob(JobEntity $job) : void $this->execute($userId); } - private function deleteWatchDate(MovieEntity $movie, int $userId, Date $watchDate, bool $overwriteExistingData) : void - { - if ($overwriteExistingData === false) { - return; - } - - $this->movieApi->deleteHistoryByIdAndDate($movie->getId(), $userId, $watchDate); - - $this->logger->info(sprintf('Trakt history import: Deleted watch dates not existing in trakt for movie %s at %s', $movie->getTitle(), $watchDate)); - } - private function importMovieHistory( string $traktClientId, string $traktUserName, @@ -98,8 +87,6 @@ private function importMovieHistory( $localWatchDate = Date::createFromString($localHistoryEntry['watched_at']); if ($latestTraktWatchDateToPlaysMap->containsDate($localWatchDate) === false) { - $this->deleteWatchDate($movie, $userId, $localWatchDate, $overwriteExistingData); - continue; } From 03a09e93f31390b76daf60b0760855a21fd260a4 Mon Sep 17 00:00:00 2001 From: Lee Peuker Date: Wed, 28 Dec 2022 19:12:36 +0100 Subject: [PATCH 4/4] Small performance improvement --- src/Service/Trakt/ImportWatchedMovies.php | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/Service/Trakt/ImportWatchedMovies.php b/src/Service/Trakt/ImportWatchedMovies.php index 48207a29..b5644924 100644 --- a/src/Service/Trakt/ImportWatchedMovies.php +++ b/src/Service/Trakt/ImportWatchedMovies.php @@ -127,6 +127,12 @@ private function isWatchedCacheUpToDate(int $userId, Api\Trakt\ValueObject\User\ private function removeWatchesNoLongerExistingInTrakt(int $userId, Api\Trakt\ValueObject\User\Movie\Watched\DtoList $traktWatchedMovies, bool $overwriteExistingData) : void { + if ($overwriteExistingData === false) { + $this->logger->debug('Trakt history import: Skipping removing outdated watch dates, no overwrite set'); + + exit; + } + foreach ($this->traktApi->fetchUniqueCachedTraktIds($userId) as $cachedTraktId) { if ($traktWatchedMovies->containsTraktId($cachedTraktId) === true) { continue; @@ -134,10 +140,6 @@ private function removeWatchesNoLongerExistingInTrakt(int $userId, Api\Trakt\Val $this->traktApi->removeWatchCacheByTraktId($userId, $cachedTraktId); - if ($overwriteExistingData === false) { - continue; - } - $this->movieApi->deleteHistoryForUserByTraktId($userId, $cachedTraktId); $this->logger->info('Trakt history import: Removed outdated watch dates for movie with trakt id: ' . $cachedTraktId); }