From a5274bef2f3eece1f63c601075320f677368fec3 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 3 Jun 2024 08:29:43 +0000 Subject: [PATCH 1/2] Bump com.google.guava:guava from 33.2.0-jre to 33.2.1-jre Bumps [com.google.guava:guava](https://github.com/google/guava) from 33.2.0-jre to 33.2.1-jre. - [Release notes](https://github.com/google/guava/releases) - [Commits](https://github.com/google/guava/commits) --- updated-dependencies: - dependency-name: com.google.guava:guava dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- build.gradle.kts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle.kts b/build.gradle.kts index 02a601bd..bfa80884 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -10,7 +10,7 @@ val reflectionsVersion = "0.10.2" val guiceVersion = "7.0.0" val liquibaseCoreVersion = "4.28.0" val quartzVersion = "2.5.0-rc1" -val guavaVersion = "33.2.0-jre" +val guavaVersion = "33.2.1-jre" val playwrightVersion = "1.44.0" val jsoupVersion = "1.17.2" val gsonVersion = "2.11.0" From b51436b95074648f415628e35c91515953211237 Mon Sep 17 00:00:00 2001 From: Ziedelth Date: Tue, 4 Jun 2024 17:52:16 +0200 Subject: [PATCH 2/2] Bug fix #503 --- .../fr/shikkanime/jobs/FetchOldEpisodesJob.kt | 81 +++++++++---------- .../fr/shikkanime/services/ImageService.kt | 7 +- .../jobs/FetchOldEpisodesJobTest.kt | 46 ++++++++++- .../shikkanime/services/ImageServiceTest.kt | 8 +- 4 files changed, 98 insertions(+), 44 deletions(-) diff --git a/src/main/kotlin/fr/shikkanime/jobs/FetchOldEpisodesJob.kt b/src/main/kotlin/fr/shikkanime/jobs/FetchOldEpisodesJob.kt index b96ee7a4..3619579d 100644 --- a/src/main/kotlin/fr/shikkanime/jobs/FetchOldEpisodesJob.kt +++ b/src/main/kotlin/fr/shikkanime/jobs/FetchOldEpisodesJob.kt @@ -1,5 +1,6 @@ package fr.shikkanime.jobs +import com.google.gson.JsonObject import com.google.inject.Inject import fr.shikkanime.caches.CountryCodeIdKeyCache import fr.shikkanime.entities.Anime @@ -11,7 +12,6 @@ import fr.shikkanime.entities.enums.CountryCode import fr.shikkanime.entities.enums.EpisodeType import fr.shikkanime.entities.enums.Platform import fr.shikkanime.exceptions.EpisodeNoSubtitlesOrVoiceException -import fr.shikkanime.exceptions.EpisodeNotAvailableInCountryException import fr.shikkanime.platforms.AbstractPlatform.Episode import fr.shikkanime.platforms.AnimationDigitalNetworkPlatform import fr.shikkanime.services.AnimeService @@ -198,33 +198,31 @@ class FetchOldEpisodesJob : AbstractJob { val crunchyrollEpisodesCache = MapCache>(duration = Duration.ofDays(7)) { - try { - val episodes = mutableListOf() + runBlocking { + try { + val episodes = mutableListOf() + val accessToken = CrunchyrollWrapper.getAnonymousAccessToken() - val accessToken = runBlocking { CrunchyrollWrapper.getAnonymousAccessToken() } - val seasons = runBlocking { CrunchyrollWrapper.getSeasonsBySeriesId(it.countryCode.locale, accessToken, it.id) } - .filter { season -> season.subtitleLocales.contains(it.countryCode.locale) } + CrunchyrollWrapper.getSeasonsBySeriesId(it.countryCode.locale, accessToken, it.id) + .filter { season -> season.subtitleLocales.contains(it.countryCode.locale) } + .forEach { season -> episodes.addAll(CrunchyrollWrapper.getEpisodesBySeasonId(it.countryCode.locale, accessToken, season.id)) } - // Add original episodes - seasons.parallelStream().forEach { season -> - runBlocking { episodes.addAll(CrunchyrollWrapper.getEpisodesBySeasonId(it.countryCode.locale, accessToken, season.id)) } - } + // Need to list all available variants + val variants = mutableSetOf() + episodes.forEach { episode -> variants.addAll(episode.versions ?: listOf(CrunchyrollWrapper.Version(episode.id))) } + // Remove duplicates and already fetched episodes + val missingVariants = variants.distinctBy { variant -> variant.guid } + .filter { variant -> episodes.none { it.id == variant.guid } } - // Need to list all available variants - val variants = mutableSetOf() - episodes.forEach { episode -> variants.addAll(episode.versions ?: listOf(CrunchyrollWrapper.Version(episode.id))) } - // Remove duplicates and already fetched episodes - val missingVariants = variants.distinctBy { variant -> variant.guid } - .filter { variant -> episodes.none { it.id == variant.guid } } + missingVariants.parallelStream().forEach { variant -> + runBlocking { episodes.add(CrunchyrollWrapper.getEpisode(it.countryCode.locale, accessToken, variant.guid)) } + } - missingVariants.parallelStream().forEach { variant -> - runBlocking { episodes.add(CrunchyrollWrapper.getEpisode(it.countryCode.locale, accessToken, variant.guid)) } + return@runBlocking episodes + } catch (e: Exception) { + logger.log(Level.SEVERE, "Error while fetching Crunchyroll episodes", e) + return@runBlocking emptyList() } - - return@MapCache episodes - } catch (e: Exception) { - logger.log(Level.SEVERE, "Error while fetching Crunchyroll episodes", e) - return@MapCache emptyList() } } @@ -232,16 +230,7 @@ class FetchOldEpisodesJob : AbstractJob { val accessToken = CrunchyrollWrapper.getAnonymousAccessToken() val platformEpisodes = mutableListOf() - val series = simulcasts.flatMap { simulcastId -> - CrunchyrollWrapper.getBrowse( - countryCode.locale, - accessToken, - sortBy = CrunchyrollWrapper.SortType.POPULARITY, - type = CrunchyrollWrapper.MediaType.SERIES, - 200, - simulcast = simulcastId - ) - }.distinctBy { it.getAsString("id") } + val series = getSeries(countryCode, accessToken, simulcasts) series.forEach { serie -> val seasonRegex = " Saison (\\d)".toRegex() @@ -290,9 +279,6 @@ class FetchOldEpisodesJob : AbstractJob { val duration = episode.durationMs / 1000 val description = episode.description?.replace('\n', ' ')?.takeIf { it.isNotBlank() } - if (!episode.eligibleRegion.contains(countryCode.name)) - throw EpisodeNotAvailableInCountryException("Episode of $animeName is not available in ${countryCode.name}") - if (!isDubbed && (episode.subtitleLocales.isEmpty() || !episode.subtitleLocales.contains(countryCode.locale))) throw EpisodeNoSubtitlesOrVoiceException("Episode is not available in ${countryCode.name} with subtitles or voice") @@ -319,11 +305,7 @@ class FetchOldEpisodesJob : AbstractJob { ) ) } catch (e: Exception) { - logger.log( - Level.SEVERE, - "Error while converting episode (Episode ID: ${episode.id})", - e - ) + logger.warning("Error while converting episode (Episode ID: ${episode.id}) : ${e.message}") } } } @@ -331,6 +313,23 @@ class FetchOldEpisodesJob : AbstractJob { return platformEpisodes } + suspend fun getSeries( + countryCode: CountryCode, + accessToken: String, + simulcasts: Set, + ): List { + return simulcasts.flatMap { simulcastId -> + CrunchyrollWrapper.getBrowse( + countryCode.locale, + accessToken, + sortBy = CrunchyrollWrapper.SortType.POPULARITY, + type = CrunchyrollWrapper.MediaType.SERIES, + 200, + simulcast = simulcastId + ) + }.distinctBy { it.getAsString("id") } + } + private fun getNumberAndEpisodeType(episode: CrunchyrollWrapper.Episode): Pair { var number = episode.number ?: -1 val specialEpisodeRegex = "SP(\\d*)".toRegex() diff --git a/src/main/kotlin/fr/shikkanime/services/ImageService.kt b/src/main/kotlin/fr/shikkanime/services/ImageService.kt index bb410f18..86d1b2ff 100644 --- a/src/main/kotlin/fr/shikkanime/services/ImageService.kt +++ b/src/main/kotlin/fr/shikkanime/services/ImageService.kt @@ -41,7 +41,7 @@ object ImageService { ) : Serializable private val logger = LoggerFactory.getLogger(javaClass) - private val threadPool = Executors.newFixedThreadPool(4) + private var threadPool = Executors.newFixedThreadPool(4) val cache = mutableListOf() private val change = AtomicBoolean(false) private const val CACHE_FILE_NUMBER = 4 @@ -309,6 +309,11 @@ object ImageService { addAll(true) } + fun clearPool() { + threadPool.shutdown() + threadPool = Executors.newFixedThreadPool(4) + } + fun addAll(bypass: Boolean = false) { val animeService = Constant.injector.getInstance(AnimeService::class.java) val episodeMappingService = Constant.injector.getInstance(EpisodeMappingService::class.java) diff --git a/src/test/kotlin/fr/shikkanime/jobs/FetchOldEpisodesJobTest.kt b/src/test/kotlin/fr/shikkanime/jobs/FetchOldEpisodesJobTest.kt index 3d7bbe76..28080e26 100644 --- a/src/test/kotlin/fr/shikkanime/jobs/FetchOldEpisodesJobTest.kt +++ b/src/test/kotlin/fr/shikkanime/jobs/FetchOldEpisodesJobTest.kt @@ -5,9 +5,12 @@ import fr.shikkanime.caches.CountryCodeIdKeyCache import fr.shikkanime.entities.Config import fr.shikkanime.entities.enums.ConfigPropertyKey import fr.shikkanime.entities.enums.CountryCode -import fr.shikkanime.services.ConfigService +import fr.shikkanime.services.* import fr.shikkanime.utils.Constant import fr.shikkanime.utils.MapCache +import fr.shikkanime.utils.ObjectParser.getAsString +import fr.shikkanime.wrappers.CrunchyrollWrapper +import kotlinx.coroutines.runBlocking import org.junit.jupiter.api.AfterEach import org.junit.jupiter.api.Assertions.* import org.junit.jupiter.api.BeforeEach @@ -21,6 +24,15 @@ class FetchOldEpisodesJobTest { @Inject private lateinit var configService: ConfigService + @Inject + private lateinit var animeService: AnimeService + + @Inject + private lateinit var episodeMappingService: EpisodeMappingService + + @Inject + private lateinit var episodeVariantService: EpisodeVariantService + @BeforeEach fun setUp() { Constant.injector.injectMembers(this) @@ -28,8 +40,12 @@ class FetchOldEpisodesJobTest { @AfterEach fun tearDown() { + episodeVariantService.deleteAll() + episodeMappingService.deleteAll() + animeService.deleteAll() configService.deleteAll() MapCache.invalidate(Config::class.java) + ImageService.clearPool() } @Test @@ -63,4 +79,32 @@ class FetchOldEpisodesJobTest { assertEquals("winter-2023", simulcasts.first()) assertEquals("spring-2023", simulcasts.last()) } + + @Test + fun `fix issue #503`() { + val dates = LocalDate.of(2023, 10, 4).datesUntil(LocalDate.of(2023, 10, 14)).toList() + configService.save(Config(propertyKey = ConfigPropertyKey.SIMULCAST_RANGE.key, propertyValue = "20")) + configService.save(Config(propertyKey = ConfigPropertyKey.LAST_FETCH_OLD_EPISODES.key, propertyValue = "2023-10-14")) + configService.save(Config(propertyKey = ConfigPropertyKey.FETCH_OLD_EPISODES_RANGE.key, propertyValue = "14")) + configService.save(Config(propertyKey = ConfigPropertyKey.FETCH_OLD_EPISODES_LIMIT.key, propertyValue = "8")) + MapCache.invalidate(Config::class.java) + val simulcasts = fetchOldEpisodesJob.getSimulcasts(dates) + + assertTrue(simulcasts.isNotEmpty()) + assertEquals(2, simulcasts.size) + assertEquals("summer-2023", simulcasts.first()) + assertEquals("fall-2023", simulcasts.last()) + + val accessToken = runBlocking { CrunchyrollWrapper.getAnonymousAccessToken() } + val series = runBlocking { fetchOldEpisodesJob.getSeries(CountryCode.FR, accessToken, simulcasts) } + assertTrue(series.any { it.getAsString("id") == "GXJHM3NJ5" }) + fetchOldEpisodesJob.run() + + val animes = animeService.findAll() + assertTrue(animes.any { it.name == "CARDFIGHT!! VANGUARD overDress" }) + val anime = animes.first { it.name == "CARDFIGHT!! VANGUARD overDress" } + val episodes = episodeMappingService.findAllByAnime(anime) + // If episodes contains the episode 13 season 3, and episode 12 season 3, it means that the job has worked + assertTrue(episodes.any { it.season == 3 && it.number == 13 }) + } } \ No newline at end of file diff --git a/src/test/kotlin/fr/shikkanime/services/ImageServiceTest.kt b/src/test/kotlin/fr/shikkanime/services/ImageServiceTest.kt index b3d48f40..ccc4226b 100644 --- a/src/test/kotlin/fr/shikkanime/services/ImageServiceTest.kt +++ b/src/test/kotlin/fr/shikkanime/services/ImageServiceTest.kt @@ -19,7 +19,13 @@ class ImageServiceTest { @Test fun add() { val uuid = UUID.randomUUID() - ImageService.add(uuid, ImageService.Type.IMAGE, "https://www.shikkanime.fr/assets/img/dark_logo.png", 128, 128) + ImageService.add( + uuid, + ImageService.Type.IMAGE, + "https://www.crunchyroll.com/imgsrv/display/thumbnail/1920x1080/catalog/crunchyroll/8bfa5ecce45d2d497f88f0b1a0f511df.jpe", + 128, + 128 + ) var i = 0 while (ImageService[uuid, ImageService.Type.IMAGE] == null || ImageService[uuid, ImageService.Type.IMAGE]?.bytes?.isEmpty() == true) {