Merge pull request #505 from Shikkanime/dev

Dev

build.gradle.kts

@@ -10,7 +10,7 @@ val reflectionsVersion = "0.10.2"
 val guiceVersion = "7.0.0"
 val liquibaseCoreVersion = "4.28.0"
 val quartzVersion = "2.5.0-rc1"
-val guavaVersion = "33.2.0-jre"
+val guavaVersion = "33.2.1-jre"
 val playwrightVersion = "1.44.0"
 val jsoupVersion = "1.17.2"
 val gsonVersion = "2.11.0"

src/main/kotlin/fr/shikkanime/jobs/FetchOldEpisodesJob.kt

@@ -1,5 +1,6 @@
 package fr.shikkanime.jobs
 
+import com.google.gson.JsonObject
 import com.google.inject.Inject
 import fr.shikkanime.caches.CountryCodeIdKeyCache
 import fr.shikkanime.entities.Anime
@@ -11,7 +12,6 @@ import fr.shikkanime.entities.enums.CountryCode
 import fr.shikkanime.entities.enums.EpisodeType
 import fr.shikkanime.entities.enums.Platform
 import fr.shikkanime.exceptions.EpisodeNoSubtitlesOrVoiceException
-import fr.shikkanime.exceptions.EpisodeNotAvailableInCountryException
 import fr.shikkanime.platforms.AbstractPlatform.Episode
 import fr.shikkanime.platforms.AnimationDigitalNetworkPlatform
 import fr.shikkanime.services.AnimeService
@@ -198,50 +198,39 @@ class FetchOldEpisodesJob : AbstractJob {
 
     val crunchyrollEpisodesCache =
         MapCache<CountryCodeIdKeyCache, List<CrunchyrollWrapper.Episode>>(duration = Duration.ofDays(7)) {
-            try {
-                val episodes = mutableListOf<CrunchyrollWrapper.Episode>()
+            runBlocking {
+                try {
+                    val episodes = mutableListOf<CrunchyrollWrapper.Episode>()
+                    val accessToken = CrunchyrollWrapper.getAnonymousAccessToken()
 
-                val accessToken = runBlocking { CrunchyrollWrapper.getAnonymousAccessToken() }
-                val seasons = runBlocking { CrunchyrollWrapper.getSeasonsBySeriesId(it.countryCode.locale, accessToken, it.id) }
-                    .filter { season -> season.subtitleLocales.contains(it.countryCode.locale) }
+                    CrunchyrollWrapper.getSeasonsBySeriesId(it.countryCode.locale, accessToken, it.id)
+                        .filter { season -> season.subtitleLocales.contains(it.countryCode.locale) }
+                        .forEach { season -> episodes.addAll(CrunchyrollWrapper.getEpisodesBySeasonId(it.countryCode.locale, accessToken, season.id)) }
 
-                // Add original episodes
-                seasons.parallelStream().forEach { season ->
-                    runBlocking { episodes.addAll(CrunchyrollWrapper.getEpisodesBySeasonId(it.countryCode.locale, accessToken, season.id)) }
-                }
+                    // Need to list all available variants
+                    val variants = mutableSetOf<CrunchyrollWrapper.Version>()
+                    episodes.forEach { episode -> variants.addAll(episode.versions ?: listOf(CrunchyrollWrapper.Version(episode.id))) }
+                    // Remove duplicates and already fetched episodes
+                    val missingVariants = variants.distinctBy { variant -> variant.guid }
+                        .filter { variant -> episodes.none { it.id == variant.guid } }
 
-                // Need to list all available variants
-                val variants = mutableSetOf<CrunchyrollWrapper.Version>()
-                episodes.forEach { episode -> variants.addAll(episode.versions ?: listOf(CrunchyrollWrapper.Version(episode.id))) }
-                // Remove duplicates and already fetched episodes
-                val missingVariants = variants.distinctBy { variant -> variant.guid }
-                    .filter { variant -> episodes.none { it.id == variant.guid } }
+                    missingVariants.parallelStream().forEach { variant ->
+                        runBlocking { episodes.add(CrunchyrollWrapper.getEpisode(it.countryCode.locale, accessToken, variant.guid)) }
+                    }
 
-                missingVariants.parallelStream().forEach { variant ->
-                    runBlocking { episodes.add(CrunchyrollWrapper.getEpisode(it.countryCode.locale, accessToken, variant.guid)) }
+                    return@runBlocking episodes
+                } catch (e: Exception) {
+                    logger.log(Level.SEVERE, "Error while fetching Crunchyroll episodes", e)
+                    return@runBlocking emptyList()
                 }
-
-                return@MapCache episodes
-            } catch (e: Exception) {
-                logger.log(Level.SEVERE, "Error while fetching Crunchyroll episodes", e)
-                return@MapCache emptyList()
             }
         }
 
     private suspend fun fetchCrunchyroll(countryCode: CountryCode, simulcasts: Set<String>): List<Episode> {
         val accessToken = CrunchyrollWrapper.getAnonymousAccessToken()
         val platformEpisodes = mutableListOf<Episode>()
 
-        val series = simulcasts.flatMap { simulcastId ->
-            CrunchyrollWrapper.getBrowse(
-                countryCode.locale,
-                accessToken,
-                sortBy = CrunchyrollWrapper.SortType.POPULARITY,
-                type = CrunchyrollWrapper.MediaType.SERIES,
-                200,
-                simulcast = simulcastId
-            )
-        }.distinctBy { it.getAsString("id") }
+        val series = getSeries(countryCode, accessToken, simulcasts)
 
         series.forEach { serie ->
             val seasonRegex = " Saison (\\d)".toRegex()
@@ -290,9 +279,6 @@ class FetchOldEpisodesJob : AbstractJob {
                     val duration = episode.durationMs / 1000
                     val description = episode.description?.replace('\n', ' ')?.takeIf { it.isNotBlank() }
 
-                    if (!episode.eligibleRegion.contains(countryCode.name))
-                        throw EpisodeNotAvailableInCountryException("Episode of $animeName is not available in ${countryCode.name}")
-
                     if (!isDubbed && (episode.subtitleLocales.isEmpty() || !episode.subtitleLocales.contains(countryCode.locale)))
                         throw EpisodeNoSubtitlesOrVoiceException("Episode is not available in ${countryCode.name} with subtitles or voice")
 
@@ -319,18 +305,31 @@ class FetchOldEpisodesJob : AbstractJob {
                         )
                     )
                 } catch (e: Exception) {
-                    logger.log(
-                        Level.SEVERE,
-                        "Error while converting episode (Episode ID: ${episode.id})",
-                        e
-                    )
+                    logger.warning("Error while converting episode (Episode ID: ${episode.id}) : ${e.message}")
                 }
             }
         }
 
         return platformEpisodes
     }
 
+    suspend fun getSeries(
+        countryCode: CountryCode,
+        accessToken: String,
+        simulcasts: Set<String>,
+    ): List<JsonObject> {
+        return simulcasts.flatMap { simulcastId ->
+            CrunchyrollWrapper.getBrowse(
+                countryCode.locale,
+                accessToken,
+                sortBy = CrunchyrollWrapper.SortType.POPULARITY,
+                type = CrunchyrollWrapper.MediaType.SERIES,
+                200,
+                simulcast = simulcastId
+            )
+        }.distinctBy { it.getAsString("id") }
+    }
+
     private fun getNumberAndEpisodeType(episode: CrunchyrollWrapper.Episode): Pair<Int, EpisodeType> {
         var number = episode.number ?: -1
         val specialEpisodeRegex = "SP(\\d*)".toRegex()

src/main/kotlin/fr/shikkanime/services/ImageService.kt

@@ -41,7 +41,7 @@ object ImageService {
     ) : Serializable
 
     private val logger = LoggerFactory.getLogger(javaClass)
-    private val threadPool = Executors.newFixedThreadPool(4)
+    private var threadPool = Executors.newFixedThreadPool(4)
     val cache = mutableListOf<Image>()
     private val change = AtomicBoolean(false)
     private const val CACHE_FILE_NUMBER = 4
@@ -309,6 +309,11 @@ object ImageService {
         addAll(true)
     }
 
+    fun clearPool() {
+        threadPool.shutdown()
+        threadPool = Executors.newFixedThreadPool(4)
+    }
+
     fun addAll(bypass: Boolean = false) {
         val animeService = Constant.injector.getInstance(AnimeService::class.java)
         val episodeMappingService = Constant.injector.getInstance(EpisodeMappingService::class.java)

src/test/kotlin/fr/shikkanime/jobs/FetchOldEpisodesJobTest.kt

@@ -5,9 +5,12 @@ import fr.shikkanime.caches.CountryCodeIdKeyCache
 import fr.shikkanime.entities.Config
 import fr.shikkanime.entities.enums.ConfigPropertyKey
 import fr.shikkanime.entities.enums.CountryCode
-import fr.shikkanime.services.ConfigService
+import fr.shikkanime.services.*
 import fr.shikkanime.utils.Constant
 import fr.shikkanime.utils.MapCache
+import fr.shikkanime.utils.ObjectParser.getAsString
+import fr.shikkanime.wrappers.CrunchyrollWrapper
+import kotlinx.coroutines.runBlocking
 import org.junit.jupiter.api.AfterEach
 import org.junit.jupiter.api.Assertions.*
 import org.junit.jupiter.api.BeforeEach
@@ -21,15 +24,28 @@ class FetchOldEpisodesJobTest {
     @Inject
     private lateinit var configService: ConfigService
 
+    @Inject
+    private lateinit var animeService: AnimeService
+
+    @Inject
+    private lateinit var episodeMappingService: EpisodeMappingService
+
+    @Inject
+    private lateinit var episodeVariantService: EpisodeVariantService
+
     @BeforeEach
     fun setUp() {
         Constant.injector.injectMembers(this)
     }
 
     @AfterEach
     fun tearDown() {
+        episodeVariantService.deleteAll()
+        episodeMappingService.deleteAll()
+        animeService.deleteAll()
         configService.deleteAll()
         MapCache.invalidate(Config::class.java)
+        ImageService.clearPool()
     }
 
     @Test
@@ -63,4 +79,32 @@ class FetchOldEpisodesJobTest {
         assertEquals("winter-2023", simulcasts.first())
         assertEquals("spring-2023", simulcasts.last())
     }
+
+    @Test
+    fun `fix issue #503`() {
+        val dates = LocalDate.of(2023, 10, 4).datesUntil(LocalDate.of(2023, 10, 14)).toList()
+        configService.save(Config(propertyKey = ConfigPropertyKey.SIMULCAST_RANGE.key, propertyValue = "20"))
+        configService.save(Config(propertyKey = ConfigPropertyKey.LAST_FETCH_OLD_EPISODES.key, propertyValue = "2023-10-14"))
+        configService.save(Config(propertyKey = ConfigPropertyKey.FETCH_OLD_EPISODES_RANGE.key, propertyValue = "14"))
+        configService.save(Config(propertyKey = ConfigPropertyKey.FETCH_OLD_EPISODES_LIMIT.key, propertyValue = "8"))
+        MapCache.invalidate(Config::class.java)
+        val simulcasts = fetchOldEpisodesJob.getSimulcasts(dates)
+
+        assertTrue(simulcasts.isNotEmpty())
+        assertEquals(2, simulcasts.size)
+        assertEquals("summer-2023", simulcasts.first())
+        assertEquals("fall-2023", simulcasts.last())
+
+        val accessToken = runBlocking { CrunchyrollWrapper.getAnonymousAccessToken() }
+        val series = runBlocking { fetchOldEpisodesJob.getSeries(CountryCode.FR, accessToken, simulcasts) }
+        assertTrue(series.any { it.getAsString("id") == "GXJHM3NJ5" })
+        fetchOldEpisodesJob.run()
+
+        val animes = animeService.findAll()
+        assertTrue(animes.any { it.name == "CARDFIGHT!! VANGUARD overDress" })
+        val anime = animes.first { it.name == "CARDFIGHT!! VANGUARD overDress" }
+        val episodes = episodeMappingService.findAllByAnime(anime)
+        // If episodes contains the episode 13 season 3, and episode 12 season 3, it means that the job has worked
+        assertTrue(episodes.any { it.season == 3 && it.number == 13 })
+    }
 }

src/test/kotlin/fr/shikkanime/services/ImageServiceTest.kt

@@ -19,7 +19,13 @@ class ImageServiceTest {
     @Test
     fun add() {
         val uuid = UUID.randomUUID()
-        ImageService.add(uuid, ImageService.Type.IMAGE, "https://www.shikkanime.fr/assets/img/dark_logo.png", 128, 128)
+        ImageService.add(
+            uuid,
+            ImageService.Type.IMAGE,
+            "https://www.crunchyroll.com/imgsrv/display/thumbnail/1920x1080/catalog/crunchyroll/8bfa5ecce45d2d497f88f0b1a0f511df.jpe",
+            128,
+            128
+        )
         var i = 0
 
         while (ImageService[uuid, ImageService.Type.IMAGE] == null || ImageService[uuid, ImageService.Type.IMAGE]?.bytes?.isEmpty() == true) {