Merge pull request #169 from Shikkanime/dev

Upgrade old episode detection

src/main/kotlin/fr/shikkanime/controllers/site/SiteController.kt

@@ -32,12 +32,15 @@ class SiteController {
     @Path
     @Get
     private fun home(): Response {
+        val findAll = simulcastCacheService.findAll()!!
+        val currentSimulcast = findAll.first()
+
         return Response.template(
             Link.HOME,
             mutableMapOf(
                 "animes" to animeCacheService.findAllBy(
                     CountryCode.FR,
-                    null,
+                    currentSimulcast.uuid,
                     listOf(SortParameter("name", SortParameter.Order.ASC)),
                     1,
                     6
@@ -94,7 +97,8 @@ class SiteController {
         val currentSimulcast = findAll.first { "${it.season.lowercase()}-${it.year}" == slug }
 
         return Response.template(
-            Link.CATALOG,
+            Link.CATALOG.template,
+            currentSimulcast.label,
             mutableMapOf(
                 "simulcasts" to findAll,
                 "currentSimulcast" to currentSimulcast,
@@ -138,7 +142,7 @@ class SiteController {
                         SortParameter("langType", SortParameter.Order.ASC),
                     ),
                     1,
-                    12
+                    24
                 )!!.data
             )
         )

src/main/kotlin/fr/shikkanime/converters/simulcast/SimulcastToSimulcastDtoConverter.kt

@@ -10,6 +10,13 @@ class SimulcastToSimulcastDtoConverter : AbstractConverter<Simulcast, SimulcastD
             uuid = from.uuid,
             season = from.season!!,
             year = from.year!!,
+            label = when (from.season) {
+                "WINTER" -> "Hiver"
+                "SPRING" -> "Printemps"
+                "SUMMER" -> "Été"
+                "AUTUMN" -> "Automne"
+                else -> "Inconnu"
+            } + " ${from.year}"
         )
     }
 }

src/main/kotlin/fr/shikkanime/dtos/SimulcastDto.kt

@@ -7,4 +7,5 @@ data class SimulcastDto(
     val uuid: UUID?,
     val season: String,
     val year: Int,
+    val label: String,
 ) : Serializable

src/main/kotlin/fr/shikkanime/services/EpisodeService.kt

@@ -115,6 +115,14 @@ class EpisodeService : AbstractService<Episode, EpisodeRepository>() {
             entity.anime = animeService.update(anime)
         }
 
+        if (1000 < (entity.title?.length ?: 0)) {
+            entity.title = entity.title!!.substring(0, 1000)
+        }
+
+        if (1000 < (entity.description?.length ?: 0)) {
+            entity.description = entity.description!!.substring(0, 1000)
+        }
+
         val savedEntity = super.save(entity)
         addImage(savedEntity.uuid!!, savedEntity.image!!)
         MapCache.invalidate(Episode::class.java)

src/main/kotlin/fr/shikkanime/utils/HttpRequest.kt

@@ -11,7 +11,7 @@ import org.jsoup.Jsoup
 import org.jsoup.nodes.Document
 import kotlin.system.measureTimeMillis
 
-private const val TIMEOUT = 15_000L
+private const val TIMEOUT = 60_000L
 private val logger = LoggerFactory.getLogger(HttpRequest::class.java)
 
 class HttpRequest : AutoCloseable {

src/main/kotlin/fr/shikkanime/utils/ObjectParser.kt

@@ -48,7 +48,7 @@ object ObjectParser {
     }
 
     fun JsonObject.getAsInt(key: String): Int? {
-        return this[key]?.asInt
+        return if (this[key] != null && !this[key].isJsonNull) this[key]?.asInt else null
     }
 
     fun JsonObject.getAsInt(key: String, default: Int): Int {

src/main/kotlin/fr/shikkanime/wrappers/CrunchyrollWrapper.kt

@@ -86,7 +86,7 @@ object CrunchyrollWrapper {
             ),
         )
 
-        require(response.status.value == 200) { "Failed to get media list" }
+        require(response.status.value == 200) { "Failed to get media list (${response.status.value})" }
 
         return ObjectParser.fromJson(response.bodyAsText()).getAsJsonArray("data")?.map { it.asJsonObject }
             ?: throw Exception("Failed to get media list")
@@ -106,6 +106,34 @@ object CrunchyrollWrapper {
             ?: throw Exception("Failed to get media object")
     }
 
+    suspend fun getSeasons(locale: String, accessToken: String, cms: CMS, seriesId: String): List<JsonObject> {
+        val response = HttpRequest().get(
+            "${BETA_URL}cms/v2${cms.bucket}/seasons?series_id=$seriesId&Policy=${cms.policy}&Signature=${cms.signature}&Key-Pair-Id=${cms.keyPairId}&locale=$locale",
+            headers = mapOf(
+                "Authorization" to "Bearer $accessToken",
+            ),
+        )
+
+        require(response.status.value == 200) { "Failed to get seasons" }
+
+        return ObjectParser.fromJson(response.bodyAsText()).getAsJsonArray("items")?.map { it.asJsonObject }
+            ?: throw Exception("Failed to get seasons")
+    }
+
+    suspend fun getEpisodes(locale: String, accessToken: String, cms: CMS, seasonId: String): List<JsonObject> {
+        val response = HttpRequest().get(
+            "${BETA_URL}cms/v2${cms.bucket}/episodes?season_id=$seasonId&Policy=${cms.policy}&Signature=${cms.signature}&Key-Pair-Id=${cms.keyPairId}&locale=$locale",
+            headers = mapOf(
+                "Authorization" to "Bearer $accessToken",
+            ),
+        )
+
+        require(response.status.value == 200) { "Failed to get episodes" }
+
+        return ObjectParser.fromJson(response.bodyAsText()).getAsJsonArray("items")?.map { it.asJsonObject }
+            ?: throw Exception("Failed to get episodes")
+    }
+
     suspend fun getSimulcasts(locale: String, accessToken: String): List<JsonObject> {
         val response = HttpRequest().get(
             "${BASE_URL}content/v1/season_list?locale=$locale",

src/main/resources/templates/_freemarker_implicit.ftl

@@ -15,7 +15,7 @@
 [#-- @ftlvariable name="config" type="fr.shikkanime.dtos.ConfigDto" --]
 [#-- @ftlvariable name="animes" type="kotlin.collections.AbstractList<fr.shikkanime.dtos.AnimeDto>" --]
 [#-- @ftlvariable name="episodes" type="kotlin.collections.AbstractList<fr.shikkanime.dtos.EpisodeDto>" --]
-[#-- @ftlvariable name="size" type="kotlin.NumbersKt" --]
+[#-- @ftlvariable name="size" type="java.lang.Integer" --]
 [#-- @ftlvariable name="originalSize" type="java.lang.String" --]
 [#-- @ftlvariable name="compressedSize" type="java.lang.String" --]
 [#-- @ftlvariable name="simulcasts" type="kotlin.collections.AbstractList<fr.shikkanime.dtos.SimulcastDto>" --]

src/main/resources/templates/admin/images.ftl

@@ -10,7 +10,7 @@
             <div class="row g-3">
                 <div class="col-md-4">
                     <label class="form-label" for="size">Size</label>
-                    <input id="size" name="size" type="number" class="form-control disabled" value="${size}" disabled>
+                    <input id="size" name="size" type="number" class="form-control disabled" value="${size?c}" disabled>
                 </div>
 
                 <div class="col-md-4">

src/main/resources/templates/site/catalog.ftl

@@ -2,28 +2,15 @@
 <#import "components/episode.ftl" as episodeComponent />
 <#import "components/anime.ftl" as animeComponent />
 
-<#function getPrefixSimulcast(season)>
-    <#switch season>
-        <#case "WINTER">
-            <#return "Hiver">
-        <#case "SPRING">
-            <#return "Printemps">
-        <#case "SUMMER">
-            <#return "Été">
-        <#case "AUTUMN">
-            <#return "Automne">
-    </#switch>
-</#function>
-
 <@navigation.display>
     <div class="mt-3">
         <button class="btn btn-dark dropdown-toggle" data-bs-toggle="dropdown" aria-expanded="false">
-            ${getPrefixSimulcast(currentSimulcast.season)} ${currentSimulcast.year?c}
+            ${currentSimulcast.label}
         </button>
 
-        <ul class="dropdown-menu dropdown-menu-dark">
+        <ul class="dropdown-menu dropdown-menu-dark" style="max-height: 300px; overflow-y: auto;">
             <#list simulcasts as simulcast>
-                <li><a class="dropdown-item" href="/catalog/${simulcast.season?lower_case}-${simulcast.year?c}">${getPrefixSimulcast(simulcast.season)} ${simulcast.year?c}</a></li>
+                <li><a class="dropdown-item" href="/catalog/${simulcast.season?lower_case}-${simulcast.year?c}">${simulcast.label}</a></li>
             </#list>
         </ul>
 

src/test/kotlin/fr/shikkanime/OldEpisodeScraper.kt

@@ -1,5 +1,6 @@
 package fr.shikkanime
 
+import fr.shikkanime.caches.CountryCodeAnimeIdKeyCache
 import fr.shikkanime.entities.Config
 import fr.shikkanime.entities.Episode
 import fr.shikkanime.entities.enums.ConfigPropertyKey
@@ -10,6 +11,7 @@ import fr.shikkanime.services.ConfigService
 import fr.shikkanime.services.EpisodeService
 import fr.shikkanime.utils.Constant
 import fr.shikkanime.utils.HttpRequest
+import fr.shikkanime.utils.ObjectParser.getAsInt
 import fr.shikkanime.utils.ObjectParser.getAsString
 import fr.shikkanime.wrappers.CrunchyrollWrapper
 import kotlinx.coroutines.runBlocking
@@ -55,6 +57,9 @@ fun main() {
         .map { ZonedDateTime.of(it.atTime(23, 59, 59), Constant.utcZoneId) }
         .sorted()
 
+    val minDate = dates.minOrNull()!!
+    val maxDate = dates.maxOrNull()!!
+
     val simulcasts = dates.map {
         "${Constant.seasons[(it.monthValue - 1) / 3]}-${it.year}".lowercase().replace("autumn", "fall")
     }.toSet()
@@ -88,11 +93,8 @@ fun main() {
 //        }
 //    }
 
-    val minDate = dates.minOrNull()!!
-    val maxDate = dates.maxOrNull()!!
     val accessToken = runBlocking { CrunchyrollWrapper.getAnonymousAccessToken() }
-    var page = 1
-    val fetch = 1000
+    val cms = runBlocking { CrunchyrollWrapper.getCMS(accessToken) }
 
     val series = simulcasts.flatMap { simulcastId ->
         runBlocking {
@@ -105,45 +107,54 @@ fun main() {
                 simulcast = simulcastId
             )
         }
-    }.map { jsonObject -> jsonObject.getAsString("title")!!.lowercase() }.toSet()
-    println("Simulcasts: $series")
+    }
+
+    val titles = series.map { jsonObject -> jsonObject.getAsString("title")!!.lowercase() }.toSet()
+    val ids = series.map { jsonObject -> jsonObject.getAsString("id")!! }.toSet()
+    println("Simulcasts: $titles")
 
-    crunchyrollPlatform.simulcasts[CountryCode.FR] = series
+    crunchyrollPlatform.simulcasts[CountryCode.FR] = titles
 
-    while (true) {
-        val fetchApi = runBlocking { CrunchyrollWrapper.getBrowse(CountryCode.FR.locale, accessToken, size = fetch, start = (page - 1) * fetch) }.toMutableList()
+    series.forEach {
+        val postersTall = it.getAsJsonObject("images").getAsJsonArray("poster_tall")[0].asJsonArray
+        val postersWide = it.getAsJsonObject("images").getAsJsonArray("poster_wide")[0].asJsonArray
+        val image = postersTall?.maxByOrNull { poster -> poster.asJsonObject.getAsInt("width")!! }?.asJsonObject?.getAsString("source")!!
+        val banner = postersWide?.maxByOrNull { poster -> poster.asJsonObject.getAsInt("width")!! }?.asJsonObject?.getAsString("source")!!
+        val description = it.getAsString("description")
 
-        val episodeDates = fetchApi
-            .map { episodeJson ->
-                val episodeMetadata = episodeJson.getAsJsonObject("episode_metadata")
-                episodeJson to requireNotNull(episodeMetadata.getAsString("premium_available_date")?.let { ZonedDateTime.parse(it) }) { "Release date is null" }
-            }.toMutableList()
+        crunchyrollPlatform.animeInfoCache[CountryCodeAnimeIdKeyCache(CountryCode.FR, it.getAsString("id")!!)] = CrunchyrollPlatform.CrunchyrollAnimeContent(image = image, banner = banner, description = description,)
+    }
+
+    val episodeIds = ids.parallelStream().map { seriesId ->
+        runBlocking { CrunchyrollWrapper.getSeasons(CountryCode.FR.locale, accessToken, cms, seriesId) }
+            .filter { jsonObject -> jsonObject.getAsJsonArray("subtitle_locales").map { it.asString }.contains(CountryCode.FR.locale) }
+            .map { jsonObject -> jsonObject.getAsString("id")!! }
+            .flatMap { id -> runBlocking { CrunchyrollWrapper.getEpisodes(CountryCode.FR.locale, accessToken, cms, id) } }
+            .map { jsonObject -> jsonObject.getAsString("id")!! }
+    }.toList().flatten().toSet()
 
-        val lastDate = episodeDates.maxByOrNull { it.second }!!.second
-        val firstDate = episodeDates.minByOrNull { it.second }!!.second
-        episodeDates.removeIf { it.second !in minDate..maxDate }
+    episodeIds.chunked(25).parallelStream().forEach { episodeIdsChunked ->
+        val `object` = runBlocking { CrunchyrollWrapper.getObject(CountryCode.FR.locale, accessToken, cms, *episodeIdsChunked.toTypedArray()) }
 
-        episodeDates.forEach { (episodeJson, _) ->
+        `object`.forEach { episodeJson ->
             try {
                 episodes.add(crunchyrollPlatform.convertJsonEpisode(
                     CountryCode.FR,
-                    episodeJson
+                    episodeJson,
                 ))
-            } catch (_: Exception) {
-
+            } catch (e: Exception) {
+                println("Error while converting episode (Episode ID: ${episodeJson.getAsString("id")}): ${e.message}")
+                e.printStackTrace()
             }
         }
-
-        if (lastDate.isBefore(minDate) || firstDate.isAfter(maxDate)) {
-            break
-        }
-
-        page++
     }
 
     httpRequest.close()
 
-    episodes.forEach { episode ->
+    episodes.removeIf { it.releaseDateTime.isBefore(minDate) || it.releaseDateTime.isAfter(maxDate) }
+
+    episodes.sortedBy { it.releaseDateTime }.forEach { episode ->
+        episode.anime?.releaseDateTime = episodes.filter { it.anime?.name == episode.anime?.name }.minOf { it.anime!!.releaseDateTime }
         episodeService.save(episode)
     }