From a54d57af9e1a1c4c267e0cea3dd7c819cdf3864d Mon Sep 17 00:00:00 2001 From: Ziedelth Date: Mon, 12 Feb 2024 00:22:50 +0100 Subject: [PATCH] Upgrade old episode detection --- .../controllers/site/SiteController.kt | 10 ++- .../SimulcastToSimulcastDtoConverter.kt | 7 ++ .../kotlin/fr/shikkanime/dtos/SimulcastDto.kt | 1 + .../fr/shikkanime/services/EpisodeService.kt | 8 +++ .../kotlin/fr/shikkanime/utils/HttpRequest.kt | 2 +- .../fr/shikkanime/utils/ObjectParser.kt | 2 +- .../shikkanime/wrappers/CrunchyrollWrapper.kt | 30 ++++++++- .../templates/_freemarker_implicit.ftl | 2 +- src/main/resources/templates/admin/images.ftl | 2 +- src/main/resources/templates/site/catalog.ftl | 19 +----- .../kotlin/fr/shikkanime/OldEpisodeScraper.kt | 67 +++++++++++-------- 11 files changed, 98 insertions(+), 52 deletions(-) diff --git a/src/main/kotlin/fr/shikkanime/controllers/site/SiteController.kt b/src/main/kotlin/fr/shikkanime/controllers/site/SiteController.kt index 05268a20..2e012351 100644 --- a/src/main/kotlin/fr/shikkanime/controllers/site/SiteController.kt +++ b/src/main/kotlin/fr/shikkanime/controllers/site/SiteController.kt @@ -32,12 +32,15 @@ class SiteController { @Path @Get private fun home(): Response { + val findAll = simulcastCacheService.findAll()!! + val currentSimulcast = findAll.first() + return Response.template( Link.HOME, mutableMapOf( "animes" to animeCacheService.findAllBy( CountryCode.FR, - null, + currentSimulcast.uuid, listOf(SortParameter("name", SortParameter.Order.ASC)), 1, 6 @@ -94,7 +97,8 @@ class SiteController { val currentSimulcast = findAll.first { "${it.season.lowercase()}-${it.year}" == slug } return Response.template( - Link.CATALOG, + Link.CATALOG.template, + currentSimulcast.label, mutableMapOf( "simulcasts" to findAll, "currentSimulcast" to currentSimulcast, @@ -138,7 +142,7 @@ class SiteController { SortParameter("langType", SortParameter.Order.ASC), ), 1, - 12 + 24 )!!.data ) ) diff --git a/src/main/kotlin/fr/shikkanime/converters/simulcast/SimulcastToSimulcastDtoConverter.kt b/src/main/kotlin/fr/shikkanime/converters/simulcast/SimulcastToSimulcastDtoConverter.kt index 92a5594b..45a2f5c5 100644 --- a/src/main/kotlin/fr/shikkanime/converters/simulcast/SimulcastToSimulcastDtoConverter.kt +++ b/src/main/kotlin/fr/shikkanime/converters/simulcast/SimulcastToSimulcastDtoConverter.kt @@ -10,6 +10,13 @@ class SimulcastToSimulcastDtoConverter : AbstractConverter "Hiver" + "SPRING" -> "Printemps" + "SUMMER" -> "Été" + "AUTUMN" -> "Automne" + else -> "Inconnu" + } + " ${from.year}" ) } } \ No newline at end of file diff --git a/src/main/kotlin/fr/shikkanime/dtos/SimulcastDto.kt b/src/main/kotlin/fr/shikkanime/dtos/SimulcastDto.kt index 31c2a7c6..80ff8e34 100644 --- a/src/main/kotlin/fr/shikkanime/dtos/SimulcastDto.kt +++ b/src/main/kotlin/fr/shikkanime/dtos/SimulcastDto.kt @@ -7,4 +7,5 @@ data class SimulcastDto( val uuid: UUID?, val season: String, val year: Int, + val label: String, ) : Serializable diff --git a/src/main/kotlin/fr/shikkanime/services/EpisodeService.kt b/src/main/kotlin/fr/shikkanime/services/EpisodeService.kt index 43f60c72..f2ab165c 100644 --- a/src/main/kotlin/fr/shikkanime/services/EpisodeService.kt +++ b/src/main/kotlin/fr/shikkanime/services/EpisodeService.kt @@ -115,6 +115,14 @@ class EpisodeService : AbstractService() { entity.anime = animeService.update(anime) } + if (1000 < (entity.title?.length ?: 0)) { + entity.title = entity.title!!.substring(0, 1000) + } + + if (1000 < (entity.description?.length ?: 0)) { + entity.description = entity.description!!.substring(0, 1000) + } + val savedEntity = super.save(entity) addImage(savedEntity.uuid!!, savedEntity.image!!) MapCache.invalidate(Episode::class.java) diff --git a/src/main/kotlin/fr/shikkanime/utils/HttpRequest.kt b/src/main/kotlin/fr/shikkanime/utils/HttpRequest.kt index 8d875fb3..9a644f12 100644 --- a/src/main/kotlin/fr/shikkanime/utils/HttpRequest.kt +++ b/src/main/kotlin/fr/shikkanime/utils/HttpRequest.kt @@ -11,7 +11,7 @@ import org.jsoup.Jsoup import org.jsoup.nodes.Document import kotlin.system.measureTimeMillis -private const val TIMEOUT = 15_000L +private const val TIMEOUT = 60_000L private val logger = LoggerFactory.getLogger(HttpRequest::class.java) class HttpRequest : AutoCloseable { diff --git a/src/main/kotlin/fr/shikkanime/utils/ObjectParser.kt b/src/main/kotlin/fr/shikkanime/utils/ObjectParser.kt index a6afcc05..7f5bd773 100644 --- a/src/main/kotlin/fr/shikkanime/utils/ObjectParser.kt +++ b/src/main/kotlin/fr/shikkanime/utils/ObjectParser.kt @@ -48,7 +48,7 @@ object ObjectParser { } fun JsonObject.getAsInt(key: String): Int? { - return this[key]?.asInt + return if (this[key] != null && !this[key].isJsonNull) this[key]?.asInt else null } fun JsonObject.getAsInt(key: String, default: Int): Int { diff --git a/src/main/kotlin/fr/shikkanime/wrappers/CrunchyrollWrapper.kt b/src/main/kotlin/fr/shikkanime/wrappers/CrunchyrollWrapper.kt index 664479e6..c6a679ee 100644 --- a/src/main/kotlin/fr/shikkanime/wrappers/CrunchyrollWrapper.kt +++ b/src/main/kotlin/fr/shikkanime/wrappers/CrunchyrollWrapper.kt @@ -86,7 +86,7 @@ object CrunchyrollWrapper { ), ) - require(response.status.value == 200) { "Failed to get media list" } + require(response.status.value == 200) { "Failed to get media list (${response.status.value})" } return ObjectParser.fromJson(response.bodyAsText()).getAsJsonArray("data")?.map { it.asJsonObject } ?: throw Exception("Failed to get media list") @@ -106,6 +106,34 @@ object CrunchyrollWrapper { ?: throw Exception("Failed to get media object") } + suspend fun getSeasons(locale: String, accessToken: String, cms: CMS, seriesId: String): List { + val response = HttpRequest().get( + "${BETA_URL}cms/v2${cms.bucket}/seasons?series_id=$seriesId&Policy=${cms.policy}&Signature=${cms.signature}&Key-Pair-Id=${cms.keyPairId}&locale=$locale", + headers = mapOf( + "Authorization" to "Bearer $accessToken", + ), + ) + + require(response.status.value == 200) { "Failed to get seasons" } + + return ObjectParser.fromJson(response.bodyAsText()).getAsJsonArray("items")?.map { it.asJsonObject } + ?: throw Exception("Failed to get seasons") + } + + suspend fun getEpisodes(locale: String, accessToken: String, cms: CMS, seasonId: String): List { + val response = HttpRequest().get( + "${BETA_URL}cms/v2${cms.bucket}/episodes?season_id=$seasonId&Policy=${cms.policy}&Signature=${cms.signature}&Key-Pair-Id=${cms.keyPairId}&locale=$locale", + headers = mapOf( + "Authorization" to "Bearer $accessToken", + ), + ) + + require(response.status.value == 200) { "Failed to get episodes" } + + return ObjectParser.fromJson(response.bodyAsText()).getAsJsonArray("items")?.map { it.asJsonObject } + ?: throw Exception("Failed to get episodes") + } + suspend fun getSimulcasts(locale: String, accessToken: String): List { val response = HttpRequest().get( "${BASE_URL}content/v1/season_list?locale=$locale", diff --git a/src/main/resources/templates/_freemarker_implicit.ftl b/src/main/resources/templates/_freemarker_implicit.ftl index b08a6a6c..2d754afb 100644 --- a/src/main/resources/templates/_freemarker_implicit.ftl +++ b/src/main/resources/templates/_freemarker_implicit.ftl @@ -15,7 +15,7 @@ [#-- @ftlvariable name="config" type="fr.shikkanime.dtos.ConfigDto" --] [#-- @ftlvariable name="animes" type="kotlin.collections.AbstractList" --] [#-- @ftlvariable name="episodes" type="kotlin.collections.AbstractList" --] -[#-- @ftlvariable name="size" type="kotlin.NumbersKt" --] +[#-- @ftlvariable name="size" type="java.lang.Integer" --] [#-- @ftlvariable name="originalSize" type="java.lang.String" --] [#-- @ftlvariable name="compressedSize" type="java.lang.String" --] [#-- @ftlvariable name="simulcasts" type="kotlin.collections.AbstractList" --] diff --git a/src/main/resources/templates/admin/images.ftl b/src/main/resources/templates/admin/images.ftl index 9c096d17..7e366858 100644 --- a/src/main/resources/templates/admin/images.ftl +++ b/src/main/resources/templates/admin/images.ftl @@ -10,7 +10,7 @@
- +
diff --git a/src/main/resources/templates/site/catalog.ftl b/src/main/resources/templates/site/catalog.ftl index a2ef4403..fc4f6d65 100644 --- a/src/main/resources/templates/site/catalog.ftl +++ b/src/main/resources/templates/site/catalog.ftl @@ -2,28 +2,15 @@ <#import "components/episode.ftl" as episodeComponent /> <#import "components/anime.ftl" as animeComponent /> -<#function getPrefixSimulcast(season)> - <#switch season> - <#case "WINTER"> - <#return "Hiver"> - <#case "SPRING"> - <#return "Printemps"> - <#case "SUMMER"> - <#return "Été"> - <#case "AUTUMN"> - <#return "Automne"> - - - <@navigation.display>
-
    + diff --git a/src/test/kotlin/fr/shikkanime/OldEpisodeScraper.kt b/src/test/kotlin/fr/shikkanime/OldEpisodeScraper.kt index 1d033f91..1e58f4c1 100644 --- a/src/test/kotlin/fr/shikkanime/OldEpisodeScraper.kt +++ b/src/test/kotlin/fr/shikkanime/OldEpisodeScraper.kt @@ -1,5 +1,6 @@ package fr.shikkanime +import fr.shikkanime.caches.CountryCodeAnimeIdKeyCache import fr.shikkanime.entities.Config import fr.shikkanime.entities.Episode import fr.shikkanime.entities.enums.ConfigPropertyKey @@ -10,6 +11,7 @@ import fr.shikkanime.services.ConfigService import fr.shikkanime.services.EpisodeService import fr.shikkanime.utils.Constant import fr.shikkanime.utils.HttpRequest +import fr.shikkanime.utils.ObjectParser.getAsInt import fr.shikkanime.utils.ObjectParser.getAsString import fr.shikkanime.wrappers.CrunchyrollWrapper import kotlinx.coroutines.runBlocking @@ -55,6 +57,9 @@ fun main() { .map { ZonedDateTime.of(it.atTime(23, 59, 59), Constant.utcZoneId) } .sorted() + val minDate = dates.minOrNull()!! + val maxDate = dates.maxOrNull()!! + val simulcasts = dates.map { "${Constant.seasons[(it.monthValue - 1) / 3]}-${it.year}".lowercase().replace("autumn", "fall") }.toSet() @@ -88,11 +93,8 @@ fun main() { // } // } - val minDate = dates.minOrNull()!! - val maxDate = dates.maxOrNull()!! val accessToken = runBlocking { CrunchyrollWrapper.getAnonymousAccessToken() } - var page = 1 - val fetch = 1000 + val cms = runBlocking { CrunchyrollWrapper.getCMS(accessToken) } val series = simulcasts.flatMap { simulcastId -> runBlocking { @@ -105,45 +107,54 @@ fun main() { simulcast = simulcastId ) } - }.map { jsonObject -> jsonObject.getAsString("title")!!.lowercase() }.toSet() - println("Simulcasts: $series") + } + + val titles = series.map { jsonObject -> jsonObject.getAsString("title")!!.lowercase() }.toSet() + val ids = series.map { jsonObject -> jsonObject.getAsString("id")!! }.toSet() + println("Simulcasts: $titles") - crunchyrollPlatform.simulcasts[CountryCode.FR] = series + crunchyrollPlatform.simulcasts[CountryCode.FR] = titles - while (true) { - val fetchApi = runBlocking { CrunchyrollWrapper.getBrowse(CountryCode.FR.locale, accessToken, size = fetch, start = (page - 1) * fetch) }.toMutableList() + series.forEach { + val postersTall = it.getAsJsonObject("images").getAsJsonArray("poster_tall")[0].asJsonArray + val postersWide = it.getAsJsonObject("images").getAsJsonArray("poster_wide")[0].asJsonArray + val image = postersTall?.maxByOrNull { poster -> poster.asJsonObject.getAsInt("width")!! }?.asJsonObject?.getAsString("source")!! + val banner = postersWide?.maxByOrNull { poster -> poster.asJsonObject.getAsInt("width")!! }?.asJsonObject?.getAsString("source")!! + val description = it.getAsString("description") - val episodeDates = fetchApi - .map { episodeJson -> - val episodeMetadata = episodeJson.getAsJsonObject("episode_metadata") - episodeJson to requireNotNull(episodeMetadata.getAsString("premium_available_date")?.let { ZonedDateTime.parse(it) }) { "Release date is null" } - }.toMutableList() + crunchyrollPlatform.animeInfoCache[CountryCodeAnimeIdKeyCache(CountryCode.FR, it.getAsString("id")!!)] = CrunchyrollPlatform.CrunchyrollAnimeContent(image = image, banner = banner, description = description,) + } + + val episodeIds = ids.parallelStream().map { seriesId -> + runBlocking { CrunchyrollWrapper.getSeasons(CountryCode.FR.locale, accessToken, cms, seriesId) } + .filter { jsonObject -> jsonObject.getAsJsonArray("subtitle_locales").map { it.asString }.contains(CountryCode.FR.locale) } + .map { jsonObject -> jsonObject.getAsString("id")!! } + .flatMap { id -> runBlocking { CrunchyrollWrapper.getEpisodes(CountryCode.FR.locale, accessToken, cms, id) } } + .map { jsonObject -> jsonObject.getAsString("id")!! } + }.toList().flatten().toSet() - val lastDate = episodeDates.maxByOrNull { it.second }!!.second - val firstDate = episodeDates.minByOrNull { it.second }!!.second - episodeDates.removeIf { it.second !in minDate..maxDate } + episodeIds.chunked(25).parallelStream().forEach { episodeIdsChunked -> + val `object` = runBlocking { CrunchyrollWrapper.getObject(CountryCode.FR.locale, accessToken, cms, *episodeIdsChunked.toTypedArray()) } - episodeDates.forEach { (episodeJson, _) -> + `object`.forEach { episodeJson -> try { episodes.add(crunchyrollPlatform.convertJsonEpisode( CountryCode.FR, - episodeJson + episodeJson, )) - } catch (_: Exception) { - + } catch (e: Exception) { + println("Error while converting episode (Episode ID: ${episodeJson.getAsString("id")}): ${e.message}") + e.printStackTrace() } } - - if (lastDate.isBefore(minDate) || firstDate.isAfter(maxDate)) { - break - } - - page++ } httpRequest.close() - episodes.forEach { episode -> + episodes.removeIf { it.releaseDateTime.isBefore(minDate) || it.releaseDateTime.isAfter(maxDate) } + + episodes.sortedBy { it.releaseDateTime }.forEach { episode -> + episode.anime?.releaseDateTime = episodes.filter { it.anime?.name == episode.anime?.name }.minOf { it.anime!!.releaseDateTime } episodeService.save(episode) }