diff --git a/src/main/kotlin/fr/shikkanime/jobs/FetchDeprecatedEpisodeJob.kt b/src/main/kotlin/fr/shikkanime/jobs/FetchDeprecatedEpisodeJob.kt index 3e56cf7d..2858415f 100644 --- a/src/main/kotlin/fr/shikkanime/jobs/FetchDeprecatedEpisodeJob.kt +++ b/src/main/kotlin/fr/shikkanime/jobs/FetchDeprecatedEpisodeJob.kt @@ -2,9 +2,9 @@ package fr.shikkanime.jobs import com.google.gson.JsonObject import com.google.inject.Inject +import fr.shikkanime.caches.CountryCodeAnimeIdKeyCache import fr.shikkanime.entities.Episode import fr.shikkanime.entities.enums.ConfigPropertyKey -import fr.shikkanime.entities.enums.CountryCode import fr.shikkanime.entities.enums.EpisodeType import fr.shikkanime.entities.enums.Platform import fr.shikkanime.services.EpisodeService @@ -20,6 +20,9 @@ import fr.shikkanime.wrappers.AnimationDigitalNetworkWrapper import fr.shikkanime.wrappers.CrunchyrollWrapper import fr.shikkanime.wrappers.PrimeVideoWrapper import kotlinx.coroutines.runBlocking +import java.net.URLEncoder +import java.nio.charset.StandardCharsets +import java.time.Duration import java.time.ZonedDateTime import java.util.logging.Level @@ -28,6 +31,9 @@ private const val IMAGE_NULL_ERROR = "Image is null" class FetchDeprecatedEpisodeJob : AbstractJob { private val logger = LoggerFactory.getLogger(javaClass) + var accessToken: String = "" + lateinit var cms: CrunchyrollWrapper.CMS + @Inject private lateinit var episodeService: EpisodeService @@ -55,7 +61,9 @@ class FetchDeprecatedEpisodeJob : AbstractJob { val httpRequest = HttpRequest() val anonymousAccessToken = runBlocking { CrunchyrollWrapper.getAnonymousAccessToken() } + accessToken = anonymousAccessToken val cms = runBlocking { CrunchyrollWrapper.getCMS(anonymousAccessToken) } + this.cms = cms var count = 0 episodes.forEachIndexed { index, episode -> @@ -84,7 +92,7 @@ class FetchDeprecatedEpisodeJob : AbstractJob { } }${episode.number}" - private fun update( + fun update( episode: Episode, httpRequest: HttpRequest, anonymousAccessToken: String, @@ -112,6 +120,19 @@ class FetchDeprecatedEpisodeJob : AbstractJob { needUpdate = true } + if (episode.platform!! == Platform.CRUN) { + val url = buildCrunchyrollEpisodeUrl(content, episode) + + if (url.contains("media-")) { + logger.warning("Please update the episode URL for ${getIdentifier(episode)}") + } + + if (url != episode.url) { + episode.url = url + needUpdate = true + } + } + if (image != episode.image) { episode.image = image ImageService.remove(episode.uuid!!, ImageService.Type.IMAGE) @@ -130,15 +151,11 @@ class FetchDeprecatedEpisodeJob : AbstractJob { return needUpdate } - private fun normalizeUrl(platform: Platform, countryCode: CountryCode, url: String): String { - return when (platform) { - Platform.CRUN -> { - val other = "https://www.crunchyroll.com/${countryCode.name.lowercase()}/" - url.replace("https://www.crunchyroll.com/", other) - } - - else -> url - } + fun buildCrunchyrollEpisodeUrl(content: JsonObject, episode: Episode): String { + val id = content.getAsString("id")!! + val slugTitle = content.getAsString("slug_title") + val url = CrunchyrollWrapper.buildUrl(episode.anime!!.countryCode!!, id, slugTitle) + return url } private suspend fun normalizeContent( @@ -155,11 +172,8 @@ class FetchDeprecatedEpisodeJob : AbstractJob { } Platform.CRUN -> { - val id = getCrunchyrollId(episode.url!!) ?: return run { - logger.warning("Please update the episode URL for ${getIdentifier(episode)}") - crunchyrollExternalIdToId(httpRequest, episode, accessToken, cms) - } - + val id = + getCrunchyrollEpisodeId(episode.url!!) ?: return crunchyrollExternalIdToId(httpRequest, episode) CrunchyrollWrapper.getObject(episode.anime!!.countryCode!!.locale, accessToken, cms, id)[0] } @@ -177,30 +191,62 @@ class FetchDeprecatedEpisodeJob : AbstractJob { } } - @Deprecated("This method is deprecated due to Crunchyroll no redirecting to the correct page") - private suspend fun crunchyrollExternalIdToId( + private val episodesInfoCache = MapCache>(Duration.ofDays(1)) { + runBlocking { + val episodes = CrunchyrollWrapper.getSeasons(it.countryCode.locale, accessToken, cms, it.animeId) + .flatMap { season -> + CrunchyrollWrapper.getEpisodes( + it.countryCode.locale, + accessToken, + cms, + season.getAsString("id")!! + ) + } + .map { it.getAsString("id")!! } + .chunked(25) + + episodes.flatMap { chunk -> + CrunchyrollWrapper.getObject(it.countryCode.locale, accessToken, cms, *chunk.toTypedArray()) + } + } + } + + fun crunchyrollExternalIdToId( httpRequest: HttpRequest, episode: Episode, - accessToken: String, - cms: CrunchyrollWrapper.CMS ): JsonObject? { - try { + val selector = "div[data-t=\"search-series-card\"]" + val titleSelector = "a[tabindex=\"0\"]" + + val content = try { httpRequest.getBrowser( - normalizeUrl( - episode.platform!!, - episode.anime!!.countryCode!!, - episode.url!! - ) + "https://www.crunchyroll.com/${episode.anime!!.countryCode!!.name.lowercase()}/search?q=${ + URLEncoder.encode( + episode.anime!!.name!!, + StandardCharsets.UTF_8 + ) + }", + selector ) } catch (e: Exception) { return null } - val id = getCrunchyrollId(httpRequest.lastPageUrl!!) ?: return null - return CrunchyrollWrapper.getObject(episode.anime!!.countryCode!!.locale, accessToken, cms, id)[0] + val seriesCard = content.select(selector) + val serieCard = seriesCard.find { it.select(titleSelector).attr("title") == episode.anime!!.name } + ?: throw Exception("Failed to find serie card for ${episode.anime!!.name}") + val seriesId = getCrunchyrollSeriesId(serieCard.select(titleSelector).attr("href")) + ?: throw Exception("Failed to find serie id for ${episode.anime!!.name}") + val allEpisodes = + episodesInfoCache[CountryCodeAnimeIdKeyCache(episode.anime!!.countryCode!!, seriesId)] ?: return null + return allEpisodes.find { it.getAsString("external_id")?.contains(getExternalId(episode.url!!)!!) == true } } - fun getCrunchyrollId(url: String) = "/watch/([A-Z0-9]+)".toRegex().find(url)?.groupValues?.get(1) + private fun getExternalId(url: String) = "-([0-9]+)".toRegex().find(url)?.groupValues?.get(1) + + private fun getCrunchyrollSeriesId(url: String) = "/series/([A-Z0-9]+)".toRegex().find(url)?.groupValues?.get(1) + + fun getCrunchyrollEpisodeId(url: String) = "/watch/([A-Z0-9]+)".toRegex().find(url)?.groupValues?.get(1) fun normalizeTitle(platform: Platform, content: JsonObject): String? { var title = when (platform) { diff --git a/src/main/kotlin/fr/shikkanime/platforms/CrunchyrollPlatform.kt b/src/main/kotlin/fr/shikkanime/platforms/CrunchyrollPlatform.kt index f19e066e..160188b6 100644 --- a/src/main/kotlin/fr/shikkanime/platforms/CrunchyrollPlatform.kt +++ b/src/main/kotlin/fr/shikkanime/platforms/CrunchyrollPlatform.kt @@ -293,16 +293,7 @@ class CrunchyrollPlatform : AbstractPlatform { + // @DEPRECATED + val externalId = jsonObject.getAsString("external_id")?.split(".")?.last() ?: "" + val deprecatedHash = "${countryCode}-${getPlatform()}-$externalId-$langType" + if (hashCache.contains(deprecatedHash)) throw EpisodeAlreadyReleasedException() + // @DEPRECATED + + val id = requireNotNull(jsonObject.getAsString("id")) { "Id is null" } + val hash = "${countryCode}-${getPlatform()}-$id-$langType" + if (hashCache.contains(hash)) throw EpisodeAlreadyReleasedException() + return Pair(id, hash) + } + private fun convertXMLEpisode(countryCode: CountryCode, jsonObject: JsonObject): Episode { val animeName = jsonObject.getAsString("crunchyroll:seriesTitle") ?: throw Exception("Anime name is null") diff --git a/src/main/kotlin/fr/shikkanime/repositories/EpisodeRepository.kt b/src/main/kotlin/fr/shikkanime/repositories/EpisodeRepository.kt index 8d1a7801..47009894 100644 --- a/src/main/kotlin/fr/shikkanime/repositories/EpisodeRepository.kt +++ b/src/main/kotlin/fr/shikkanime/repositories/EpisodeRepository.kt @@ -145,7 +145,8 @@ class EpisodeRepository : AbstractRepository() { AND ( (lastUpdateDateTime < :lastUpdateDateTime OR lastUpdateDateTime IS NULL) OR (description IS NULL OR description = '') OR - image = :defaultImage + image = :defaultImage OR + url LIKE '%media-%' ) """.trimIndent(), getEntityClass() diff --git a/src/main/kotlin/fr/shikkanime/utils/HttpRequest.kt b/src/main/kotlin/fr/shikkanime/utils/HttpRequest.kt index 6d21acb5..8854ba6e 100644 --- a/src/main/kotlin/fr/shikkanime/utils/HttpRequest.kt +++ b/src/main/kotlin/fr/shikkanime/utils/HttpRequest.kt @@ -12,6 +12,7 @@ import org.jsoup.nodes.Document import kotlin.system.measureTimeMillis private const val TIMEOUT = 60_000L +private const val BROWSER_TIMEOUT = 15_000L private val logger = LoggerFactory.getLogger(HttpRequest::class.java) class HttpRequest : AutoCloseable { @@ -74,8 +75,8 @@ class HttpRequest : AutoCloseable { browser = Constant.playwright.firefox().launch(Constant.launchOptions) page = browser?.newPage() - page?.setDefaultTimeout(TIMEOUT.toDouble()) - page?.setDefaultNavigationTimeout(TIMEOUT.toDouble()) + page?.setDefaultTimeout(BROWSER_TIMEOUT.toDouble()) + page?.setDefaultNavigationTimeout(BROWSER_TIMEOUT.toDouble()) isBrowserInitialized = true } @@ -107,9 +108,6 @@ class HttpRequest : AutoCloseable { return Jsoup.parse(content ?: throw Exception("Content is null")) } - val lastPageUrl: String? - get() = page?.url() - override fun close() { page?.close() browser?.close() diff --git a/src/main/kotlin/fr/shikkanime/wrappers/CrunchyrollWrapper.kt b/src/main/kotlin/fr/shikkanime/wrappers/CrunchyrollWrapper.kt index 6868d456..342e2be5 100644 --- a/src/main/kotlin/fr/shikkanime/wrappers/CrunchyrollWrapper.kt +++ b/src/main/kotlin/fr/shikkanime/wrappers/CrunchyrollWrapper.kt @@ -1,6 +1,7 @@ package fr.shikkanime.wrappers import com.google.gson.JsonObject +import fr.shikkanime.entities.enums.CountryCode import fr.shikkanime.utils.HttpRequest import fr.shikkanime.utils.ObjectParser import fr.shikkanime.utils.ObjectParser.getAsString @@ -148,4 +149,7 @@ object CrunchyrollWrapper { return ObjectParser.fromJson(response.bodyAsText()).getAsJsonArray("items")?.map { it.asJsonObject } ?: throw Exception("Failed to get simulcasts") } + + fun buildUrl(countryCode: CountryCode, id: String, slugTitle: String?) = + "https://www.crunchyroll.com/${countryCode.name.lowercase()}/watch/$id/${slugTitle ?: ""}" } \ No newline at end of file diff --git a/src/test/kotlin/fr/shikkanime/jobs/FetchDeprecatedEpisodeJobTest.kt b/src/test/kotlin/fr/shikkanime/jobs/FetchDeprecatedEpisodeJobTest.kt index ab61184e..5aadef5e 100644 --- a/src/test/kotlin/fr/shikkanime/jobs/FetchDeprecatedEpisodeJobTest.kt +++ b/src/test/kotlin/fr/shikkanime/jobs/FetchDeprecatedEpisodeJobTest.kt @@ -21,7 +21,6 @@ import kotlinx.coroutines.runBlocking import org.junit.jupiter.api.AfterEach import org.junit.jupiter.api.Assertions.assertEquals import org.junit.jupiter.api.BeforeEach -import org.junit.jupiter.api.Disabled import org.junit.jupiter.api.Test import java.io.File import java.util.* @@ -105,43 +104,64 @@ class FetchDeprecatedEpisodeJobTest { fun normalizeUrl() { assertEquals( "GMKUXPD53", - fetchDeprecatedEpisodeJob.getCrunchyrollId("https://www.crunchyroll.com/fr/watch/GMKUXPD53/") + fetchDeprecatedEpisodeJob.getCrunchyrollEpisodeId("https://www.crunchyroll.com/fr/watch/GMKUXPD53/") ) assertEquals( "G14U415N4", - fetchDeprecatedEpisodeJob.getCrunchyrollId("https://www.crunchyroll.com/fr/watch/G14U415N4/the-panicked-foolish-angel-and-demon") + fetchDeprecatedEpisodeJob.getCrunchyrollEpisodeId("https://www.crunchyroll.com/fr/watch/G14U415N4/the-panicked-foolish-angel-and-demon") ) assertEquals( "G14U415D2", - fetchDeprecatedEpisodeJob.getCrunchyrollId("https://www.crunchyroll.com/fr/watch/G14U415D2/natsukawa-senpai-is-super-good-looking") + fetchDeprecatedEpisodeJob.getCrunchyrollEpisodeId("https://www.crunchyroll.com/fr/watch/G14U415D2/natsukawa-senpai-is-super-good-looking") ) assertEquals( "G8WUN158J", - fetchDeprecatedEpisodeJob.getCrunchyrollId("https://www.crunchyroll.com/fr/watch/G8WUN158J/") + fetchDeprecatedEpisodeJob.getCrunchyrollEpisodeId("https://www.crunchyroll.com/fr/watch/G8WUN158J/") ) assertEquals( "GEVUZD021", - fetchDeprecatedEpisodeJob.getCrunchyrollId("https://www.crunchyroll.com/fr/watch/GEVUZD021/becoming-a-three-star-chef") + fetchDeprecatedEpisodeJob.getCrunchyrollEpisodeId("https://www.crunchyroll.com/fr/watch/GEVUZD021/becoming-a-three-star-chef") ) assertEquals( "GK9U3KWN4", - fetchDeprecatedEpisodeJob.getCrunchyrollId("https://www.crunchyroll.com/fr/watch/GK9U3KWN4/yukis-world") + fetchDeprecatedEpisodeJob.getCrunchyrollEpisodeId("https://www.crunchyroll.com/fr/watch/GK9U3KWN4/yukis-world") ) } @Test - @Disabled("Crunchyroll redirect to a 404 page") fun bug() { - val normalizeUrl = "https://www.crunchyroll.com/fr/media-918855" - - val lastPage = HttpRequest().use { - it.getBrowser(normalizeUrl) - it.lastPageUrl!! - } - - println(lastPage) - val id = fetchDeprecatedEpisodeJob.getCrunchyrollId(lastPage) - assertEquals("GVWU07GP0", id) + val token = runBlocking { CrunchyrollWrapper.getAnonymousAccessToken() } + val cms = runBlocking { CrunchyrollWrapper.getCMS(token) } + + val episode = Episode( + platform = Platform.CRUN, + anime = Anime( + countryCode = CountryCode.FR, + name = "Villainess Level 99: I May Be the Hidden Boss But I'm Not the Demon Lord", + image = "https://www.crunchyroll.com/imgsrv/display/thumbnail/480x720/catalog/crunchyroll/9cf39e672287c0b7d81d6ce6ba897b25.jpe", + banner = "https://www.crunchyroll.com/imgsrv/display/thumbnail/1920x1080/catalog/crunchyroll/b759905ae99ec12686f372129ce96799.jpe", + description = "Cette étudiante japonaise discrète est réincarnée dans le corps d’Eumiella Dolkness, la méchante de son otome game préféré. Aspirant toujours à une vie tranquille, elle n’est pas vraiment ravie et décide d’abandonner ses fonctions maléfiques. Jusqu'à ce que son côté gamer entre en jeu et qu'elle atteigne accidentellement le niveau 99 ! À présent, tout le monde la soupçonne d'être l'infâme Maître des Démons…", + slug = "villainess-level-99" + ), + episodeType = EpisodeType.EPISODE, + langType = LangType.SUBTITLES, + hash = "FR-CRUN-918565-SUBTITLES", + season = 1, + number = 9, + title = "Le boss caché se fait démarcher par un pays ennemi", + url = "https://www.crunchyroll.com/media-918565", + image = "https://www.crunchyroll.com/imgsrv/display/thumbnail/1920x1080/catalog/crunchyroll/f4afb9fbdd5a99bcdfbe349e6d00acb2.jpe", + duration = 1420 + ) + + fetchDeprecatedEpisodeJob.accessToken = token + fetchDeprecatedEpisodeJob.cms = cms + val content = fetchDeprecatedEpisodeJob.crunchyrollExternalIdToId(HttpRequest(), episode)!! + assertEquals("G7PU418J7", content.getAsString("id")) + assertEquals( + "https://www.crunchyroll.com/fr/watch/G7PU418J7/the-hidden-boss-is-solicited-by-an-enemy-nation", + fetchDeprecatedEpisodeJob.buildCrunchyrollEpisodeUrl(content, episode) + ) } @Test