Skip to content

Commit

Permalink
Merge pull request #251 from Shikkanime/dev
Browse files Browse the repository at this point in the history 
Update deprecated crunchyroll url
  • Loading branch information
@Ziedelth
Ziedelth authored Mar 5, 2024
2 parents 3faa750 + 9a06b69 commit cf2c6fe
Show file tree
Hide file tree
Showing 6 changed files with 141 additions and 64 deletions.
104 changes: 75 additions & 29 deletions src/main/kotlin/fr/shikkanime/jobs/FetchDeprecatedEpisodeJob.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,9 +2,9 @@ package fr.shikkanime.jobs

import com.google.gson.JsonObject
import com.google.inject.Inject
import fr.shikkanime.caches.CountryCodeAnimeIdKeyCache
import fr.shikkanime.entities.Episode
import fr.shikkanime.entities.enums.ConfigPropertyKey
import fr.shikkanime.entities.enums.CountryCode
import fr.shikkanime.entities.enums.EpisodeType
import fr.shikkanime.entities.enums.Platform
import fr.shikkanime.services.EpisodeService
Expand All @@ -20,6 +20,9 @@ import fr.shikkanime.wrappers.AnimationDigitalNetworkWrapper
import fr.shikkanime.wrappers.CrunchyrollWrapper
import fr.shikkanime.wrappers.PrimeVideoWrapper
import kotlinx.coroutines.runBlocking
import java.net.URLEncoder
import java.nio.charset.StandardCharsets
import java.time.Duration
import java.time.ZonedDateTime
import java.util.logging.Level

Expand All @@ -28,6 +31,9 @@ private const val IMAGE_NULL_ERROR = "Image is null"
class FetchDeprecatedEpisodeJob : AbstractJob {
private val logger = LoggerFactory.getLogger(javaClass)

var accessToken: String = ""
lateinit var cms: CrunchyrollWrapper.CMS

@Inject
private lateinit var episodeService: EpisodeService

Expand Down Expand Up @@ -55,7 +61,9 @@ class FetchDeprecatedEpisodeJob : AbstractJob {

val httpRequest = HttpRequest()
val anonymousAccessToken = runBlocking { CrunchyrollWrapper.getAnonymousAccessToken() }
accessToken = anonymousAccessToken
val cms = runBlocking { CrunchyrollWrapper.getCMS(anonymousAccessToken) }
this.cms = cms
var count = 0

episodes.forEachIndexed { index, episode ->
Expand Down Expand Up @@ -84,7 +92,7 @@ class FetchDeprecatedEpisodeJob : AbstractJob {
}
}${episode.number}"

private fun update(
fun update(
episode: Episode,
httpRequest: HttpRequest,
anonymousAccessToken: String,
Expand Down Expand Up @@ -112,6 +120,19 @@ class FetchDeprecatedEpisodeJob : AbstractJob {
needUpdate = true
}

if (episode.platform!! == Platform.CRUN) {
val url = buildCrunchyrollEpisodeUrl(content, episode)

if (url.contains("media-")) {
logger.warning("Please update the episode URL for ${getIdentifier(episode)}")
}

if (url != episode.url) {
episode.url = url
needUpdate = true
}
}

if (image != episode.image) {
episode.image = image
ImageService.remove(episode.uuid!!, ImageService.Type.IMAGE)
Expand All @@ -130,15 +151,11 @@ class FetchDeprecatedEpisodeJob : AbstractJob {
return needUpdate
}

private fun normalizeUrl(platform: Platform, countryCode: CountryCode, url: String): String {
return when (platform) {
Platform.CRUN -> {
val other = "https://www.crunchyroll.com/${countryCode.name.lowercase()}/"
url.replace("https://www.crunchyroll.com/", other)
}

else -> url
}
fun buildCrunchyrollEpisodeUrl(content: JsonObject, episode: Episode): String {
val id = content.getAsString("id")!!
val slugTitle = content.getAsString("slug_title")
val url = CrunchyrollWrapper.buildUrl(episode.anime!!.countryCode!!, id, slugTitle)
return url
}

private suspend fun normalizeContent(
Expand All @@ -155,11 +172,8 @@ class FetchDeprecatedEpisodeJob : AbstractJob {
}

Platform.CRUN -> {
val id = getCrunchyrollId(episode.url!!) ?: return run {
logger.warning("Please update the episode URL for ${getIdentifier(episode)}")
crunchyrollExternalIdToId(httpRequest, episode, accessToken, cms)
}

val id =
getCrunchyrollEpisodeId(episode.url!!) ?: return crunchyrollExternalIdToId(httpRequest, episode)
CrunchyrollWrapper.getObject(episode.anime!!.countryCode!!.locale, accessToken, cms, id)[0]
}

Expand All @@ -177,30 +191,62 @@ class FetchDeprecatedEpisodeJob : AbstractJob {
}
}

@Deprecated("This method is deprecated due to Crunchyroll no redirecting to the correct page")
private suspend fun crunchyrollExternalIdToId(
private val episodesInfoCache = MapCache<CountryCodeAnimeIdKeyCache, List<JsonObject>>(Duration.ofDays(1)) {
runBlocking {
val episodes = CrunchyrollWrapper.getSeasons(it.countryCode.locale, accessToken, cms, it.animeId)
.flatMap { season ->
CrunchyrollWrapper.getEpisodes(
it.countryCode.locale,
accessToken,
cms,
season.getAsString("id")!!
)
}
.map { it.getAsString("id")!! }
.chunked(25)

episodes.flatMap { chunk ->
CrunchyrollWrapper.getObject(it.countryCode.locale, accessToken, cms, *chunk.toTypedArray())
}
}
}

fun crunchyrollExternalIdToId(
httpRequest: HttpRequest,
episode: Episode,
accessToken: String,
cms: CrunchyrollWrapper.CMS
): JsonObject? {
try {
val selector = "div[data-t=\"search-series-card\"]"
val titleSelector = "a[tabindex=\"0\"]"

val content = try {
httpRequest.getBrowser(
normalizeUrl(
episode.platform!!,
episode.anime!!.countryCode!!,
episode.url!!
)
"https://www.crunchyroll.com/${episode.anime!!.countryCode!!.name.lowercase()}/search?q=${
URLEncoder.encode(
episode.anime!!.name!!,
StandardCharsets.UTF_8
)
}",
selector
)
} catch (e: Exception) {
return null
}

val id = getCrunchyrollId(httpRequest.lastPageUrl!!) ?: return null
return CrunchyrollWrapper.getObject(episode.anime!!.countryCode!!.locale, accessToken, cms, id)[0]
val seriesCard = content.select(selector)
val serieCard = seriesCard.find { it.select(titleSelector).attr("title") == episode.anime!!.name }
?: throw Exception("Failed to find serie card for ${episode.anime!!.name}")
val seriesId = getCrunchyrollSeriesId(serieCard.select(titleSelector).attr("href"))
?: throw Exception("Failed to find serie id for ${episode.anime!!.name}")
val allEpisodes =
episodesInfoCache[CountryCodeAnimeIdKeyCache(episode.anime!!.countryCode!!, seriesId)] ?: return null
return allEpisodes.find { it.getAsString("external_id")?.contains(getExternalId(episode.url!!)!!) == true }
}

fun getCrunchyrollId(url: String) = "/watch/([A-Z0-9]+)".toRegex().find(url)?.groupValues?.get(1)
private fun getExternalId(url: String) = "-([0-9]+)".toRegex().find(url)?.groupValues?.get(1)

private fun getCrunchyrollSeriesId(url: String) = "/series/([A-Z0-9]+)".toRegex().find(url)?.groupValues?.get(1)

fun getCrunchyrollEpisodeId(url: String) = "/watch/([A-Z0-9]+)".toRegex().find(url)?.groupValues?.get(1)

fun normalizeTitle(platform: Platform, content: JsonObject): String? {
var title = when (platform) {
Expand Down
30 changes: 19 additions & 11 deletions src/main/kotlin/fr/shikkanime/platforms/CrunchyrollPlatform.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -293,16 +293,7 @@ class CrunchyrollPlatform : AbstractPlatform<CrunchyrollConfiguration, CountryCo
)

val langType = if (isDubbed) LangType.VOICE else LangType.SUBTITLES

// @DEPRECATED
val externalId = jsonObject.getAsString("external_id")?.split(".")?.last() ?: ""
val deprecatedHash = "${countryCode}-${getPlatform()}-$externalId-$langType"
if (hashCache.contains(deprecatedHash)) throw EpisodeAlreadyReleasedException()
// @DEPRECATED

val id = requireNotNull(jsonObject.getAsString("id")) { "Id is null" }
val hash = "${countryCode}-${getPlatform()}-$id-$langType"
if (hashCache.contains(hash)) throw EpisodeAlreadyReleasedException()
val (id, hash) = getEpisodeIdAndHash(jsonObject, countryCode, langType)

val releaseDate =
requireNotNull(
Expand All @@ -322,7 +313,7 @@ class CrunchyrollPlatform : AbstractPlatform<CrunchyrollConfiguration, CountryCo

val title = jsonObject.getAsString("title")
val slugTitle = jsonObject.getAsString("slug_title")
val url = "https://www.crunchyroll.com/${countryCode.name.lowercase()}/watch/$id/$slugTitle"
val url = CrunchyrollWrapper.buildUrl(countryCode, id, slugTitle)

val thumbnailArray = jsonObject.getAsJsonObject("images")?.getAsJsonArray("thumbnail")
val biggestImage = thumbnailArray?.get(0)?.asJsonArray?.maxByOrNull { it.asJsonObject.getAsInt("width") ?: 0 }
Expand Down Expand Up @@ -373,6 +364,23 @@ class CrunchyrollPlatform : AbstractPlatform<CrunchyrollConfiguration, CountryCo
)
}

private fun getEpisodeIdAndHash(
jsonObject: JsonObject,
countryCode: CountryCode,
langType: LangType
): Pair<String, String> {
// @DEPRECATED
val externalId = jsonObject.getAsString("external_id")?.split(".")?.last() ?: ""
val deprecatedHash = "${countryCode}-${getPlatform()}-$externalId-$langType"
if (hashCache.contains(deprecatedHash)) throw EpisodeAlreadyReleasedException()
// @DEPRECATED

val id = requireNotNull(jsonObject.getAsString("id")) { "Id is null" }
val hash = "${countryCode}-${getPlatform()}-$id-$langType"
if (hashCache.contains(hash)) throw EpisodeAlreadyReleasedException()
return Pair(id, hash)
}


private fun convertXMLEpisode(countryCode: CountryCode, jsonObject: JsonObject): Episode {
val animeName = jsonObject.getAsString("crunchyroll:seriesTitle") ?: throw Exception("Anime name is null")
Expand Down
3 changes: 2 additions & 1 deletion src/main/kotlin/fr/shikkanime/repositories/EpisodeRepository.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -145,7 +145,8 @@ class EpisodeRepository : AbstractRepository<Episode>() {
AND (
(lastUpdateDateTime < :lastUpdateDateTime OR lastUpdateDateTime IS NULL) OR
(description IS NULL OR description = '') OR
image = :defaultImage
image = :defaultImage OR
url LIKE '%media-%'
)
""".trimIndent(),
getEntityClass()
Expand Down
8 changes: 3 additions & 5 deletions src/main/kotlin/fr/shikkanime/utils/HttpRequest.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ import org.jsoup.nodes.Document
import kotlin.system.measureTimeMillis

private const val TIMEOUT = 60_000L
private const val BROWSER_TIMEOUT = 15_000L
private val logger = LoggerFactory.getLogger(HttpRequest::class.java)

class HttpRequest : AutoCloseable {
Expand Down Expand Up @@ -74,8 +75,8 @@ class HttpRequest : AutoCloseable {

browser = Constant.playwright.firefox().launch(Constant.launchOptions)
page = browser?.newPage()
page?.setDefaultTimeout(TIMEOUT.toDouble())
page?.setDefaultNavigationTimeout(TIMEOUT.toDouble())
page?.setDefaultTimeout(BROWSER_TIMEOUT.toDouble())
page?.setDefaultNavigationTimeout(BROWSER_TIMEOUT.toDouble())
isBrowserInitialized = true
}

Expand Down Expand Up @@ -107,9 +108,6 @@ class HttpRequest : AutoCloseable {
return Jsoup.parse(content ?: throw Exception("Content is null"))
}

val lastPageUrl: String?
get() = page?.url()

override fun close() {
page?.close()
browser?.close()
Expand Down
4 changes: 4 additions & 0 deletions src/main/kotlin/fr/shikkanime/wrappers/CrunchyrollWrapper.kt
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package fr.shikkanime.wrappers

import com.google.gson.JsonObject
import fr.shikkanime.entities.enums.CountryCode
import fr.shikkanime.utils.HttpRequest
import fr.shikkanime.utils.ObjectParser
import fr.shikkanime.utils.ObjectParser.getAsString
Expand Down Expand Up @@ -148,4 +149,7 @@ object CrunchyrollWrapper {
return ObjectParser.fromJson(response.bodyAsText()).getAsJsonArray("items")?.map { it.asJsonObject }
?: throw Exception("Failed to get simulcasts")
}

fun buildUrl(countryCode: CountryCode, id: String, slugTitle: String?) =
"https://www.crunchyroll.com/${countryCode.name.lowercase()}/watch/$id/${slugTitle ?: ""}"
}
56 changes: 38 additions & 18 deletions src/test/kotlin/fr/shikkanime/jobs/FetchDeprecatedEpisodeJobTest.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,6 @@ import kotlinx.coroutines.runBlocking
import org.junit.jupiter.api.AfterEach
import org.junit.jupiter.api.Assertions.assertEquals
import org.junit.jupiter.api.BeforeEach
import org.junit.jupiter.api.Disabled
import org.junit.jupiter.api.Test
import java.io.File
import java.util.*
Expand Down Expand Up @@ -105,43 +104,64 @@ class FetchDeprecatedEpisodeJobTest {
fun normalizeUrl() {
assertEquals(
"GMKUXPD53",
fetchDeprecatedEpisodeJob.getCrunchyrollId("https://www.crunchyroll.com/fr/watch/GMKUXPD53/")
fetchDeprecatedEpisodeJob.getCrunchyrollEpisodeId("https://www.crunchyroll.com/fr/watch/GMKUXPD53/")
)
assertEquals(
"G14U415N4",
fetchDeprecatedEpisodeJob.getCrunchyrollId("https://www.crunchyroll.com/fr/watch/G14U415N4/the-panicked-foolish-angel-and-demon")
fetchDeprecatedEpisodeJob.getCrunchyrollEpisodeId("https://www.crunchyroll.com/fr/watch/G14U415N4/the-panicked-foolish-angel-and-demon")
)
assertEquals(
"G14U415D2",
fetchDeprecatedEpisodeJob.getCrunchyrollId("https://www.crunchyroll.com/fr/watch/G14U415D2/natsukawa-senpai-is-super-good-looking")
fetchDeprecatedEpisodeJob.getCrunchyrollEpisodeId("https://www.crunchyroll.com/fr/watch/G14U415D2/natsukawa-senpai-is-super-good-looking")
)
assertEquals(
"G8WUN158J",
fetchDeprecatedEpisodeJob.getCrunchyrollId("https://www.crunchyroll.com/fr/watch/G8WUN158J/")
fetchDeprecatedEpisodeJob.getCrunchyrollEpisodeId("https://www.crunchyroll.com/fr/watch/G8WUN158J/")
)
assertEquals(
"GEVUZD021",
fetchDeprecatedEpisodeJob.getCrunchyrollId("https://www.crunchyroll.com/fr/watch/GEVUZD021/becoming-a-three-star-chef")
fetchDeprecatedEpisodeJob.getCrunchyrollEpisodeId("https://www.crunchyroll.com/fr/watch/GEVUZD021/becoming-a-three-star-chef")
)
assertEquals(
"GK9U3KWN4",
fetchDeprecatedEpisodeJob.getCrunchyrollId("https://www.crunchyroll.com/fr/watch/GK9U3KWN4/yukis-world")
fetchDeprecatedEpisodeJob.getCrunchyrollEpisodeId("https://www.crunchyroll.com/fr/watch/GK9U3KWN4/yukis-world")
)
}

@Test
@Disabled("Crunchyroll redirect to a 404 page")
fun bug() {
val normalizeUrl = "https://www.crunchyroll.com/fr/media-918855"

val lastPage = HttpRequest().use {
it.getBrowser(normalizeUrl)
it.lastPageUrl!!
}

println(lastPage)
val id = fetchDeprecatedEpisodeJob.getCrunchyrollId(lastPage)
assertEquals("GVWU07GP0", id)
val token = runBlocking { CrunchyrollWrapper.getAnonymousAccessToken() }
val cms = runBlocking { CrunchyrollWrapper.getCMS(token) }

val episode = Episode(
platform = Platform.CRUN,
anime = Anime(
countryCode = CountryCode.FR,
name = "Villainess Level 99: I May Be the Hidden Boss But I'm Not the Demon Lord",
image = "https://www.crunchyroll.com/imgsrv/display/thumbnail/480x720/catalog/crunchyroll/9cf39e672287c0b7d81d6ce6ba897b25.jpe",
banner = "https://www.crunchyroll.com/imgsrv/display/thumbnail/1920x1080/catalog/crunchyroll/b759905ae99ec12686f372129ce96799.jpe",
description = "Cette étudiante japonaise discrète est réincarnée dans le corps d’Eumiella Dolkness, la méchante de son otome game préféré. Aspirant toujours à une vie tranquille, elle n’est pas vraiment ravie et décide d’abandonner ses fonctions maléfiques. Jusqu'à ce que son côté gamer entre en jeu et qu'elle atteigne accidentellement le niveau 99 ! À présent, tout le monde la soupçonne d'être l'infâme Maître des Démons…",
slug = "villainess-level-99"
),
episodeType = EpisodeType.EPISODE,
langType = LangType.SUBTITLES,
hash = "FR-CRUN-918565-SUBTITLES",
season = 1,
number = 9,
title = "Le boss caché se fait démarcher par un pays ennemi",
url = "https://www.crunchyroll.com/media-918565",
image = "https://www.crunchyroll.com/imgsrv/display/thumbnail/1920x1080/catalog/crunchyroll/f4afb9fbdd5a99bcdfbe349e6d00acb2.jpe",
duration = 1420
)

fetchDeprecatedEpisodeJob.accessToken = token
fetchDeprecatedEpisodeJob.cms = cms
val content = fetchDeprecatedEpisodeJob.crunchyrollExternalIdToId(HttpRequest(), episode)!!
assertEquals("G7PU418J7", content.getAsString("id"))
assertEquals(
"https://www.crunchyroll.com/fr/watch/G7PU418J7/the-hidden-boss-is-solicited-by-an-enemy-nation",
fetchDeprecatedEpisodeJob.buildCrunchyrollEpisodeUrl(content, episode)
)
}

@Test
Expand Down

0 comments on commit cf2c6fe

Please sign in to comment.