Skip to content

Commit

Permalink
Merge pull request #256 from Shikkanime/dev
Browse files Browse the repository at this point in the history 
Dev
  • Loading branch information
@Ziedelth
Ziedelth committed Mar 7, 2024
2 parents 9968ed0 + 2d6e252 commit 1112957
Show file tree
Hide file tree
Showing 4 changed files with 33 additions and 162 deletions.
5 changes: 2 additions & 3 deletions build.gradle.kts
View file
Original file line number Diff line number Diff line change
@@ -1,11 +1,10 @@
import java.net.URI

val ktorVersion = "2.3.9"
val kotlinVersion = "1.9.22"
val ktorSwaggerUiVersion = "2.7.4"
val hibernateCoreVersion = "6.4.4.Final"
val ehcacheVersion = "3.10.8"
val glassfishJaxbVersion = "4.0.4"
val glassfishJaxbVersion = "4.0.5"
val hibernateSearchVersion = "7.1.0.Final"
val tikaVersion = "3.0.0-BETA"
val postgresqlVersion = "42.7.2"
Expand All @@ -29,7 +28,7 @@ val junitVersion = "5.10.2"
val h2Version = "2.2.224"

plugins {
kotlin("jvm") version "1.9.22"
kotlin("jvm") version "1.9.23"
id("io.ktor.plugin") version "2.3.9"
jacoco
id("org.sonarqube") version "4.4.1.3373"
Expand Down
79 changes: 3 additions & 76 deletions src/main/kotlin/fr/shikkanime/jobs/FetchDeprecatedEpisodeJob.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,15 +2,13 @@ package fr.shikkanime.jobs

import com.google.gson.JsonObject
import com.google.inject.Inject
import fr.shikkanime.caches.CountryCodeAnimeIdKeyCache
import fr.shikkanime.entities.Episode
import fr.shikkanime.entities.enums.ConfigPropertyKey
import fr.shikkanime.entities.enums.EpisodeType
import fr.shikkanime.entities.enums.Platform
import fr.shikkanime.services.EpisodeService
import fr.shikkanime.services.ImageService
import fr.shikkanime.services.caches.ConfigCacheService
import fr.shikkanime.utils.HttpRequest
import fr.shikkanime.utils.LoggerFactory
import fr.shikkanime.utils.MapCache
import fr.shikkanime.utils.ObjectParser.getAsInt
Expand All @@ -20,9 +18,6 @@ import fr.shikkanime.wrappers.AnimationDigitalNetworkWrapper
import fr.shikkanime.wrappers.CrunchyrollWrapper
import fr.shikkanime.wrappers.PrimeVideoWrapper
import kotlinx.coroutines.runBlocking
import java.net.URLEncoder
import java.nio.charset.StandardCharsets
import java.time.Duration
import java.time.ZonedDateTime
import java.util.logging.Level

Expand All @@ -31,9 +26,6 @@ private const val IMAGE_NULL_ERROR = "Image is null"
class FetchDeprecatedEpisodeJob : AbstractJob {
private val logger = LoggerFactory.getLogger(javaClass)

var accessToken: String = ""
lateinit var cms: CrunchyrollWrapper.CMS

@Inject
private lateinit var episodeService: EpisodeService

Expand Down Expand Up @@ -63,23 +55,18 @@ class FetchDeprecatedEpisodeJob : AbstractJob {
return
}

val httpRequest = HttpRequest()
val anonymousAccessToken = runBlocking { CrunchyrollWrapper.getAnonymousAccessToken() }
accessToken = anonymousAccessToken
val cms = runBlocking { CrunchyrollWrapper.getCMS(anonymousAccessToken) }
this.cms = cms
var count = 0

episodes.forEachIndexed { index, episode ->
logger.info("Fetching episode description ${index + 1}/${episodes.size}")

if (update(episode, httpRequest, anonymousAccessToken, cms, now)) {
if (update(episode, anonymousAccessToken, cms, now)) {
count++
}
}

httpRequest.close()

if (count <= 0) {
return
}
Expand All @@ -98,7 +85,6 @@ class FetchDeprecatedEpisodeJob : AbstractJob {

fun update(
episode: Episode,
httpRequest: HttpRequest,
anonymousAccessToken: String,
cms: CrunchyrollWrapper.CMS,
now: ZonedDateTime,
Expand All @@ -108,7 +94,7 @@ class FetchDeprecatedEpisodeJob : AbstractJob {

try {
val content =
runBlocking { normalizeContent(episode, httpRequest, anonymousAccessToken, cms) } ?: return false
runBlocking { normalizeContent(episode, anonymousAccessToken, cms) } ?: return false
val title = normalizeTitle(episode.platform!!, content)
val description = normalizeDescription(episode.platform!!, content)
val image = normalizeImage(episode.platform!!, content)
Expand Down Expand Up @@ -160,7 +146,6 @@ class FetchDeprecatedEpisodeJob : AbstractJob {

private suspend fun normalizeContent(
episode: Episode,
httpRequest: HttpRequest,
accessToken: String,
cms: CrunchyrollWrapper.CMS
): JsonObject? {
Expand All @@ -172,8 +157,7 @@ class FetchDeprecatedEpisodeJob : AbstractJob {
}

Platform.CRUN -> {
val id =
getCrunchyrollEpisodeId(episode.url!!) ?: return crunchyrollExternalIdToId(httpRequest, episode)
val id = getCrunchyrollEpisodeId(episode.url!!) ?: return null
CrunchyrollWrapper.getObject(episode.anime!!.countryCode!!.locale, accessToken, cms, id)[0]
}

Expand All @@ -191,63 +175,6 @@ class FetchDeprecatedEpisodeJob : AbstractJob {
}
}

private val episodesInfoCache = MapCache<CountryCodeAnimeIdKeyCache, List<JsonObject>>(Duration.ofDays(1)) {
runBlocking {
val episodes = CrunchyrollWrapper.getSeasons(it.countryCode.locale, accessToken, cms, it.animeId)
.flatMap { season ->
CrunchyrollWrapper.getEpisodes(
it.countryCode.locale,
accessToken,
cms,
season.getAsString("id")!!
)
}
.map { it.getAsString("id")!! }
.chunked(25)

episodes.flatMap { chunk ->
CrunchyrollWrapper.getObject(it.countryCode.locale, accessToken, cms, *chunk.toTypedArray())
}
}
}

fun crunchyrollExternalIdToId(
httpRequest: HttpRequest,
episode: Episode,
): JsonObject? {
val selector = "div[data-t=\"search-series-card\"]"
val titleSelector = "a[tabindex=\"0\"]"

val content = try {
httpRequest.getBrowser(
"https://www.crunchyroll.com/${episode.anime!!.countryCode!!.name.lowercase()}/search?q=${
URLEncoder.encode(
episode.anime!!.name!!,
StandardCharsets.UTF_8
)
}",
selector
)
} catch (e: Exception) {
return null
}

val seriesCard = content.select(selector)
val serieCard = seriesCard.find { it.select(titleSelector).attr("title") == episode.anime!!.name }
?: throw Exception("Failed to find serie card for ${episode.anime!!.name}")
val seriesId = getCrunchyrollSeriesId(serieCard.select(titleSelector).attr("href"))
?: throw Exception("Failed to find serie id for ${episode.anime!!.name}")
val allEpisodes =
episodesInfoCache[CountryCodeAnimeIdKeyCache(episode.anime!!.countryCode!!, seriesId)] ?: return null

val externalId = "EPI.${getExternalId(episode.url!!)}"
return allEpisodes.find { it.getAsString("external_id") == externalId }
}

private fun getExternalId(url: String) = "-([0-9]{6})".toRegex().find(url)?.groupValues?.get(1)

private fun getCrunchyrollSeriesId(url: String) = "/series/([A-Z0-9]+)".toRegex().find(url)?.groupValues?.get(1)

fun getCrunchyrollEpisodeId(url: String) = "/watch/([A-Z0-9]+)".toRegex().find(url)?.groupValues?.get(1)

fun normalizeTitle(platform: Platform, content: JsonObject): String? {
Expand Down
38 changes: 28 additions & 10 deletions src/main/resources/db/changelog/2024/03/02-changelog.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,12 +4,14 @@
xmlns="http://www.liquibase.org/xml/ns/dbchangelog"
xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog
http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-4.26.xsd" objectQuotingStrategy="QUOTE_ONLY_RESERVED_WORDS">
<property global="false" name="id" value="1709569841763"/>
<property global="false" name="id" value="1709800525900"/>
<property global="false" name="author" value="Ziedelth"/>

<changeSet id="${id}-1" author="${author}">
<preConditions onFail="MARK_RAN">
<indexExists tableName="anime" indexName="idx_anime_country_code"/>
<not>
<indexExists tableName="anime" indexName="idx_anime_country_code"/>
</not>
</preConditions>

<createIndex indexName="idx_anime_country_code" tableName="anime">
Expand All @@ -19,7 +21,9 @@

<changeSet id="${id}-2" author="${author}">
<preConditions onFail="MARK_RAN">
<indexExists tableName="episode" indexName="idx_episode_episode_type"/>
<not>
<indexExists tableName="episode" indexName="idx_episode_episode_type"/>
</not>
</preConditions>

<createIndex indexName="idx_episode_episode_type" tableName="episode">
Expand All @@ -29,7 +33,9 @@

<changeSet id="${id}-3" author="${author}">
<preConditions onFail="MARK_RAN">
<indexExists tableName="episode" indexName="idx_episode_lang_type"/>
<not>
<indexExists tableName="episode" indexName="idx_episode_lang_type"/>
</not>
</preConditions>

<createIndex indexName="idx_episode_lang_type" tableName="episode">
Expand All @@ -39,7 +45,9 @@

<changeSet id="${id}-4" author="${author}">
<preConditions onFail="MARK_RAN">
<indexExists tableName="episode" indexName="idx_episode_number"/>
<not>
<indexExists tableName="episode" indexName="idx_episode_number"/>
</not>
</preConditions>

<createIndex indexName="idx_episode_number" tableName="episode">
Expand All @@ -49,7 +57,9 @@

<changeSet id="${id}-5" author="${author}">
<preConditions onFail="MARK_RAN">
<indexExists tableName="episode" indexName="idx_episode_release_date_time"/>
<not>
<indexExists tableName="episode" indexName="idx_episode_release_date_time"/>
</not>
</preConditions>

<createIndex indexName="idx_episode_release_date_time" tableName="episode">
Expand All @@ -59,7 +69,9 @@

<changeSet id="${id}-6" author="${author}">
<preConditions onFail="MARK_RAN">
<indexExists tableName="episode" indexName="idx_episode_season"/>
<not>
<indexExists tableName="episode" indexName="idx_episode_season"/>
</not>
</preConditions>

<createIndex indexName="idx_episode_season" tableName="episode">
Expand All @@ -69,7 +81,9 @@

<changeSet id="${id}-7" author="${author}">
<preConditions onFail="MARK_RAN">
<indexExists tableName="simulcast" indexName="idx_simulcast_season"/>
<not>
<indexExists tableName="simulcast" indexName="idx_simulcast_season"/>
</not>
</preConditions>

<createIndex indexName="idx_simulcast_season"
Expand All @@ -80,7 +94,9 @@

<changeSet id="${id}-8" author="${author}">
<preConditions onFail="MARK_RAN">
<indexExists tableName="simulcast" indexName="idx_simulcast_season_year"/>
<not>
<indexExists tableName="simulcast" indexName="idx_simulcast_season_year"/>
</not>
</preConditions>

<createIndex indexName="idx_simulcast_season_year"
Expand All @@ -92,7 +108,9 @@

<changeSet id="${id}-9" author="${author}">
<preConditions onFail="MARK_RAN">
<indexExists tableName="simulcast" indexName="idx_simulcast_year"/>
<not>
<indexExists tableName="simulcast" indexName="idx_simulcast_year"/>
</not>
</preConditions>

<createIndex indexName="idx_simulcast_year"
Expand Down
73 changes: 0 additions & 73 deletions src/test/kotlin/fr/shikkanime/jobs/FetchDeprecatedEpisodeJobTest.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,6 @@ import fr.shikkanime.services.AnimeService
import fr.shikkanime.services.ConfigService
import fr.shikkanime.services.EpisodeService
import fr.shikkanime.utils.Constant
import fr.shikkanime.utils.HttpRequest
import fr.shikkanime.utils.ObjectParser
import fr.shikkanime.utils.ObjectParser.getAsString
import fr.shikkanime.wrappers.AnimationDigitalNetworkWrapper
Expand Down Expand Up @@ -128,78 +127,6 @@ class FetchDeprecatedEpisodeJobTest {
)
}

@Test
fun bug() {
val token = runBlocking { CrunchyrollWrapper.getAnonymousAccessToken() }
val cms = runBlocking { CrunchyrollWrapper.getCMS(token) }

val episode = Episode(
platform = Platform.CRUN,
anime = Anime(
countryCode = CountryCode.FR,
name = "Villainess Level 99: I May Be the Hidden Boss But I'm Not the Demon Lord",
image = "https://www.crunchyroll.com/imgsrv/display/thumbnail/480x720/catalog/crunchyroll/9cf39e672287c0b7d81d6ce6ba897b25.jpe",
banner = "https://www.crunchyroll.com/imgsrv/display/thumbnail/1920x1080/catalog/crunchyroll/b759905ae99ec12686f372129ce96799.jpe",
description = "Cette étudiante japonaise discrète est réincarnée dans le corps d’Eumiella Dolkness, la méchante de son otome game préféré. Aspirant toujours à une vie tranquille, elle n’est pas vraiment ravie et décide d’abandonner ses fonctions maléfiques. Jusqu'à ce que son côté gamer entre en jeu et qu'elle atteigne accidentellement le niveau 99 ! À présent, tout le monde la soupçonne d'être l'infâme Maître des Démons…",
slug = "villainess-level-99"
),
episodeType = EpisodeType.EPISODE,
langType = LangType.SUBTITLES,
hash = "FR-CRUN-918565-SUBTITLES",
season = 1,
number = 9,
title = "Le boss caché se fait démarcher par un pays ennemi",
url = "https://www.crunchyroll.com/media-918565",
image = "https://www.crunchyroll.com/imgsrv/display/thumbnail/1920x1080/catalog/crunchyroll/f4afb9fbdd5a99bcdfbe349e6d00acb2.jpe",
duration = 1420
)

fetchDeprecatedEpisodeJob.accessToken = token
fetchDeprecatedEpisodeJob.cms = cms
val content = fetchDeprecatedEpisodeJob.crunchyrollExternalIdToId(HttpRequest(), episode)!!
assertEquals("G7PU418J7", content.getAsString("id"))
assertEquals(
"https://www.crunchyroll.com/fr/watch/G7PU418J7/the-hidden-boss-is-solicited-by-an-enemy-nation",
fetchDeprecatedEpisodeJob.buildCrunchyrollEpisodeUrl(content, episode)
)
}

@Test
fun bug2() {
val token = runBlocking { CrunchyrollWrapper.getAnonymousAccessToken() }
val cms = runBlocking { CrunchyrollWrapper.getCMS(token) }

val episode = Episode(
platform = Platform.CRUN,
anime = Anime(
countryCode = CountryCode.FR,
name = "Bottom-Tier Character Tomozaki",
image = "https://www.crunchyroll.com/imgsrv/display/thumbnail/480x720/catalog/crunchyroll/fa8c0b715dda49a4cbb8094c4136b382.jpe",
banner = "https://www.crunchyroll.com/imgsrv/display/thumbnail/1920x1080/catalog/crunchyroll/b2dbd10a57e485f3ba4fcab6116f7625.jpe",
description = "Tomozaki Fumiya est à la fois l'un des meilleurs gamers du Japon et un lycéen des plus solitaires. Un jour, celui qui voit la vie comme un jeu sans intérêt rencontre l'héroïne parfaite de son établissement, Hinami Aoi... qui lui ordonne de prendre sa vie en main aussi sérieusement qu'il vit ses parties de jeu vidéo ! La vie est-elle un jeu sans intérêt ou le plus fin des plaisirs ? Avec Hinami aux manettes, une petite révolution s'amorce dans l'existence de Tomozaki !",
slug = "bottom-tier-character-tomozaki"
),
episodeType = EpisodeType.EPISODE,
langType = LangType.SUBTITLES,
hash = "FR-CRUN-917959-SUBTITLES",
season = 2,
number = 1,
title = "Collecter des informations sans s’ennuyer, c’est parfait",
url = "https://www.crunchyroll.com/fr/episode-1-the-best-games-make-reconnaissance-fun-917959",
image = "https://img1.ak.crunchyroll.com/i/spire4-tmb/63cf5c6f4cbc2a0beac3c3f10b8fe3791704287894_full.jpg",
duration = 1422
)

fetchDeprecatedEpisodeJob.accessToken = token
fetchDeprecatedEpisodeJob.cms = cms
val content = fetchDeprecatedEpisodeJob.crunchyrollExternalIdToId(HttpRequest(), episode)!!
assertEquals("G7PU413X5", content.getAsString("id"))
assertEquals(
"https://www.crunchyroll.com/fr/watch/G7PU413X5/the-best-games-make-reconnaissance-fun",
fetchDeprecatedEpisodeJob.buildCrunchyrollEpisodeUrl(content, episode)
)
}

@Test
fun normalizeDescription() {
val content = runBlocking { AnimationDigitalNetworkWrapper.getShowVideo(24108) }
Expand Down

0 comments on commit 1112957

Please sign in to comment.