Merge pull request #355 from Shikkanime/dev

Dev

.github/dependabot.yml

@@ -12,6 +12,10 @@ updates:
       - "Ziedelth"
     reviewers:
       - "Ziedelth"
+    groups:
+      ktor:
+        patterns:
+          - "*ktor*"
 
   - package-ecosystem: "github-actions"
     directory: "/"

build.gradle.kts

@@ -1,4 +1,4 @@
-val ktorVersion = "2.3.9"
+val ktorVersion = "2.3.10"
 val ktorSwaggerUiVersion = "2.8.0"
 val hibernateCoreVersion = "6.4.4.Final"
 val ehcacheVersion = "3.10.8"
@@ -15,21 +15,21 @@ val playwrightVersion = "1.42.0"
 val jsoupVersion = "1.17.2"
 val gsonVersion = "2.10.1"
 val openCvVersion = "4.9.0-0"
-val bcprovVersion = "1.77"
+val bcprovVersion = "1.78"
 val javaImageScalingVersion = "0.8.6"
 val firebaseVersion = "9.2.0"
 
-val jdaVersion = "5.0.0-beta.21"
+val jdaVersion = "5.0.0-beta.22"
 val twitter4jVersion = "4.0.7"
 val twitter4jV2Version = "1.4.3"
 
 val junitVersion = "5.10.2"
 val h2Version = "2.2.224"
 
 plugins {
-    kotlin("jvm") version "2.0.0-Beta5"
+    kotlin("jvm") version "2.0.0-RC1"
     kotlin("kapt") version "1.9.23"
-    id("io.ktor.plugin") version "2.3.9"
+    id("io.ktor.plugin") version "2.3.10"
     jacoco
     id("org.sonarqube") version "5.0.0.4638"
 }

src/main/kotlin/fr/shikkanime/Application.kt

@@ -10,7 +10,6 @@ import fr.shikkanime.services.MemberService
 import fr.shikkanime.utils.Constant
 import fr.shikkanime.utils.JobManager
 import fr.shikkanime.utils.LoggerFactory
-import fr.shikkanime.utils.StringUtils
 import io.ktor.server.application.*
 import io.ktor.server.engine.*
 import io.ktor.server.netty.*
@@ -24,7 +23,10 @@ fun main() {
 }
 
 fun initAll(adminPassword: AtomicReference<String>?, port: Int = 37100, wait: Boolean = true): NettyApplicationEngine {
+    Constant.injector.getInstance(AnimeService::class.java).preIndex()
+
     ImageService.loadCache()
+    ImageService.addAll()
 
     if (adminPassword != null) {
         val memberService = Constant.injector.getInstance(MemberService::class.java)
@@ -36,20 +38,6 @@ fun initAll(adminPassword: AtomicReference<String>?, port: Int = 37100, wait: Bo
         }
     }
 
-    val animeService = Constant.injector.getInstance(AnimeService::class.java)
-    animeService.preIndex()
-
-    animeService.findAll().forEach {
-        val toSlug = StringUtils.toSlug(StringUtils.getShortName(it.name!!))
-
-        if (it.slug != toSlug) {
-            it.slug = toSlug
-            animeService.update(it)
-        }
-    }
-
-    ImageService.addAll()
-
     logger.info("Starting jobs...")
     // Every 10 seconds
     JobManager.scheduleJob("*/10 * * * * ?", MetricJob::class.java)

src/main/kotlin/fr/shikkanime/entities/enums/ConfigPropertyKey.kt

@@ -29,4 +29,6 @@ enum class ConfigPropertyKey(val key: String) {
     CRUNCHYROLL_FETCH_API_SIZE("crunchyroll_fetch_api_size"),
     ANIMATION_DITIGAL_NETWORK_SIMULCAST_DETECTION_REGEX("animation_digital_network_simulcast_detection_regex"),
     ANIME_EPISODES_SIZE_LIMIT("anime_episodes_size_limit"),
+    DISNEY_PLUS_AUTHORIZATION("disney_plus_authorization"),
+    DISNEY_PLUS_REFRESH_TOKEN("disney_plus_refresh_token"),
 }

src/main/kotlin/fr/shikkanime/jobs/FetchDeprecatedEpisodeJob.kt

@@ -13,6 +13,7 @@ import fr.shikkanime.utils.LoggerFactory
 import fr.shikkanime.utils.MapCache
 import fr.shikkanime.utils.ObjectParser.getAsInt
 import fr.shikkanime.utils.ObjectParser.getAsString
+import fr.shikkanime.utils.StringUtils
 import fr.shikkanime.utils.withUTC
 import fr.shikkanime.wrappers.AnimationDigitalNetworkWrapper
 import fr.shikkanime.wrappers.CrunchyrollWrapper
@@ -126,6 +127,16 @@ class FetchDeprecatedEpisodeJob : AbstractJob {
                 needUpdate = true
             }
 
+            if (episode.platform == Platform.CRUN) {
+                val id = getCrunchyrollEpisodeId(episode.url!!) ?: return false
+                val hash = StringUtils.getHash(episode.anime!!.countryCode!!, episode.platform!!, id, episode.langType!!)
+
+                if (hash != episode.hash) {
+                    episode.hash = hash
+                    needUpdate = true
+                }
+            }
+
             episode.lastUpdateDateTime = now
             episodeService.update(episode)
         } catch (e: Exception) {

src/main/kotlin/fr/shikkanime/platforms/AnimationDigitalNetworkPlatform.kt

@@ -162,7 +162,7 @@ class AnimationDigitalNetworkPlatform :
                 ),
                 episodeType = episodeType,
                 langType = langType,
-                hash = "${countryCode}-${getPlatform()}-$id-$langType",
+                hash = StringUtils.getHash(countryCode, getPlatform(), id.toString(), langType),
                 releaseDateTime = releaseDate,
                 season = season,
                 number = number,

src/main/kotlin/fr/shikkanime/platforms/CrunchyrollPlatform.kt

@@ -170,7 +170,9 @@ class CrunchyrollPlatform : AbstractPlatform<CrunchyrollConfiguration, CountryCo
         )
 
         val langType = if (isDubbed) LangType.VOICE else LangType.SUBTITLES
-        val (id, hash) = getEpisodeIdAndHash(jsonObject, countryCode, langType)
+        val id = requireNotNull(jsonObject.getAsString("id")) { "Id is null" }
+        val hash = StringUtils.getHash(countryCode, getPlatform(), id, langType)
+        if (hashCache.contains(hash)) throw EpisodeAlreadyReleasedException()
 
         val releaseDate =
             requireNotNull(
@@ -239,21 +241,4 @@ class CrunchyrollPlatform : AbstractPlatform<CrunchyrollConfiguration, CountryCo
             description = description
         )
     }
-
-    private fun getEpisodeIdAndHash(
-        jsonObject: JsonObject,
-        countryCode: CountryCode,
-        langType: LangType
-    ): Pair<String, String> {
-        // @DEPRECATED
-        val externalId = jsonObject.getAsString("external_id")?.split(".")?.last() ?: ""
-        val deprecatedHash = "${countryCode}-${getPlatform()}-$externalId-$langType"
-        if (hashCache.contains(deprecatedHash)) throw EpisodeAlreadyReleasedException()
-        // @DEPRECATED
-
-        val id = requireNotNull(jsonObject.getAsString("id")) { "Id is null" }
-        val hash = "${countryCode}-${getPlatform()}-$id-$langType"
-        if (hashCache.contains(hash)) throw EpisodeAlreadyReleasedException()
-        return Pair(id, hash)
-    }
 }

src/main/kotlin/fr/shikkanime/platforms/DisneyPlusPlatform.kt

@@ -1,93 +1,59 @@
 package fr.shikkanime.platforms
 
-import com.google.gson.JsonArray
 import com.google.gson.JsonObject
+import com.google.inject.Inject
 import fr.shikkanime.caches.CountryCodeDisneyPlusSimulcastKeyCache
 import fr.shikkanime.entities.Anime
+import fr.shikkanime.entities.Config
 import fr.shikkanime.entities.Episode
-import fr.shikkanime.entities.enums.CountryCode
-import fr.shikkanime.entities.enums.EpisodeType
-import fr.shikkanime.entities.enums.LangType
-import fr.shikkanime.entities.enums.Platform
+import fr.shikkanime.entities.enums.*
 import fr.shikkanime.exceptions.AnimeException
 import fr.shikkanime.platforms.configuration.DisneyPlusConfiguration
-import fr.shikkanime.utils.*
-import fr.shikkanime.utils.ObjectParser.getAsBoolean
+import fr.shikkanime.services.caches.ConfigCacheService
+import fr.shikkanime.utils.MapCache
 import fr.shikkanime.utils.ObjectParser.getAsInt
 import fr.shikkanime.utils.ObjectParser.getAsLong
 import fr.shikkanime.utils.ObjectParser.getAsString
-import io.ktor.client.statement.*
+import fr.shikkanime.utils.StringUtils
+import fr.shikkanime.utils.isEqualOrAfter
+import fr.shikkanime.utils.withUTC
+import fr.shikkanime.wrappers.DisneyPlusWrapper
+import kotlinx.coroutines.runBlocking
 import java.io.File
+import java.time.Duration
 import java.time.LocalTime
 import java.time.ZonedDateTime
 import java.time.format.DateTimeFormatter
 import java.util.logging.Level
 
 class DisneyPlusPlatform :
-    AbstractPlatform<DisneyPlusConfiguration, CountryCodeDisneyPlusSimulcastKeyCache, JsonArray>() {
+    AbstractPlatform<DisneyPlusConfiguration, CountryCodeDisneyPlusSimulcastKeyCache, List<JsonObject>>() {
+    @Inject
+    private lateinit var configCacheService: ConfigCacheService
+
     override fun getPlatform(): Platform = Platform.DISN
 
     override fun getConfigurationClass() = DisneyPlusConfiguration::class.java
 
-    override suspend fun fetchApiContent(
-        key: CountryCodeDisneyPlusSimulcastKeyCache,
-        zonedDateTime: ZonedDateTime
-    ): JsonArray {
-        check(configuration!!.authorization.isNotBlank()) { "Authorization is null" }
-        check(configuration!!.refreshToken.isNotBlank()) { "Refresh token is null" }
-        val httpRequest = HttpRequest()
-
-        val loginDevice = httpRequest.post(
-            "https://disney.api.edge.bamgrid.com/graph/v1/device/graphql",
-            headers = mapOf(
-                "Authorization" to configuration!!.authorization,
-            ),
-            body = ObjectParser.toJson(
-                mapOf(
-                    "operationName" to "refreshToken",
-                    "query" to "mutation refreshToken(\$input:RefreshTokenInput!){refreshToken(refreshToken:\$input){activeSession{sessionId}}}",
-                    "variables" to mapOf(
-                        "input" to mapOf(
-                            "refreshToken" to configuration!!.refreshToken
-                        )
-                    ),
-                )
+    private val identifiers = MapCache<CountryCode, String>(Duration.ofHours(3).plusMinutes(30), listOf(Config::class.java)) {
+        return@MapCache runBlocking {
+            DisneyPlusWrapper.getAccessToken(
+                configCacheService.getValueAsString(ConfigPropertyKey.DISNEY_PLUS_AUTHORIZATION),
+                configCacheService.getValueAsString(ConfigPropertyKey.DISNEY_PLUS_REFRESH_TOKEN)
             )
-        )
-
-        check(loginDevice.status.value == 200) { "Failed to login to Disney+" }
-        val loginDeviceJson = ObjectParser.fromJson(loginDevice.bodyAsText(), JsonObject::class.java)
-        val accessToken = loginDeviceJson.getAsJsonObject("extensions").getAsJsonObject("sdk").getAsJsonObject("token")
-            .getAsString("accessToken")
-
-        val seasonsResponse = httpRequest.get(
-            "https://disney.content.edge.bamgrid.com/svc/content/DmcSeriesBundle/version/5.1/region/${key.countryCode.name}/audience/k-false,l-true/maturity/1850/language/${key.countryCode.locale}/encodedSeriesId/${key.disneyPlusSimulcast.name}",
-            mapOf("Authorization" to "Bearer $accessToken")
-        )
-        check(seasonsResponse.status.value == 200) { "Failed to fetch Disney+ content" }
-        val seasonsJson = ObjectParser.fromJson(seasonsResponse.bodyAsText(), JsonObject::class.java)
-        val seasons = seasonsJson.getAsJsonObject("data").getAsJsonObject("DmcSeriesBundle").getAsJsonObject("seasons")
-            .getAsJsonArray("seasons").mapNotNull { it.asJsonObject.getAsString("seasonId") }
-        val episodes = JsonArray()
-
-        seasons.forEach { season ->
-            var page = 1
-            var hasMore: Boolean
-
-            do {
-                val url =
-                    "https://disney.content.edge.bamgrid.com/svc/content/DmcEpisodes/version/5.1/region/${key.countryCode.name}/audience/k-false,l-true/maturity/1850/language/${key.countryCode.locale}/seasonId/${season}/pageSize/15/page/${page++}"
-                val response = httpRequest.get(url, mapOf("Authorization" to "Bearer $accessToken"))
-                check(response.status.value == 200) { "Failed to fetch Disney+ content" }
-                val json = ObjectParser.fromJson(response.bodyAsText(), JsonObject::class.java)
-
-                val dmcEpisodesMeta = json.getAsJsonObject("data").getAsJsonObject("DmcEpisodes")
-                hasMore = dmcEpisodesMeta.getAsJsonObject("meta").getAsBoolean("hasMore") ?: false
-                dmcEpisodesMeta.getAsJsonArray("videos").forEach { episodes.add(it) }
-            } while (hasMore)
         }
+    }
+
+    private val seasons = MapCache<CountryCodeDisneyPlusSimulcastKeyCache, List<String>>(Duration.ofDays(1)) {
+        val accessToken = identifiers[it.countryCode]!!
+        return@MapCache runBlocking { DisneyPlusWrapper.getSeasons(accessToken, it.countryCode, it.disneyPlusSimulcast.name) }
+    }
 
-        return episodes
+    override suspend fun fetchApiContent(
+        key: CountryCodeDisneyPlusSimulcastKeyCache,
+        zonedDateTime: ZonedDateTime
+    ) = this.seasons[key]!!.flatMap { season ->
+        DisneyPlusWrapper.getEpisodes(identifiers[key.countryCode]!!, key.countryCode, season)
     }
 
     override fun fetchEpisodes(zonedDateTime: ZonedDateTime, bypassFileContent: File?): List<Episode> {
@@ -102,7 +68,7 @@ class DisneyPlusPlatform :
 
                 api.forEach {
                     try {
-                        list.add(convertEpisode(countryCode, simulcast, it.asJsonObject, zonedDateTime))
+                        list.add(convertEpisode(countryCode, simulcast, it, zonedDateTime))
                     } catch (_: AnimeException) {
                         // Ignore
                     } catch (e: Exception) {
@@ -178,7 +144,7 @@ class DisneyPlusPlatform :
             ),
             episodeType = EpisodeType.EPISODE,
             langType = langType,
-            hash = "${countryCode}-${getPlatform()}-$id-$langType",
+            hash = StringUtils.getHash(countryCode, getPlatform(), id.toString(), langType),
             releaseDateTime = releaseDateTime,
             season = season,
             number = number,

src/main/kotlin/fr/shikkanime/platforms/NetflixPlatform.kt

@@ -46,6 +46,7 @@ class NetflixPlatform : AbstractPlatform<NetflixConfiguration, CountryCodeNetfli
             val image = episode.selectFirst(".episode-thumbnail-image")?.attr("src") ?: return@mapNotNull null
             val imageWithoutParams = image.substringBefore("?")
             val episodeDescription = episode.selectFirst(".epsiode-synopsis")?.text()
+            val computedId = EncryptionManager.toSHA512("$id-${season}-$episodeNumber").substring(0..<8)
 
             Episode(
                 platform = getPlatform(),
@@ -60,14 +61,12 @@ class NetflixPlatform : AbstractPlatform<NetflixConfiguration, CountryCodeNetfli
                 ),
                 episodeType = EpisodeType.EPISODE,
                 langType = LangType.SUBTITLES,
-                hash = "${key.countryCode}-${getPlatform()}-${
-                    EncryptionManager.toSHA512("$id-${season}-$episodeNumber").substring(0..<8)
-                }-${LangType.SUBTITLES}",
+                hash = StringUtils.getHash(key.countryCode, getPlatform(), computedId, LangType.SUBTITLES),
                 releaseDateTime = releaseDateTime,
                 season = season,
                 number = episodeNumber,
                 title = episodeTitle,
-                url = "https://www.netflix.com/${key.countryCode.name.lowercase()}/title/$id",
+                url = "https://www.netflix.com/${key.countryCode.name.lowercase()}/title/$computedId",
                 image = imageWithoutParams,
                 duration = durationInSeconds,
                 description = episodeDescription,

src/main/kotlin/fr/shikkanime/platforms/PrimeVideoPlatform.kt

@@ -51,7 +51,7 @@ class PrimeVideoPlatform :
                 ),
                 episodeType = EpisodeType.EPISODE,
                 langType = LangType.SUBTITLES,
-                hash = it.getAsString("id")!!,
+                hash = StringUtils.getHash(key.countryCode, getPlatform(), it.getAsString("id")!!, LangType.SUBTITLES),
                 releaseDateTime = releaseDateTime,
                 season = it.getAsInt("season")!!,
                 number = it.getAsInt("number")!!,

src/main/kotlin/fr/shikkanime/platforms/configuration/DisneyPlusConfiguration.kt

@@ -2,10 +2,7 @@ package fr.shikkanime.platforms.configuration
 
 import io.ktor.http.*
 
-data class DisneyPlusConfiguration(
-    var authorization: String = "",
-    var refreshToken: String = "",
-) : PlatformConfiguration<DisneyPlusConfiguration.DisneyPlusSimulcast>() {
+class DisneyPlusConfiguration : PlatformConfiguration<DisneyPlusConfiguration.DisneyPlusSimulcast>() {
     data class DisneyPlusSimulcast(
         var releaseDay: Int = 1,
         var releaseTime: String = "",
@@ -57,29 +54,4 @@ data class DisneyPlusConfiguration(
     }
 
     override fun newPlatformSimulcast() = DisneyPlusSimulcast()
-
-    override fun of(parameters: Parameters) {
-        super.of(parameters)
-        parameters["authorization"]?.let { authorization = it }
-        parameters["refreshToken"]?.let { refreshToken = it }
-    }
-
-    override fun toConfigurationFields() = super.toConfigurationFields().apply {
-        add(
-            ConfigurationField(
-                label = "Authorization",
-                name = "authorization",
-                type = "text",
-                value = authorization
-            )
-        )
-        add(
-            ConfigurationField(
-                label = "Refresh token",
-                name = "refreshToken",
-                type = "text",
-                value = refreshToken
-            )
-        )
-    }
 }

src/main/kotlin/fr/shikkanime/utils/StringUtils.kt

@@ -1,8 +1,10 @@
 package fr.shikkanime.utils
 
 import fr.shikkanime.dtos.EpisodeDto
+import fr.shikkanime.entities.enums.CountryCode
 import fr.shikkanime.entities.enums.EpisodeType
 import fr.shikkanime.entities.enums.LangType
+import fr.shikkanime.entities.enums.Platform
 import java.text.Normalizer
 import java.util.*
 import java.util.regex.Pattern
@@ -74,4 +76,6 @@ object StringUtils {
     fun unSanitizeXSS(input: String): String = input.replace("&lt;", "<")
         .replace("&gt;", ">")
         .replace("&quot;", "\"")
+
+    fun getHash(countryCode: CountryCode, platform: Platform, id: String, langType: LangType) = "${countryCode}-${platform}-$id-$langType"
 }