From d7cfcfc578a3bd657e013ed641929d0eb099759f Mon Sep 17 00:00:00 2001 From: Ziedelth Date: Thu, 7 Mar 2024 10:23:05 +0100 Subject: [PATCH 1/4] Fix indexes creation and remove external id from FetchDeprecatedEpisodeJob.kt --- .../jobs/FetchDeprecatedEpisodeJob.kt | 79 +------------------ .../db/changelog/2024/03/02-changelog.xml | 38 ++++++--- .../jobs/FetchDeprecatedEpisodeJobTest.kt | 73 ----------------- 3 files changed, 31 insertions(+), 159 deletions(-) diff --git a/src/main/kotlin/fr/shikkanime/jobs/FetchDeprecatedEpisodeJob.kt b/src/main/kotlin/fr/shikkanime/jobs/FetchDeprecatedEpisodeJob.kt index e17e510e..64bf2d23 100644 --- a/src/main/kotlin/fr/shikkanime/jobs/FetchDeprecatedEpisodeJob.kt +++ b/src/main/kotlin/fr/shikkanime/jobs/FetchDeprecatedEpisodeJob.kt @@ -2,7 +2,6 @@ package fr.shikkanime.jobs import com.google.gson.JsonObject import com.google.inject.Inject -import fr.shikkanime.caches.CountryCodeAnimeIdKeyCache import fr.shikkanime.entities.Episode import fr.shikkanime.entities.enums.ConfigPropertyKey import fr.shikkanime.entities.enums.EpisodeType @@ -10,7 +9,6 @@ import fr.shikkanime.entities.enums.Platform import fr.shikkanime.services.EpisodeService import fr.shikkanime.services.ImageService import fr.shikkanime.services.caches.ConfigCacheService -import fr.shikkanime.utils.HttpRequest import fr.shikkanime.utils.LoggerFactory import fr.shikkanime.utils.MapCache import fr.shikkanime.utils.ObjectParser.getAsInt @@ -20,9 +18,6 @@ import fr.shikkanime.wrappers.AnimationDigitalNetworkWrapper import fr.shikkanime.wrappers.CrunchyrollWrapper import fr.shikkanime.wrappers.PrimeVideoWrapper import kotlinx.coroutines.runBlocking -import java.net.URLEncoder -import java.nio.charset.StandardCharsets -import java.time.Duration import java.time.ZonedDateTime import java.util.logging.Level @@ -31,9 +26,6 @@ private const val IMAGE_NULL_ERROR = "Image is null" class FetchDeprecatedEpisodeJob : AbstractJob { private val logger = LoggerFactory.getLogger(javaClass) - var accessToken: String = "" - lateinit var cms: CrunchyrollWrapper.CMS - @Inject private lateinit var episodeService: EpisodeService @@ -63,23 +55,18 @@ class FetchDeprecatedEpisodeJob : AbstractJob { return } - val httpRequest = HttpRequest() val anonymousAccessToken = runBlocking { CrunchyrollWrapper.getAnonymousAccessToken() } - accessToken = anonymousAccessToken val cms = runBlocking { CrunchyrollWrapper.getCMS(anonymousAccessToken) } - this.cms = cms var count = 0 episodes.forEachIndexed { index, episode -> logger.info("Fetching episode description ${index + 1}/${episodes.size}") - if (update(episode, httpRequest, anonymousAccessToken, cms, now)) { + if (update(episode, anonymousAccessToken, cms, now)) { count++ } } - httpRequest.close() - if (count <= 0) { return } @@ -98,7 +85,6 @@ class FetchDeprecatedEpisodeJob : AbstractJob { fun update( episode: Episode, - httpRequest: HttpRequest, anonymousAccessToken: String, cms: CrunchyrollWrapper.CMS, now: ZonedDateTime, @@ -108,7 +94,7 @@ class FetchDeprecatedEpisodeJob : AbstractJob { try { val content = - runBlocking { normalizeContent(episode, httpRequest, anonymousAccessToken, cms) } ?: return false + runBlocking { normalizeContent(episode, anonymousAccessToken, cms) } ?: return false val title = normalizeTitle(episode.platform!!, content) val description = normalizeDescription(episode.platform!!, content) val image = normalizeImage(episode.platform!!, content) @@ -160,7 +146,6 @@ class FetchDeprecatedEpisodeJob : AbstractJob { private suspend fun normalizeContent( episode: Episode, - httpRequest: HttpRequest, accessToken: String, cms: CrunchyrollWrapper.CMS ): JsonObject? { @@ -172,8 +157,7 @@ class FetchDeprecatedEpisodeJob : AbstractJob { } Platform.CRUN -> { - val id = - getCrunchyrollEpisodeId(episode.url!!) ?: return crunchyrollExternalIdToId(httpRequest, episode) + val id = getCrunchyrollEpisodeId(episode.url!!) ?: return null CrunchyrollWrapper.getObject(episode.anime!!.countryCode!!.locale, accessToken, cms, id)[0] } @@ -191,63 +175,6 @@ class FetchDeprecatedEpisodeJob : AbstractJob { } } - private val episodesInfoCache = MapCache>(Duration.ofDays(1)) { - runBlocking { - val episodes = CrunchyrollWrapper.getSeasons(it.countryCode.locale, accessToken, cms, it.animeId) - .flatMap { season -> - CrunchyrollWrapper.getEpisodes( - it.countryCode.locale, - accessToken, - cms, - season.getAsString("id")!! - ) - } - .map { it.getAsString("id")!! } - .chunked(25) - - episodes.flatMap { chunk -> - CrunchyrollWrapper.getObject(it.countryCode.locale, accessToken, cms, *chunk.toTypedArray()) - } - } - } - - fun crunchyrollExternalIdToId( - httpRequest: HttpRequest, - episode: Episode, - ): JsonObject? { - val selector = "div[data-t=\"search-series-card\"]" - val titleSelector = "a[tabindex=\"0\"]" - - val content = try { - httpRequest.getBrowser( - "https://www.crunchyroll.com/${episode.anime!!.countryCode!!.name.lowercase()}/search?q=${ - URLEncoder.encode( - episode.anime!!.name!!, - StandardCharsets.UTF_8 - ) - }", - selector - ) - } catch (e: Exception) { - return null - } - - val seriesCard = content.select(selector) - val serieCard = seriesCard.find { it.select(titleSelector).attr("title") == episode.anime!!.name } - ?: throw Exception("Failed to find serie card for ${episode.anime!!.name}") - val seriesId = getCrunchyrollSeriesId(serieCard.select(titleSelector).attr("href")) - ?: throw Exception("Failed to find serie id for ${episode.anime!!.name}") - val allEpisodes = - episodesInfoCache[CountryCodeAnimeIdKeyCache(episode.anime!!.countryCode!!, seriesId)] ?: return null - - val externalId = "EPI.${getExternalId(episode.url!!)}" - return allEpisodes.find { it.getAsString("external_id") == externalId } - } - - private fun getExternalId(url: String) = "-([0-9]{6})".toRegex().find(url)?.groupValues?.get(1) - - private fun getCrunchyrollSeriesId(url: String) = "/series/([A-Z0-9]+)".toRegex().find(url)?.groupValues?.get(1) - fun getCrunchyrollEpisodeId(url: String) = "/watch/([A-Z0-9]+)".toRegex().find(url)?.groupValues?.get(1) fun normalizeTitle(platform: Platform, content: JsonObject): String? { diff --git a/src/main/resources/db/changelog/2024/03/02-changelog.xml b/src/main/resources/db/changelog/2024/03/02-changelog.xml index 8f429b37..f89888ed 100644 --- a/src/main/resources/db/changelog/2024/03/02-changelog.xml +++ b/src/main/resources/db/changelog/2024/03/02-changelog.xml @@ -4,12 +4,14 @@ xmlns="http://www.liquibase.org/xml/ns/dbchangelog" xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-4.26.xsd" objectQuotingStrategy="QUOTE_ONLY_RESERVED_WORDS"> - + - + + + @@ -19,7 +21,9 @@ - + + + @@ -29,7 +33,9 @@ - + + + @@ -39,7 +45,9 @@ - + + + @@ -49,7 +57,9 @@ - + + + @@ -59,7 +69,9 @@ - + + + @@ -69,7 +81,9 @@ - + + + - + + + - + + + Date: Thu, 7 Mar 2024 09:46:15 +0000 Subject: [PATCH 2/4] Bump org.glassfish.jaxb:jaxb-runtime from 4.0.4 to 4.0.5 Bumps org.glassfish.jaxb:jaxb-runtime from 4.0.4 to 4.0.5. --- updated-dependencies: - dependency-name: org.glassfish.jaxb:jaxb-runtime dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- build.gradle.kts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle.kts b/build.gradle.kts index ef4a9e86..a973a5b7 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -5,7 +5,7 @@ val kotlinVersion = "1.9.22" val ktorSwaggerUiVersion = "2.7.4" val hibernateCoreVersion = "6.4.4.Final" val ehcacheVersion = "3.10.8" -val glassfishJaxbVersion = "4.0.4" +val glassfishJaxbVersion = "4.0.5" val hibernateSearchVersion = "7.1.0.Final" val tikaVersion = "3.0.0-BETA" val postgresqlVersion = "42.7.2" From 041934b62cd76be03a102dfbeeeff121f6cff942 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 7 Mar 2024 09:46:13 +0000 Subject: [PATCH 3/4] Bump jvm from 1.9.22 to 1.9.23 Bumps [jvm](https://github.com/JetBrains/kotlin) from 1.9.22 to 1.9.23. - [Release notes](https://github.com/JetBrains/kotlin/releases) - [Changelog](https://github.com/JetBrains/kotlin/blob/v1.9.23/ChangeLog.md) - [Commits](https://github.com/JetBrains/kotlin/compare/v1.9.22...v1.9.23) --- updated-dependencies: - dependency-name: jvm dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- build.gradle.kts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle.kts b/build.gradle.kts index a973a5b7..2b32c866 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -29,7 +29,7 @@ val junitVersion = "5.10.2" val h2Version = "2.2.224" plugins { - kotlin("jvm") version "1.9.22" + kotlin("jvm") version "1.9.23" id("io.ktor.plugin") version "2.3.9" jacoco id("org.sonarqube") version "4.4.1.3373" From 2d6e252a63a20037b634b1386894e57df99462ad Mon Sep 17 00:00:00 2001 From: Ziedelth Date: Thu, 7 Mar 2024 11:07:33 +0100 Subject: [PATCH 4/4] Remove unused kotlinVersion --- build.gradle.kts | 1 - 1 file changed, 1 deletion(-) diff --git a/build.gradle.kts b/build.gradle.kts index 2b32c866..187ecc4e 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -1,7 +1,6 @@ import java.net.URI val ktorVersion = "2.3.9" -val kotlinVersion = "1.9.22" val ktorSwaggerUiVersion = "2.7.4" val hibernateCoreVersion = "6.4.4.Final" val ehcacheVersion = "3.10.8"