From e9b652c50872c7827133294bcf89c29fa6b5ae15 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 24 Apr 2024 08:25:43 +0000 Subject: [PATCH 1/2] Bump hibernateCoreVersion from 6.4.4.Final to 6.4.5.Final Bumps `hibernateCoreVersion` from 6.4.4.Final to 6.4.5.Final. Updates `org.hibernate.orm:hibernate-core` from 6.4.4.Final to 6.4.5.Final - [Release notes](https://github.com/hibernate/hibernate-orm/releases) - [Changelog](https://github.com/hibernate/hibernate-orm/blob/6.4.5/changelog.txt) - [Commits](https://github.com/hibernate/hibernate-orm/compare/6.4.4...6.4.5) Updates `org.hibernate.orm:hibernate-jcache` from 6.4.4.Final to 6.4.5.Final - [Release notes](https://github.com/hibernate/hibernate-orm/releases) - [Changelog](https://github.com/hibernate/hibernate-orm/blob/6.4.5/changelog.txt) - [Commits](https://github.com/hibernate/hibernate-orm/compare/6.4.4...6.4.5) Updates `org.hibernate.orm:hibernate-jpamodelgen` from 6.4.4.Final to 6.4.5.Final - [Release notes](https://github.com/hibernate/hibernate-orm/releases) - [Changelog](https://github.com/hibernate/hibernate-orm/blob/6.4.5/changelog.txt) - [Commits](https://github.com/hibernate/hibernate-orm/compare/6.4.4...6.4.5) --- updated-dependencies: - dependency-name: org.hibernate.orm:hibernate-core dependency-type: direct:production update-type: version-update:semver-patch - dependency-name: org.hibernate.orm:hibernate-jcache dependency-type: direct:production update-type: version-update:semver-patch - dependency-name: org.hibernate.orm:hibernate-jpamodelgen dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] --- build.gradle.kts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.gradle.kts b/build.gradle.kts index 18b071f1..4d99a8f1 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -1,6 +1,6 @@ val ktorVersion = "2.3.10" val ktorSwaggerUiVersion = "2.8.0" -val hibernateCoreVersion = "6.4.4.Final" +val hibernateCoreVersion = "6.4.5.Final" val ehcacheVersion = "3.10.8" val glassfishJaxbVersion = "4.0.5" val hibernateSearchVersion = "7.1.1.Final" From 4f890cc2cf56a9e39b16f31527cdc8d3a075511d Mon Sep 17 00:00:00 2001 From: Ziedelth Date: Mon, 22 Apr 2024 16:44:42 +0200 Subject: [PATCH 2/2] Add job to fetch old episodes --- src/main/kotlin/fr/shikkanime/Application.kt | 1 + ...IdKeyCache.kt => CountryCodeIdKeyCache.kt} | 4 +- .../controllers/admin/AdminController.kt | 25 +- .../controllers/site/SiteController.kt | 3 +- .../kotlin/fr/shikkanime/entities/Anime.kt | 4 +- .../entities/enums/ConfigPropertyKey.kt | 2 + .../fr/shikkanime/jobs/FetchOldEpisodesJob.kt | 238 ++++++++++++++++++ .../AnimationDigitalNetworkPlatform.kt | 27 +- .../platforms/CrunchyrollPlatform.kt | 16 +- .../repositories/AnimeRepository.kt | 10 +- .../repositories/EpisodeVariantRepository.kt | 14 ++ .../fr/shikkanime/services/AnimeService.kt | 22 +- .../services/EpisodeVariantService.kt | 4 +- .../services/caches/AnimeCacheService.kt | 7 +- .../db/changelog/2024/04/08-changelog.xml | 66 +++++ .../db/changelog/db.changelog-master.xml | 1 + .../kotlin/fr/shikkanime/OldEpisodeScraper.kt | 168 ------------- .../jobs/FetchOldEpisodesJobTest.kt | 63 +++++ .../platforms/PrimeVideoPlatformTest.kt | 2 - 19 files changed, 451 insertions(+), 226 deletions(-) rename src/main/kotlin/fr/shikkanime/caches/{CountryCodeAnimeIdKeyCache.kt => CountryCodeIdKeyCache.kt} (64%) create mode 100644 src/main/kotlin/fr/shikkanime/jobs/FetchOldEpisodesJob.kt create mode 100644 src/main/resources/db/changelog/2024/04/08-changelog.xml delete mode 100644 src/test/kotlin/fr/shikkanime/OldEpisodeScraper.kt create mode 100644 src/test/kotlin/fr/shikkanime/jobs/FetchOldEpisodesJobTest.kt diff --git a/src/main/kotlin/fr/shikkanime/Application.kt b/src/main/kotlin/fr/shikkanime/Application.kt index dcac35ec..79eb0bc2 100644 --- a/src/main/kotlin/fr/shikkanime/Application.kt +++ b/src/main/kotlin/fr/shikkanime/Application.kt @@ -50,6 +50,7 @@ fun initAll(adminPassword: AtomicReference?, port: Int = 37100, wait: Bo JobManager.scheduleJob("0 0 * * * ?", SavingImageCacheJob::class.java) // Every day at midnight JobManager.scheduleJob("0 0 0 * * ?", DeleteOldMetricsJob::class.java) + JobManager.scheduleJob("0 0 15 * * ?", FetchOldEpisodesJob::class.java) // Every day at 9am JobManager.scheduleJob("0 0 9 * * ?", FetchCalendarJob::class.java) JobManager.start() diff --git a/src/main/kotlin/fr/shikkanime/caches/CountryCodeAnimeIdKeyCache.kt b/src/main/kotlin/fr/shikkanime/caches/CountryCodeIdKeyCache.kt similarity index 64% rename from src/main/kotlin/fr/shikkanime/caches/CountryCodeAnimeIdKeyCache.kt rename to src/main/kotlin/fr/shikkanime/caches/CountryCodeIdKeyCache.kt index 0de51604..9f73b834 100644 --- a/src/main/kotlin/fr/shikkanime/caches/CountryCodeAnimeIdKeyCache.kt +++ b/src/main/kotlin/fr/shikkanime/caches/CountryCodeIdKeyCache.kt @@ -2,7 +2,7 @@ package fr.shikkanime.caches import fr.shikkanime.entities.enums.CountryCode -data class CountryCodeAnimeIdKeyCache( +data class CountryCodeIdKeyCache( val countryCode: CountryCode, - val animeId: String, + val id: String, ) diff --git a/src/main/kotlin/fr/shikkanime/controllers/admin/AdminController.kt b/src/main/kotlin/fr/shikkanime/controllers/admin/AdminController.kt index 5eed64d3..d06375a2 100644 --- a/src/main/kotlin/fr/shikkanime/controllers/admin/AdminController.kt +++ b/src/main/kotlin/fr/shikkanime/controllers/admin/AdminController.kt @@ -3,15 +3,9 @@ package fr.shikkanime.controllers.admin import com.google.inject.Inject import fr.shikkanime.converters.AbstractConverter import fr.shikkanime.dtos.TokenDto -import fr.shikkanime.entities.Anime -import fr.shikkanime.entities.EpisodeMapping -import fr.shikkanime.entities.Simulcast -import fr.shikkanime.entities.enums.CountryCode -import fr.shikkanime.entities.enums.EpisodeType import fr.shikkanime.entities.enums.Link import fr.shikkanime.services.* import fr.shikkanime.services.caches.SimulcastCacheService -import fr.shikkanime.utils.MapCache import fr.shikkanime.utils.routes.AdminSessionAuthenticated import fr.shikkanime.utils.routes.Controller import fr.shikkanime.utils.routes.Path @@ -114,24 +108,7 @@ class AdminController { @Get @AdminSessionAuthenticated private fun invalidateSimulcasts(): Response { - animeService.findAll().forEach { anime -> - anime.simulcasts.clear() - animeService.update(anime) - } - - episodeMappingService.findAll() - .filter { it.variants.any { variant -> variant.audioLocale != CountryCode.FR.locale } && it.episodeType != EpisodeType.FILM } - .sortedBy { it.releaseDateTime } - .forEach { episodeMapping -> - val anime = animeService.find(episodeMapping.anime!!.uuid!!)!! - animeService.addSimulcastToAnime(anime, episodeVariantService.getSimulcast(anime, episodeMapping)) - - if (episodeMapping.anime != anime) { - animeService.update(anime) - } - } - - MapCache.invalidate(Anime::class.java, EpisodeMapping::class.java, Simulcast::class.java) + animeService.recalculateSimulcasts() return Response.redirect(Link.DASHBOARD.href) } diff --git a/src/main/kotlin/fr/shikkanime/controllers/site/SiteController.kt b/src/main/kotlin/fr/shikkanime/controllers/site/SiteController.kt index 46bc5ef0..a49570dc 100644 --- a/src/main/kotlin/fr/shikkanime/controllers/site/SiteController.kt +++ b/src/main/kotlin/fr/shikkanime/controllers/site/SiteController.kt @@ -123,7 +123,7 @@ class SiteController { @Path("animes/{slug}") @Get private fun animeDetail(@PathParam("slug") slug: String): Response { - val anime = animeCacheService.findBySlug(slug) ?: return Response.redirect("/404") + val anime = animeCacheService.findBySlug(CountryCode.FR, slug) ?: return Response.redirect("/404") val dto = AbstractConverter.convert(anime, AnimeDto::class.java) return Response.template( @@ -136,6 +136,7 @@ class SiteController { CountryCode.FR, anime.uuid, listOf( + SortParameter("releaseDateTime", SortParameter.Order.ASC), SortParameter("season", SortParameter.Order.ASC), SortParameter("episodeType", SortParameter.Order.ASC), SortParameter("number", SortParameter.Order.ASC), diff --git a/src/main/kotlin/fr/shikkanime/entities/Anime.kt b/src/main/kotlin/fr/shikkanime/entities/Anime.kt index ab49525c..c9c99bae 100644 --- a/src/main/kotlin/fr/shikkanime/entities/Anime.kt +++ b/src/main/kotlin/fr/shikkanime/entities/Anime.kt @@ -15,7 +15,7 @@ import java.util.* name = "anime", indexes = [ Index(name = "idx_anime_country_code", columnList = "country_code"), - Index(name = "idx_anime_slug", columnList = "slug"), + Index(name = "idx_anime_country_code_slug", columnList = "country_code, slug", unique = true) ] ) @Indexed @@ -45,7 +45,7 @@ class Anime( inverseJoinColumns = [JoinColumn(name = "simulcast_uuid")] ) var simulcasts: MutableSet = mutableSetOf(), - @Column(nullable = false, unique = true) + @Column(nullable = false) var slug: String? = null, @Column(nullable = false, name = "last_release_date_time") var lastReleaseDateTime: ZonedDateTime = releaseDateTime, diff --git a/src/main/kotlin/fr/shikkanime/entities/enums/ConfigPropertyKey.kt b/src/main/kotlin/fr/shikkanime/entities/enums/ConfigPropertyKey.kt index aee42396..157c2216 100644 --- a/src/main/kotlin/fr/shikkanime/entities/enums/ConfigPropertyKey.kt +++ b/src/main/kotlin/fr/shikkanime/entities/enums/ConfigPropertyKey.kt @@ -30,4 +30,6 @@ enum class ConfigPropertyKey(val key: String) { DISNEY_PLUS_AUTHORIZATION("disney_plus_authorization"), DISNEY_PLUS_REFRESH_TOKEN("disney_plus_refresh_token"), TRANSLATE_CALENDAR("translate_calendar"), + LAST_FETCH_OLD_EPISODES("last_fetch_old_episodes"), + FETCH_OLD_EPISODES_RANGE("fetch_old_episodes_range"), } \ No newline at end of file diff --git a/src/main/kotlin/fr/shikkanime/jobs/FetchOldEpisodesJob.kt b/src/main/kotlin/fr/shikkanime/jobs/FetchOldEpisodesJob.kt new file mode 100644 index 00000000..27cde4b1 --- /dev/null +++ b/src/main/kotlin/fr/shikkanime/jobs/FetchOldEpisodesJob.kt @@ -0,0 +1,238 @@ +package fr.shikkanime.jobs + +import com.google.inject.Inject +import fr.shikkanime.caches.CountryCodeIdKeyCache +import fr.shikkanime.entities.Anime +import fr.shikkanime.entities.EpisodeMapping +import fr.shikkanime.entities.EpisodeVariant +import fr.shikkanime.entities.Simulcast +import fr.shikkanime.entities.enums.ConfigPropertyKey +import fr.shikkanime.entities.enums.CountryCode +import fr.shikkanime.platforms.AbstractPlatform +import fr.shikkanime.platforms.AnimationDigitalNetworkPlatform +import fr.shikkanime.platforms.CrunchyrollPlatform +import fr.shikkanime.services.AnimeService +import fr.shikkanime.services.ConfigService +import fr.shikkanime.services.EpisodeMappingService +import fr.shikkanime.services.EpisodeVariantService +import fr.shikkanime.services.caches.ConfigCacheService +import fr.shikkanime.utils.Constant +import fr.shikkanime.utils.LoggerFactory +import fr.shikkanime.utils.MapCache +import fr.shikkanime.utils.ObjectParser.getAsInt +import fr.shikkanime.utils.ObjectParser.getAsString +import fr.shikkanime.utils.StringUtils +import fr.shikkanime.wrappers.CrunchyrollWrapper +import kotlinx.coroutines.runBlocking +import java.time.LocalDate +import java.time.Period +import java.util.logging.Level + +class FetchOldEpisodesJob : AbstractJob { + private val logger = LoggerFactory.getLogger(javaClass) + + @Inject + private lateinit var animeService: AnimeService + + @Inject + private lateinit var episodeMappingService: EpisodeMappingService + + @Inject + private lateinit var episodeVariantService: EpisodeVariantService + + @Inject + private lateinit var animationDigitalNetworkPlatform: AnimationDigitalNetworkPlatform + + @Inject + private lateinit var crunchyrollPlatform: CrunchyrollPlatform + + @Inject + private lateinit var configService: ConfigService + + @Inject + private lateinit var episodeCacheService: ConfigCacheService + + override fun run() { + val config = configService.findByName(ConfigPropertyKey.LAST_FETCH_OLD_EPISODES.key) ?: run { + logger.warning("Config ${ConfigPropertyKey.LAST_FETCH_OLD_EPISODES.key} not found") + return + } + + val range = episodeCacheService.getValueAsInt(ConfigPropertyKey.FETCH_OLD_EPISODES_RANGE) + + if (range == -1) { + logger.warning("Config ${ConfigPropertyKey.FETCH_OLD_EPISODES_RANGE.key} not found") + return + } + + val to = LocalDate.parse(config.propertyValue!!) + val from = to.minusDays(range.toLong()) + val dates = from.datesUntil(to.plusDays(1), Period.ofDays(1)).toList().sorted() + val simulcasts = dates.map { "${Constant.seasons[(it.monthValue - 1) / 3]}-${it.year}".lowercase().replace("autumn", "fall") }.toSet() + val episodes = mutableListOf() + val start = System.currentTimeMillis() + logger.info("Fetching old episodes... (From ${dates.first()} to ${dates.last()})") + + episodes.addAll(fetchAnimationDigitalNetwork(CountryCode.FR, dates)) + episodes.addAll(fetchCrunchyroll(CountryCode.FR, simulcasts)) + + episodes.removeIf { it.releaseDateTime.toLocalDate() !in dates } + + episodes.groupBy { it.anime + it.releaseDateTime.toLocalDate().toString() }.forEach { (_, animeDayEpisodes) -> + if (animeDayEpisodes.size > 5) { + logger.warning("More than 5 episodes for ${animeDayEpisodes.first().anime} on ${animeDayEpisodes.first().releaseDateTime.toLocalDate()}, removing...") + episodes.removeAll(animeDayEpisodes) + return@forEach + } + } + + logger.info("Found ${episodes.size} episodes, saving...") + var realSaved = 0 + + val variants = episodes.sortedBy { it.releaseDateTime }.map { episode -> + episodeVariantService.findByIdentifier(episode.getIdentifier()) ?: run { + realSaved++ + episodeVariantService.save(episode) + } + } + + logger.info("Saved $realSaved episodes") + logger.info("Updating mappings...") + + variants.groupBy { it.mapping!!.uuid }.forEach { (mappingUuid, variants) -> + val mapping = episodeMappingService.find(mappingUuid) ?: return@forEach + mapping.releaseDateTime = variants.minOf { it.releaseDateTime } + mapping.lastReleaseDateTime = variants.maxOf { it.releaseDateTime } + episodeMappingService.update(mapping) + } + + logger.info("Updating animes...") + + variants.groupBy { it.mapping!!.anime!!.uuid }.forEach { (animeUuid, variants) -> + val anime = animeService.find(animeUuid) ?: return@forEach + logger.info("Updating ${StringUtils.getShortName(anime.name!!)}...") + anime.releaseDateTime = variants.minOf { it.releaseDateTime } + anime.lastReleaseDateTime = variants.maxOf { it.releaseDateTime } + animeService.update(anime) + } + + logger.info("Updating simulcasts...") + animeService.recalculateSimulcasts() + MapCache.invalidate(Anime::class.java, EpisodeMapping::class.java, EpisodeVariant::class.java, Simulcast::class.java) + logger.info("Updating config to the next fetch date...") + config.propertyValue = from.plusDays(1).toString() + configService.update(config) + logger.info("Take ${(System.currentTimeMillis() - start) / 1000}s to check ${dates.size} dates") + } + + private fun fetchAnimationDigitalNetwork(countryCode: CountryCode, dates: List): List { + val episodes = mutableListOf() + + dates.forEachIndexed { _, date -> + val zonedDateTime = date.atStartOfDay(Constant.utcZoneId) + + runBlocking { + animationDigitalNetworkPlatform.fetchApiContent( + countryCode, + zonedDateTime + ) + }.forEach { episodeJson -> + try { + episodes.addAll( + animationDigitalNetworkPlatform.convertEpisode( + countryCode, + episodeJson.asJsonObject, + zonedDateTime, + false + ) + ) + } catch (e: Exception) { + logger.log(Level.SEVERE, "Error while converting episode (Episode ID: ${episodeJson.getAsString("id")})", e) + } + } + } + + return episodes + } + + private fun fetchCrunchyroll(countryCode: CountryCode, simulcasts: Set): List { + val accessToken = runBlocking { CrunchyrollWrapper.getAnonymousAccessToken() } + val cms = runBlocking { CrunchyrollWrapper.getCMS(accessToken) } + val episodes = mutableListOf() + + val series = simulcasts.flatMap { simulcastId -> + runBlocking { + CrunchyrollWrapper.getBrowse( + countryCode.locale, + accessToken, + sortBy = CrunchyrollWrapper.SortType.POPULARITY, + type = CrunchyrollWrapper.MediaType.SERIES, + 100, + simulcast = simulcastId + ) + } + } + + val titles = series.map { jsonObject -> jsonObject.getAsString("title")!!.lowercase() }.toSet() + val ids = series.map { jsonObject -> jsonObject.getAsString("id")!! }.toSet() + + crunchyrollPlatform.simulcasts[CountryCode.FR] = titles + + series.forEach { + val postersTall = it.getAsJsonObject("images").getAsJsonArray("poster_tall")[0].asJsonArray + val postersWide = it.getAsJsonObject("images").getAsJsonArray("poster_wide")[0].asJsonArray + val image = postersTall?.maxByOrNull { poster -> poster.asJsonObject.getAsInt("width")!! }?.asJsonObject?.getAsString("source")!! + val banner = postersWide?.maxByOrNull { poster -> poster.asJsonObject.getAsInt("width")!! }?.asJsonObject?.getAsString("source")!! + val description = it.getAsString("description") + + crunchyrollPlatform.animeInfoCache[CountryCodeIdKeyCache(countryCode, it.getAsString("id")!!)] = + CrunchyrollPlatform.CrunchyrollAnimeContent(image = image, banner = banner, description = description) + } + + val episodeIds = ids.parallelStream().map { seriesId -> + runBlocking { CrunchyrollWrapper.getSeasons(countryCode.locale, accessToken, cms, seriesId) } + .filter { jsonObject -> + jsonObject.getAsJsonArray("subtitle_locales").map { it.asString }.contains(countryCode.locale) + } + .map { jsonObject -> jsonObject.getAsString("id")!! } + .flatMap { id -> + runBlocking { + CrunchyrollWrapper.getEpisodes( + countryCode.locale, + accessToken, + cms, + id + ) + } + } + .map { jsonObject -> jsonObject.getAsString("id")!! } + }.toList().flatten().toSet() + + episodeIds.chunked(25).parallelStream().forEach { episodeIdsChunked -> + val `object` = runBlocking { + CrunchyrollWrapper.getObject( + countryCode.locale, + accessToken, + cms, + *episodeIdsChunked.toTypedArray() + ) + } + + `object`.forEach { episodeJson -> + try { + episodes.add( + crunchyrollPlatform.convertEpisode( + countryCode, + episodeJson, + false, + ) + ) + } catch (e: Exception) { + logger.log(Level.SEVERE, "Error while converting episode (Episode ID: ${episodeJson.getAsString("id")})", e) + } + } + } + + return episodes + } +} \ No newline at end of file diff --git a/src/main/kotlin/fr/shikkanime/platforms/AnimationDigitalNetworkPlatform.kt b/src/main/kotlin/fr/shikkanime/platforms/AnimationDigitalNetworkPlatform.kt index 9b67eb00..d5c5522f 100644 --- a/src/main/kotlin/fr/shikkanime/platforms/AnimationDigitalNetworkPlatform.kt +++ b/src/main/kotlin/fr/shikkanime/platforms/AnimationDigitalNetworkPlatform.kt @@ -54,10 +54,11 @@ class AnimationDigitalNetworkPlatform : return list } - private fun convertEpisode( + fun convertEpisode( countryCode: CountryCode, jsonObject: JsonObject, - zonedDateTime: ZonedDateTime + zonedDateTime: ZonedDateTime, + needSimulcast: Boolean = true ): List { val show = requireNotNull(jsonObject.getAsJsonObject("show")) { "Show is null" } val season = jsonObject.getAsString("season")?.toIntOrNull() ?: 1 @@ -83,19 +84,21 @@ class AnimationDigitalNetworkPlatform : ) }) && !contains) throw Exception("Anime is not an animation") - var isSimulcasted = show.getAsBoolean("simulcast") == true || - show.getAsString("firstReleaseYear") in (0..1).map { (zonedDateTime.year - it).toString() } || - contains + if (needSimulcast) { + var isSimulcasted = show.getAsBoolean("simulcast") == true || + show.getAsString("firstReleaseYear") in (0..1).map { (zonedDateTime.year - it).toString() } || + contains - val descriptionLowercase = animeDescription.lowercase() + val descriptionLowercase = animeDescription.lowercase() - isSimulcasted = isSimulcasted || - configCacheService.getValueAsString(ConfigPropertyKey.ANIMATION_DITIGAL_NETWORK_SIMULCAST_DETECTION_REGEX) - ?.let { - Regex(it).containsMatchIn(descriptionLowercase) - } == true + isSimulcasted = isSimulcasted || + configCacheService.getValueAsString(ConfigPropertyKey.ANIMATION_DITIGAL_NETWORK_SIMULCAST_DETECTION_REGEX) + ?.let { + Regex(it).containsMatchIn(descriptionLowercase) + } == true - if (!isSimulcasted) throw AnimeNotSimulcastedException("Anime is not simulcasted") + if (!isSimulcasted) throw AnimeNotSimulcastedException("Anime is not simulcasted") + } val releaseDateString = requireNotNull(jsonObject.getAsString("releaseDate")) { "Release date is null" } val releaseDate = ZonedDateTime.parse(releaseDateString) diff --git a/src/main/kotlin/fr/shikkanime/platforms/CrunchyrollPlatform.kt b/src/main/kotlin/fr/shikkanime/platforms/CrunchyrollPlatform.kt index 54558429..034978b9 100644 --- a/src/main/kotlin/fr/shikkanime/platforms/CrunchyrollPlatform.kt +++ b/src/main/kotlin/fr/shikkanime/platforms/CrunchyrollPlatform.kt @@ -2,7 +2,7 @@ package fr.shikkanime.platforms import com.google.gson.JsonObject import com.google.inject.Inject -import fr.shikkanime.caches.CountryCodeAnimeIdKeyCache +import fr.shikkanime.caches.CountryCodeIdKeyCache import fr.shikkanime.entities.enums.ConfigPropertyKey import fr.shikkanime.entities.enums.CountryCode import fr.shikkanime.entities.enums.EpisodeType @@ -64,14 +64,14 @@ class CrunchyrollPlatform : AbstractPlatform(Duration.ofDays(1)) { + val animeInfoCache = MapCache(Duration.ofDays(1)) { val (token, cms) = identifiers[it.countryCode]!! val `object` = runBlocking { CrunchyrollWrapper.getObject( it.countryCode.locale, token, cms, - it.animeId + it.id ) }[0] val postersTall = `object`.getAsJsonObject("images").getAsJsonArray("poster_tall")[0].asJsonArray @@ -137,7 +137,7 @@ class CrunchyrollPlatform : AbstractPlatform() { return bool } - fun findBySlug(slug: String): Anime? { + fun findBySlug(countryCode: CountryCode, slug: String): Anime? { return inTransaction { entityManager -> val cb = entityManager.criteriaBuilder val query = cb.createQuery(getEntityClass()) val root = query.from(getEntityClass()) - query.where(cb.equal(root[Anime_.slug], slug)) + + query.where( + cb.and( + cb.equal(root[Anime_.countryCode], countryCode), + cb.equal(root[Anime_.slug], slug) + ) + ) createReadOnlyQuery(entityManager, query) .resultList diff --git a/src/main/kotlin/fr/shikkanime/repositories/EpisodeVariantRepository.kt b/src/main/kotlin/fr/shikkanime/repositories/EpisodeVariantRepository.kt index 421eef74..620e6021 100644 --- a/src/main/kotlin/fr/shikkanime/repositories/EpisodeVariantRepository.kt +++ b/src/main/kotlin/fr/shikkanime/repositories/EpisodeVariantRepository.kt @@ -30,4 +30,18 @@ class EpisodeVariantRepository : AbstractRepository() { .resultList } } + + fun findByIdentifier(identifier: String): EpisodeVariant? { + return inTransaction { entityManager -> + val cb = entityManager.criteriaBuilder + val query = cb.createQuery(getEntityClass()) + val root = query.from(getEntityClass()) + + query.where(cb.equal(root[EpisodeVariant_.identifier], identifier)) + + createReadOnlyQuery(entityManager, query) + .resultList + .firstOrNull() + } + } } \ No newline at end of file diff --git a/src/main/kotlin/fr/shikkanime/services/AnimeService.kt b/src/main/kotlin/fr/shikkanime/services/AnimeService.kt index 06ee2896..011cc3e0 100644 --- a/src/main/kotlin/fr/shikkanime/services/AnimeService.kt +++ b/src/main/kotlin/fr/shikkanime/services/AnimeService.kt @@ -12,6 +12,7 @@ import fr.shikkanime.entities.EpisodeVariant import fr.shikkanime.entities.Simulcast import fr.shikkanime.entities.SortParameter import fr.shikkanime.entities.enums.CountryCode +import fr.shikkanime.entities.enums.EpisodeType import fr.shikkanime.entities.enums.LangType import fr.shikkanime.repositories.AnimeRepository import fr.shikkanime.utils.MapCache @@ -56,7 +57,7 @@ class AnimeService : AbstractService() { fun findAllByName(name: String, countryCode: CountryCode?, page: Int, limit: Int) = animeRepository.findAllByName(name, countryCode, page, limit) - fun findBySlug(slug: String) = animeRepository.findBySlug(slug) + fun findBySlug(countryCode: CountryCode, slug: String) = animeRepository.findBySlug(countryCode, slug) fun getWeeklyAnimes(startOfWeekDay: LocalDate, countryCode: CountryCode): List { val zoneId = ZoneId.of(countryCode.timezone) @@ -115,6 +116,25 @@ class AnimeService : AbstractService() { } } + fun recalculateSimulcasts() { + findAll().forEach { anime -> + anime.simulcasts.clear() + update(anime) + } + + episodeMappingService.findAll() + .filter { it.variants.any { variant -> variant.audioLocale != CountryCode.FR.locale } && it.episodeType != EpisodeType.FILM } + .sortedBy { it.releaseDateTime } + .forEach { episodeMapping -> + val anime = find(episodeMapping.anime!!.uuid!!)!! + addSimulcastToAnime(anime, episodeVariantService.getSimulcast(anime, episodeMapping)) + + if (episodeMapping.anime != anime) { + update(anime) + } + } + } + override fun save(entity: Anime): Anime { entity.simulcasts = entity.simulcasts.map { simulcast -> simulcastService.findBySeasonAndYear(simulcast.season!!, simulcast.year!!) ?: simulcastService.save( diff --git a/src/main/kotlin/fr/shikkanime/services/EpisodeVariantService.kt b/src/main/kotlin/fr/shikkanime/services/EpisodeVariantService.kt index cf980069..de180913 100644 --- a/src/main/kotlin/fr/shikkanime/services/EpisodeVariantService.kt +++ b/src/main/kotlin/fr/shikkanime/services/EpisodeVariantService.kt @@ -41,6 +41,8 @@ class EpisodeVariantService : AbstractService(classes = listOf(Anime::class.java)) { - animeService.findBySlug(it) + private val findBySlugCache = MapCache(classes = listOf(Anime::class.java)) { + animeService.findBySlug(it.countryCode, it.id) } private val weeklyCache = @@ -70,7 +71,7 @@ class AnimeCacheService : AbstractCacheService { fun findAllByName(name: String, countryCode: CountryCode?, page: Int, limit: Int) = findAllByNameCache[CountryCodeNamePaginationKeyCache(countryCode, name, page, limit)] - fun findBySlug(slug: String) = findBySlugCache[slug] + fun findBySlug(countryCode: CountryCode, slug: String) = findBySlugCache[CountryCodeIdKeyCache(countryCode, slug)] fun getWeeklyAnimes(startOfWeekDay: LocalDate, countryCode: CountryCode) = weeklyCache[CountryCodeLocalDateKeyCache(countryCode, startOfWeekDay)] diff --git a/src/main/resources/db/changelog/2024/04/08-changelog.xml b/src/main/resources/db/changelog/2024/04/08-changelog.xml new file mode 100644 index 00000000..5b6ecb1a --- /dev/null +++ b/src/main/resources/db/changelog/2024/04/08-changelog.xml @@ -0,0 +1,66 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + SELECT COUNT(*) + FROM config + WHERE property_key = 'last_fetch_old_episodes' + + + + + + + + + + + + SELECT COUNT(*) + FROM config + WHERE property_key = 'fetch_old_episodes_range' + + + + + + + + + \ No newline at end of file diff --git a/src/main/resources/db/changelog/db.changelog-master.xml b/src/main/resources/db/changelog/db.changelog-master.xml index bbd1edd7..7ff5294f 100644 --- a/src/main/resources/db/changelog/db.changelog-master.xml +++ b/src/main/resources/db/changelog/db.changelog-master.xml @@ -46,4 +46,5 @@ + \ No newline at end of file diff --git a/src/test/kotlin/fr/shikkanime/OldEpisodeScraper.kt b/src/test/kotlin/fr/shikkanime/OldEpisodeScraper.kt deleted file mode 100644 index e45c0641..00000000 --- a/src/test/kotlin/fr/shikkanime/OldEpisodeScraper.kt +++ /dev/null @@ -1,168 +0,0 @@ -package fr.shikkanime - -import fr.shikkanime.caches.CountryCodeAnimeIdKeyCache -import fr.shikkanime.entities.enums.CountryCode -import fr.shikkanime.platforms.AnimationDigitalNetworkPlatform -import fr.shikkanime.platforms.CrunchyrollPlatform -import fr.shikkanime.utils.Constant -import fr.shikkanime.utils.HttpRequest -import fr.shikkanime.utils.ObjectParser.getAsInt -import fr.shikkanime.utils.ObjectParser.getAsString -import fr.shikkanime.wrappers.CrunchyrollWrapper -import kotlinx.coroutines.runBlocking -import java.time.LocalDate -import java.time.Period -import java.time.format.DateTimeFormatter -import java.time.temporal.ChronoUnit - -private const val DD_MM_YYYY = "dd/MM/yyyy" -private val ofPattern = DateTimeFormatter.ofPattern(DD_MM_YYYY) - -fun main() { - val httpRequest = HttpRequest() - - println("Enter the from date you want to check ($DD_MM_YYYY):") - val checkFromDate = readlnOrNull() ?: return - val start = System.currentTimeMillis() - - val fromDate = try { - LocalDate.parse(checkFromDate, ofPattern) - } catch (e: Exception) { - println("Invalid date (${e.message})") - return - } - - println("Enter the to date you want to check ($DD_MM_YYYY):") - val checkToDate = readlnOrNull() ?: return - - val toDate = try { - LocalDate.parse(checkToDate, ofPattern) - } catch (e: Exception) { - println("Invalid date (${e.message})") - return - } - - if (ChronoUnit.DAYS.between(fromDate, toDate).toInt() <= 0) { - println("Invalid date range") - return - } - - val dates = fromDate.datesUntil(toDate.plusDays(1), Period.ofDays(1)).toList().sorted() - - val simulcasts = dates.map { - "${Constant.seasons[(it.monthValue - 1) / 3]}-${it.year}".lowercase().replace("autumn", "fall") - }.toSet() - - println("Checking ${dates.size} dates...") - println("Simulcasts: $simulcasts") - - val adnPlatform = Constant.injector.getInstance(AnimationDigitalNetworkPlatform::class.java) - val crunchyrollPlatform = Constant.injector.getInstance(CrunchyrollPlatform::class.java) - -// dates.forEach { date -> -// runBlocking { adnPlatform.fetchApiContent(CountryCode.FR, date) }.forEach { episodeJson -> -// try { -// episodes.addAll(adnPlatform.convertEpisode( -// CountryCode.FR, -// episodeJson.asJsonObject, -// date -// )) -// } catch (e: Exception) { -// e.printStackTrace() -// } -// } -// } - - val accessToken = runBlocking { CrunchyrollWrapper.getAnonymousAccessToken() } - val cms = runBlocking { CrunchyrollWrapper.getCMS(accessToken) } - - val series = simulcasts.flatMap { simulcastId -> - runBlocking { - CrunchyrollWrapper.getBrowse( - CountryCode.FR.locale, - accessToken, - sortBy = CrunchyrollWrapper.SortType.POPULARITY, - type = CrunchyrollWrapper.MediaType.SERIES, - 100, - simulcast = simulcastId - ) - } - } - - val titles = series.map { jsonObject -> jsonObject.getAsString("title")!!.lowercase() }.toSet() - val ids = series.map { jsonObject -> jsonObject.getAsString("id")!! }.toSet() - println("Simulcasts: $titles") - - crunchyrollPlatform.simulcasts.set(CountryCode.FR, titles) - - series.forEach { - val postersTall = it.getAsJsonObject("images").getAsJsonArray("poster_tall")[0].asJsonArray - val postersWide = it.getAsJsonObject("images").getAsJsonArray("poster_wide")[0].asJsonArray - val image = - postersTall?.maxByOrNull { poster -> poster.asJsonObject.getAsInt("width")!! }?.asJsonObject?.getAsString("source")!! - val banner = - postersWide?.maxByOrNull { poster -> poster.asJsonObject.getAsInt("width")!! }?.asJsonObject?.getAsString("source")!! - val description = it.getAsString("description") - - crunchyrollPlatform.animeInfoCache.set( - CountryCodeAnimeIdKeyCache(CountryCode.FR, it.getAsString("id")!!), - CrunchyrollPlatform.CrunchyrollAnimeContent(image = image, banner = banner, description = description) - ) - } - - val episodeIds = ids.parallelStream().map { seriesId -> - runBlocking { CrunchyrollWrapper.getSeasons(CountryCode.FR.locale, accessToken, cms, seriesId) } - .filter { jsonObject -> - jsonObject.getAsJsonArray("subtitle_locales").map { it.asString }.contains(CountryCode.FR.locale) - } - .map { jsonObject -> jsonObject.getAsString("id")!! } - .flatMap { id -> - runBlocking { - CrunchyrollWrapper.getEpisodes( - CountryCode.FR.locale, - accessToken, - cms, - id - ) - } - } - .map { jsonObject -> jsonObject.getAsString("id")!! } - }.toList().flatten().toSet() - - episodeIds.chunked(25).parallelStream().forEach { episodeIdsChunked -> - val `object` = runBlocking { - CrunchyrollWrapper.getObject( - CountryCode.FR.locale, - accessToken, - cms, - *episodeIdsChunked.toTypedArray() - ) - } - - `object`.forEach { episodeJson -> - try { -// episodes.add( -// crunchyrollPlatform.convertJsonEpisode( -// CountryCode.FR, -// episodeJson, -// ) -// ) - } catch (e: Exception) { - println("Error while converting episode (Episode ID: ${episodeJson.getAsString("id")}): ${e.message}") - e.printStackTrace() - } - } - } - - httpRequest.close() - -// episodes.removeIf { it.releaseDateTime.toLocalDate() !in dates } -// -// episodes.sortedBy { it.releaseDateTime }.forEach { episode -> -// episode.anime?.releaseDateTime = -// episodes.filter { it.anime?.name == episode.anime?.name }.minOf { it.anime!!.releaseDateTime } -// episodeService.save(episode) -// } - - println("Take ${(System.currentTimeMillis() - start) / 1000}s to check ${dates.size} dates") -} \ No newline at end of file diff --git a/src/test/kotlin/fr/shikkanime/jobs/FetchOldEpisodesJobTest.kt b/src/test/kotlin/fr/shikkanime/jobs/FetchOldEpisodesJobTest.kt new file mode 100644 index 00000000..7c824926 --- /dev/null +++ b/src/test/kotlin/fr/shikkanime/jobs/FetchOldEpisodesJobTest.kt @@ -0,0 +1,63 @@ +package fr.shikkanime.jobs + +import com.google.inject.Inject +import fr.shikkanime.entities.Config +import fr.shikkanime.entities.enums.ConfigPropertyKey +import fr.shikkanime.services.* +import fr.shikkanime.utils.Constant +import org.junit.jupiter.api.AfterEach +import org.junit.jupiter.api.Assertions.assertTrue +import org.junit.jupiter.api.BeforeEach +import org.junit.jupiter.api.Test + +class FetchOldEpisodesJobTest { + @Inject + private lateinit var fetchOldEpisodesJob: FetchOldEpisodesJob + + @Inject + private lateinit var configService: ConfigService + + @Inject + private lateinit var animeService: AnimeService + + @Inject + private lateinit var episodeVariantService: EpisodeVariantService + + @Inject + private lateinit var episodeMappingService: EpisodeMappingService + + @Inject + private lateinit var simulcastService: SimulcastService + + @BeforeEach + fun setUp() { + Constant.injector.injectMembers(this) + } + + @AfterEach + fun tearDown() { + configService.deleteAll() + episodeVariantService.deleteAll() + episodeMappingService.deleteAll() + animeService.deleteAll() + simulcastService.deleteAll() + } + + @Test + fun run() { + configService.save( + Config( + propertyKey = ConfigPropertyKey.LAST_FETCH_OLD_EPISODES.key, + propertyValue = "2024-01-10" + ) + ) + configService.save(Config(propertyKey = ConfigPropertyKey.FETCH_OLD_EPISODES_RANGE.key, propertyValue = "14")) + fetchOldEpisodesJob.run() + + val animes = animeService.findAll() + assertTrue(animes.isNotEmpty()) + assertTrue(animes.all { it.mappings.isNotEmpty() }) + val mappings = animes.flatMap { it.mappings } + assertTrue(mappings.all { it.variants.isNotEmpty() }) + } +} \ No newline at end of file diff --git a/src/test/kotlin/fr/shikkanime/platforms/PrimeVideoPlatformTest.kt b/src/test/kotlin/fr/shikkanime/platforms/PrimeVideoPlatformTest.kt index f6bb0d3b..91187664 100644 --- a/src/test/kotlin/fr/shikkanime/platforms/PrimeVideoPlatformTest.kt +++ b/src/test/kotlin/fr/shikkanime/platforms/PrimeVideoPlatformTest.kt @@ -1,7 +1,6 @@ package fr.shikkanime.platforms import com.google.inject.Inject -import com.microsoft.playwright.junit.UsePlaywright import fr.shikkanime.caches.CountryCodePrimeVideoSimulcastKeyCache import fr.shikkanime.entities.enums.CountryCode import fr.shikkanime.platforms.configuration.PrimeVideoConfiguration @@ -13,7 +12,6 @@ import org.junit.jupiter.api.BeforeEach import org.junit.jupiter.api.Test import java.time.ZonedDateTime -@UsePlaywright class PrimeVideoPlatformTest { @Inject private lateinit var primeVideoPlatform: PrimeVideoPlatform