From 020b7169b27e2b0f365fc5eb996f8331b040b33a Mon Sep 17 00:00:00 2001 From: Ziedelth Date: Mon, 11 Dec 2023 17:41:05 +0100 Subject: [PATCH] Update implementation and refactor Crunchyroll logic --- .gitignore | 1 + build.gradle.kts | 8 +- hibernate.cfg.xml | 2 +- src/main/kotlin/fr/shikkanime/Application.kt | 5 + .../caches/CountryCodeAnimeIdKeyCache.kt | 26 + .../shikkanime/controllers/AdminController.kt | 6 +- .../controllers/api/PlatformController.kt | 43 - .../converters/AbstractConverter.kt | 5 +- .../metric/MetricToMetricDtoConverter.kt | 6 +- .../PlatformDtoToPlatformConverter.kt | 28 - .../PlatformToPlatformDtoConverter.kt | 16 - .../kotlin/fr/shikkanime/dtos/PlatformDto.kt | 11 - .../kotlin/fr/shikkanime/entities/Anime.kt | 10 +- .../kotlin/fr/shikkanime/entities/Country.kt | 16 - .../kotlin/fr/shikkanime/entities/Episode.kt | 12 +- .../kotlin/fr/shikkanime/entities/Platform.kt | 18 - .../shikkanime/entities/enums/CountryCode.kt | 13 + .../fr/shikkanime/entities/enums/Platform.kt | 12 + .../shikkanime/exceptions/AnimeException.kt | 3 + .../AnimeNotSimulcastedException.kt | 3 + .../EpisodeAlreadyReleasedException.kt | 3 + .../shikkanime/exceptions/EpisodeException.kt | 3 + .../EpisodeNoSubtitlesOrVoiceException.kt | 3 + .../EpisodeNotAvailableInCountryException.kt | 3 + .../fr/shikkanime/jobs/FetchEpisodesJob.kt | 34 +- .../fr/shikkanime/jobs/SavingImageCacheJob.kt | 10 + .../fr/shikkanime/modules/DefaultModule.kt | 5 + .../shikkanime/platforms/AbstractPlatform.kt | 54 +- .../AnimationDigitalNetworkPlatform.kt | 104 +- .../platforms/CrunchyrollPlatform.kt | 268 +- .../platforms/DisneyPlusPlatform.kt | 207 ++ .../platforms/PlatformConfiguration.kt | 9 +- src/main/kotlin/fr/shikkanime/plugins/HTTP.kt | 4 +- .../kotlin/fr/shikkanime/plugins/Routing.kt | 5 +- .../repositories/AnimeRepository.kt | 6 +- .../repositories/CountryRepository.kt | 31 - .../repositories/EpisodeRepository.kt | 7 + .../repositories/PlatformRepository.kt | 14 - .../fr/shikkanime/services/AbstractService.kt | 2 - .../fr/shikkanime/services/AnimeService.kt | 19 +- .../fr/shikkanime/services/CountryService.kt | 31 - .../fr/shikkanime/services/EpisodeService.kt | 42 +- .../fr/shikkanime/services/ImageService.kt | 121 + .../fr/shikkanime/services/MetricService.kt | 9 +- .../fr/shikkanime/services/PlatformService.kt | 36 - .../fr/shikkanime/utils/CompressionManager.kt | 43 + .../kotlin/fr/shikkanime/utils/Constant.kt | 2 - .../kotlin/fr/shikkanime/utils/Database.kt | 6 +- .../kotlin/fr/shikkanime/utils/HttpRequest.kt | 83 +- .../kotlin/fr/shikkanime/utils/MapCache.kt | 28 +- .../fr/shikkanime/utils/ObjectParser.kt | 48 + .../db/changelog/2023/12/01-changelog.xml | 123 +- .../db/changelog/db.changelog-master.xml | 3 +- .../resources/templates/admin/_layout.ftl | 15 +- .../resources/templates/admin/dashboard.ftl | 4 +- .../resources/templates/admin/platforms.ftl | 29 +- .../kotlin/fr/shikkanime/ApplicationTest.kt | 22 - .../AnimationDigitalNetworkPlatformTest.kt | 36 + .../platforms/CrunchyrollPlatformTest.kt | 187 ++ .../crunchyroll/rss-2023-12-11T18-00-00Z.xml | 2970 +++++++++++++++++ src/test/resources/hibernate.cfg.xml | 18 + 61 files changed, 4299 insertions(+), 592 deletions(-) create mode 100644 src/main/kotlin/fr/shikkanime/caches/CountryCodeAnimeIdKeyCache.kt delete mode 100644 src/main/kotlin/fr/shikkanime/controllers/api/PlatformController.kt delete mode 100644 src/main/kotlin/fr/shikkanime/converters/platform/PlatformDtoToPlatformConverter.kt delete mode 100644 src/main/kotlin/fr/shikkanime/converters/platform/PlatformToPlatformDtoConverter.kt delete mode 100644 src/main/kotlin/fr/shikkanime/dtos/PlatformDto.kt delete mode 100644 src/main/kotlin/fr/shikkanime/entities/Country.kt delete mode 100644 src/main/kotlin/fr/shikkanime/entities/Platform.kt create mode 100644 src/main/kotlin/fr/shikkanime/entities/enums/CountryCode.kt create mode 100644 src/main/kotlin/fr/shikkanime/entities/enums/Platform.kt create mode 100644 src/main/kotlin/fr/shikkanime/exceptions/AnimeException.kt create mode 100644 src/main/kotlin/fr/shikkanime/exceptions/AnimeNotSimulcastedException.kt create mode 100644 src/main/kotlin/fr/shikkanime/exceptions/EpisodeAlreadyReleasedException.kt create mode 100644 src/main/kotlin/fr/shikkanime/exceptions/EpisodeException.kt create mode 100644 src/main/kotlin/fr/shikkanime/exceptions/EpisodeNoSubtitlesOrVoiceException.kt create mode 100644 src/main/kotlin/fr/shikkanime/exceptions/EpisodeNotAvailableInCountryException.kt create mode 100644 src/main/kotlin/fr/shikkanime/jobs/SavingImageCacheJob.kt create mode 100644 src/main/kotlin/fr/shikkanime/platforms/DisneyPlusPlatform.kt delete mode 100644 src/main/kotlin/fr/shikkanime/repositories/CountryRepository.kt delete mode 100644 src/main/kotlin/fr/shikkanime/repositories/PlatformRepository.kt delete mode 100644 src/main/kotlin/fr/shikkanime/services/CountryService.kt create mode 100644 src/main/kotlin/fr/shikkanime/services/ImageService.kt delete mode 100644 src/main/kotlin/fr/shikkanime/services/PlatformService.kt create mode 100644 src/main/kotlin/fr/shikkanime/utils/CompressionManager.kt create mode 100644 src/main/kotlin/fr/shikkanime/utils/ObjectParser.kt delete mode 100644 src/test/kotlin/fr/shikkanime/ApplicationTest.kt create mode 100644 src/test/kotlin/fr/shikkanime/platforms/AnimationDigitalNetworkPlatformTest.kt create mode 100644 src/test/kotlin/fr/shikkanime/platforms/CrunchyrollPlatformTest.kt create mode 100644 src/test/resources/crunchyroll/rss-2023-12-11T18-00-00Z.xml create mode 100644 src/test/resources/hibernate.cfg.xml diff --git a/.gitignore b/.gitignore index e1451812..807c3ff1 100644 --- a/.gitignore +++ b/.gitignore @@ -37,3 +37,4 @@ out/ /metrics.json /.jpb/ /config/ +/images-cache.shikk diff --git a/build.gradle.kts b/build.gradle.kts index 80b9c2d6..4a59f49f 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -30,7 +30,7 @@ dependencies { implementation("io.ktor:ktor-server-compression-jvm:$ktor_version") implementation("io.ktor:ktor-server-cors-jvm:$ktor_version") implementation("io.ktor:ktor-server-content-negotiation-jvm:$ktor_version") - implementation("io.ktor:ktor-serialization-gson:$ktor_version") + implementation("io.ktor:ktor-serialization-jackson:$ktor_version") implementation("io.ktor:ktor-server-freemarker-jvm:$ktor_version") implementation("io.ktor:ktor-server-netty-jvm:$ktor_version") implementation("io.ktor:ktor-client-core:$ktor_version") @@ -42,6 +42,12 @@ dependencies { implementation("org.liquibase:liquibase-core:4.25.0") implementation("org.quartz-scheduler:quartz:2.5.0-rc1") implementation("com.google.guava:guava:32.1.3-jre") + implementation("com.fasterxml.jackson.dataformat:jackson-dataformat-xml:2.16.0") + implementation("com.microsoft.playwright:playwright:1.40.0") + implementation("org.jsoup:jsoup:1.17.1") + implementation("com.google.code.gson:gson:2.10.1") + implementation("org.openpnp:opencv:4.7.0-0") testImplementation("io.ktor:ktor-server-tests-jvm:$ktor_version") testImplementation("org.jetbrains.kotlin:kotlin-test-junit:$kotlin_version") + testImplementation("com.h2database:h2:2.2.224") } diff --git a/hibernate.cfg.xml b/hibernate.cfg.xml index 6e5bd88f..425cd7fa 100644 --- a/hibernate.cfg.xml +++ b/hibernate.cfg.xml @@ -8,7 +8,7 @@ jdbc:postgresql://localhost:5432/shikkanime postgres mysecretpassword - true + false org.hibernate.context.internal.ThreadLocalSessionContext \ No newline at end of file diff --git a/src/main/kotlin/fr/shikkanime/Application.kt b/src/main/kotlin/fr/shikkanime/Application.kt index bcc41d47..e231459c 100644 --- a/src/main/kotlin/fr/shikkanime/Application.kt +++ b/src/main/kotlin/fr/shikkanime/Application.kt @@ -3,18 +3,23 @@ package fr.shikkanime import fr.shikkanime.jobs.FetchEpisodesJob import fr.shikkanime.jobs.GCJob import fr.shikkanime.jobs.MetricJob +import fr.shikkanime.jobs.SavingImageCacheJob import fr.shikkanime.plugins.configureHTTP import fr.shikkanime.plugins.configureRouting import fr.shikkanime.plugins.configureSecurity +import fr.shikkanime.services.ImageService import fr.shikkanime.utils.JobManager import io.ktor.server.application.* import io.ktor.server.engine.* import io.ktor.server.netty.* fun main() { + ImageService.loadCache() + JobManager.scheduleJob("*/10 * * * * ?", MetricJob::class.java) JobManager.scheduleJob("0 */5 * * * ?", GCJob::class.java) JobManager.scheduleJob("0 * * * * ?", FetchEpisodesJob::class.java) + JobManager.scheduleJob("0 */5 * * * ?", SavingImageCacheJob::class.java) JobManager.start() embeddedServer( diff --git a/src/main/kotlin/fr/shikkanime/caches/CountryCodeAnimeIdKeyCache.kt b/src/main/kotlin/fr/shikkanime/caches/CountryCodeAnimeIdKeyCache.kt new file mode 100644 index 00000000..57f8d266 --- /dev/null +++ b/src/main/kotlin/fr/shikkanime/caches/CountryCodeAnimeIdKeyCache.kt @@ -0,0 +1,26 @@ +package fr.shikkanime.caches + +import fr.shikkanime.entities.enums.CountryCode + +data class CountryCodeAnimeIdKeyCache( + val countryCode: CountryCode, + val animeId: String, +) { + override fun equals(other: Any?): Boolean { + if (this === other) return true + if (javaClass != other?.javaClass) return false + + other as CountryCodeAnimeIdKeyCache + + if (countryCode != other.countryCode) return false + if (animeId != other.animeId) return false + + return true + } + + override fun hashCode(): Int { + var result = countryCode.hashCode() + result = 31 * result + animeId.hashCode() + return result + } +} diff --git a/src/main/kotlin/fr/shikkanime/controllers/AdminController.kt b/src/main/kotlin/fr/shikkanime/controllers/AdminController.kt index b564eb78..2a78e62d 100644 --- a/src/main/kotlin/fr/shikkanime/controllers/AdminController.kt +++ b/src/main/kotlin/fr/shikkanime/controllers/AdminController.kt @@ -23,7 +23,8 @@ class AdminController { return TemplateResponse( "admin/platforms.ftl", "Platforms", - mutableMapOf("platforms" to Constant.abstractPlatforms) + mutableMapOf( + "platforms" to Constant.abstractPlatforms.toList().sortedBy { it.getPlatform().name.lowercase() }) ) } @@ -34,7 +35,8 @@ class AdminController { val redirectResponse = RedirectResponse("/admin/platforms") val platformName = parameters["platform"] ?: return redirectResponse - val abstractPlatform = Constant.abstractPlatforms.find { it.getPlatform().name == platformName } ?: return redirectResponse + val abstractPlatform = + Constant.abstractPlatforms.find { it.getPlatform().name == platformName } ?: return redirectResponse abstractPlatform.configuration?.of(parameters) abstractPlatform.saveConfiguration() return redirectResponse diff --git a/src/main/kotlin/fr/shikkanime/controllers/api/PlatformController.kt b/src/main/kotlin/fr/shikkanime/controllers/api/PlatformController.kt deleted file mode 100644 index 96eb9c60..00000000 --- a/src/main/kotlin/fr/shikkanime/controllers/api/PlatformController.kt +++ /dev/null @@ -1,43 +0,0 @@ -package fr.shikkanime.controllers.api - -import com.google.inject.Inject -import fr.shikkanime.converters.AbstractConverter -import fr.shikkanime.dtos.PlatformDto -import fr.shikkanime.entities.Platform -import fr.shikkanime.services.PlatformService -import fr.shikkanime.utils.routes.BodyParam -import fr.shikkanime.utils.routes.Controller -import fr.shikkanime.utils.routes.Path -import fr.shikkanime.utils.routes.Response -import fr.ziedelth.utils.routes.method.Get -import fr.ziedelth.utils.routes.method.Post -import java.util.* - -@Controller("/api/platforms") -class PlatformController { - @Inject - private lateinit var platformService: PlatformService - - @Path - @Get - private fun getPlatforms(): Response { - return Response.ok(AbstractConverter.convert(platformService.findAll(), PlatformDto::class.java)) - } - - @Path("/{uuid}") - @Get - private fun getPlatform(uuid: UUID): Response { - return Response.ok(AbstractConverter.convert(platformService.find(uuid), PlatformDto::class.java)) - } - - @Path - @Post - private fun createPlatform(@BodyParam platformDto: PlatformDto): Response { - return Response.ok( - AbstractConverter.convert( - platformService.save(AbstractConverter.convert(platformDto, Platform::class.java)), - PlatformDto::class.java - ) - ) - } -} \ No newline at end of file diff --git a/src/main/kotlin/fr/shikkanime/converters/AbstractConverter.kt b/src/main/kotlin/fr/shikkanime/converters/AbstractConverter.kt index 162357f2..4dd6868c 100644 --- a/src/main/kotlin/fr/shikkanime/converters/AbstractConverter.kt +++ b/src/main/kotlin/fr/shikkanime/converters/AbstractConverter.kt @@ -36,7 +36,10 @@ abstract class AbstractConverter { return try { method.invoke(abstractConverter, `object`) as T } catch (e: Exception) { - throw IllegalStateException("Can not convert \"${`object`.javaClass.simpleName}\" to \"${to.simpleName}\"", e) + throw IllegalStateException( + "Can not convert \"${`object`.javaClass.simpleName}\" to \"${to.simpleName}\"", + e + ) } } diff --git a/src/main/kotlin/fr/shikkanime/converters/metric/MetricToMetricDtoConverter.kt b/src/main/kotlin/fr/shikkanime/converters/metric/MetricToMetricDtoConverter.kt index 69a8e9db..3446cc73 100644 --- a/src/main/kotlin/fr/shikkanime/converters/metric/MetricToMetricDtoConverter.kt +++ b/src/main/kotlin/fr/shikkanime/converters/metric/MetricToMetricDtoConverter.kt @@ -23,9 +23,11 @@ class MetricToMetricDtoConverter : AbstractConverter() { return MetricDto( uuid = from.uuid, cpuLoad = (from.cpuLoad * 100).toString().replace(',', '.'), - averageCpuLoad = metricService.getAverageCpuLoad(minusHours, from.date)?.times(100)?.toString()?.replace(',', '.') ?: "0", + averageCpuLoad = metricService.getAverageCpuLoad(minusHours, from.date)?.times(100)?.toString() + ?.replace(',', '.') ?: "0", memoryUsage = (from.memoryUsage / 1024.0 / 1024.0).toString().replace(',', '.'), - averageMemoryUsage = metricService.getAverageMemoryUsage(minusHours, from.date)?.div(1024)?.div(1024)?.toString()?.replace(',', '.') ?: "0", + averageMemoryUsage = metricService.getAverageMemoryUsage(minusHours, from.date)?.div(1024)?.div(1024) + ?.toString()?.replace(',', '.') ?: "0", databaseSize = (from.databaseSize / 1024.0 / 1024.0).toDoublePoint(), date = from.date.withZoneSameInstant(europeParisZone).format(dateFormatter) ) diff --git a/src/main/kotlin/fr/shikkanime/converters/platform/PlatformDtoToPlatformConverter.kt b/src/main/kotlin/fr/shikkanime/converters/platform/PlatformDtoToPlatformConverter.kt deleted file mode 100644 index 58125a09..00000000 --- a/src/main/kotlin/fr/shikkanime/converters/platform/PlatformDtoToPlatformConverter.kt +++ /dev/null @@ -1,28 +0,0 @@ -package fr.shikkanime.converters.platform - -import com.google.inject.Inject -import fr.shikkanime.converters.AbstractConverter -import fr.shikkanime.dtos.PlatformDto -import fr.shikkanime.entities.Platform -import fr.shikkanime.services.PlatformService - -class PlatformDtoToPlatformConverter : AbstractConverter() { - @Inject - private lateinit var platformService: PlatformService - - override fun convert(from: PlatformDto): Platform { - if (from.uuid != null) { - val find = platformService.find(from.uuid) - - if (find != null) { - return find - } - } - - return Platform( - name = from.name, - url = from.url, - image = from.image - ) - } -} \ No newline at end of file diff --git a/src/main/kotlin/fr/shikkanime/converters/platform/PlatformToPlatformDtoConverter.kt b/src/main/kotlin/fr/shikkanime/converters/platform/PlatformToPlatformDtoConverter.kt deleted file mode 100644 index 8b962248..00000000 --- a/src/main/kotlin/fr/shikkanime/converters/platform/PlatformToPlatformDtoConverter.kt +++ /dev/null @@ -1,16 +0,0 @@ -package fr.shikkanime.converters.platform - -import fr.shikkanime.converters.AbstractConverter -import fr.shikkanime.dtos.PlatformDto -import fr.shikkanime.entities.Platform - -class PlatformToPlatformDtoConverter : AbstractConverter() { - override fun convert(from: Platform): PlatformDto { - return PlatformDto( - uuid = from.uuid, - name = from.name!!, - url = from.url!!, - image = from.image!! - ) - } -} \ No newline at end of file diff --git a/src/main/kotlin/fr/shikkanime/dtos/PlatformDto.kt b/src/main/kotlin/fr/shikkanime/dtos/PlatformDto.kt deleted file mode 100644 index 1825410e..00000000 --- a/src/main/kotlin/fr/shikkanime/dtos/PlatformDto.kt +++ /dev/null @@ -1,11 +0,0 @@ -package fr.shikkanime.dtos - -import java.io.Serializable -import java.util.* - -data class PlatformDto( - val uuid: UUID?, - val name: String, - val url: String, - val image: String -) : Serializable diff --git a/src/main/kotlin/fr/shikkanime/entities/Anime.kt b/src/main/kotlin/fr/shikkanime/entities/Anime.kt index 6ee184ec..f83f50c0 100644 --- a/src/main/kotlin/fr/shikkanime/entities/Anime.kt +++ b/src/main/kotlin/fr/shikkanime/entities/Anime.kt @@ -1,5 +1,6 @@ package fr.shikkanime.entities +import fr.shikkanime.entities.enums.CountryCode import jakarta.persistence.* import java.time.ZonedDateTime import java.util.* @@ -8,14 +9,15 @@ import java.util.* @Table(name = "anime") data class Anime( override val uuid: UUID? = null, - @ManyToOne(optional = false, fetch = FetchType.LAZY) - val country: Country? = null, + @Column(nullable = false, name = "country_code") + @Enumerated(EnumType.STRING) + val countryCode: CountryCode? = null, @Column(nullable = false) val name: String? = null, @Column(nullable = false, name = "release_date") val releaseDate: ZonedDateTime = ZonedDateTime.now(), - @Column(nullable = false) + @Column(nullable = false, columnDefinition = "VARCHAR(1000)") var image: String? = null, - @Column(nullable = true, columnDefinition = "TEXT") + @Column(nullable = true, columnDefinition = "VARCHAR(1000)") var description: String? = null, ) : ShikkEntity(uuid) diff --git a/src/main/kotlin/fr/shikkanime/entities/Country.kt b/src/main/kotlin/fr/shikkanime/entities/Country.kt deleted file mode 100644 index b0264e56..00000000 --- a/src/main/kotlin/fr/shikkanime/entities/Country.kt +++ /dev/null @@ -1,16 +0,0 @@ -package fr.shikkanime.entities - -import jakarta.persistence.Column -import jakarta.persistence.Entity -import jakarta.persistence.Table -import java.util.* - -@Entity -@Table(name = "country") -data class Country( - override val uuid: UUID? = null, - @Column(nullable = false, unique = true) - val name: String? = null, - @Column(nullable = false, unique = true, name = "country_code") - val countryCode: String? = null, -) : ShikkEntity(uuid) \ No newline at end of file diff --git a/src/main/kotlin/fr/shikkanime/entities/Episode.kt b/src/main/kotlin/fr/shikkanime/entities/Episode.kt index 9058e71d..06181bf1 100644 --- a/src/main/kotlin/fr/shikkanime/entities/Episode.kt +++ b/src/main/kotlin/fr/shikkanime/entities/Episode.kt @@ -2,6 +2,7 @@ package fr.shikkanime.entities import fr.shikkanime.entities.enums.EpisodeType import fr.shikkanime.entities.enums.LangType +import fr.shikkanime.entities.enums.Platform import jakarta.persistence.* import java.time.ZonedDateTime import java.util.* @@ -10,7 +11,8 @@ import java.util.* @Table(name = "episode") data class Episode( override val uuid: UUID? = null, - @ManyToOne(optional = false, fetch = FetchType.LAZY) + @Column(nullable = false) + @Enumerated(EnumType.STRING) var platform: Platform? = null, @ManyToOne(optional = false, fetch = FetchType.LAZY) var anime: Anime? = null, @@ -20,7 +22,7 @@ data class Episode( @Column(nullable = false, name = "lang_type") @Enumerated(EnumType.STRING) val langType: LangType? = null, - @Column(nullable = false) + @Column(nullable = false, unique = true) val hash: String? = null, @Column(nullable = false, name = "release_date") val releaseDate: ZonedDateTime = ZonedDateTime.now(), @@ -28,11 +30,11 @@ data class Episode( var season: Int? = null, @Column(nullable = false) var number: Int? = null, - @Column(nullable = true, columnDefinition = "TEXT") + @Column(nullable = true, columnDefinition = "VARCHAR(1000)") var title: String? = null, - @Column(nullable = false, columnDefinition = "TEXT") + @Column(nullable = false, columnDefinition = "VARCHAR(1000)") var url: String? = null, - @Column(nullable = false, columnDefinition = "TEXT") + @Column(nullable = false, columnDefinition = "VARCHAR(1000)") var image: String? = null, @Column(nullable = false) var duration: Long = -1 diff --git a/src/main/kotlin/fr/shikkanime/entities/Platform.kt b/src/main/kotlin/fr/shikkanime/entities/Platform.kt deleted file mode 100644 index 9146a6dd..00000000 --- a/src/main/kotlin/fr/shikkanime/entities/Platform.kt +++ /dev/null @@ -1,18 +0,0 @@ -package fr.shikkanime.entities - -import jakarta.persistence.Column -import jakarta.persistence.Entity -import jakarta.persistence.Table -import java.util.* - -@Entity -@Table(name = "platform") -data class Platform( - override val uuid: UUID? = null, - @Column(nullable = false, unique = true) - val name: String? = null, - @Column(nullable = false) - var url: String? = null, - @Column(nullable = false) - var image: String? = null -) : ShikkEntity(uuid) diff --git a/src/main/kotlin/fr/shikkanime/entities/enums/CountryCode.kt b/src/main/kotlin/fr/shikkanime/entities/enums/CountryCode.kt new file mode 100644 index 00000000..7f9af5a0 --- /dev/null +++ b/src/main/kotlin/fr/shikkanime/entities/enums/CountryCode.kt @@ -0,0 +1,13 @@ +package fr.shikkanime.entities.enums + + +enum class CountryCode(val locale: String? = null, val voice: String? = null) { + FR("fr-FR", "VF"), + ; + + companion object { + fun from(collection: Collection): Set { + return collection.map { valueOf(it.uppercase()) }.toSet() + } + } +} \ No newline at end of file diff --git a/src/main/kotlin/fr/shikkanime/entities/enums/Platform.kt b/src/main/kotlin/fr/shikkanime/entities/enums/Platform.kt new file mode 100644 index 00000000..cf40b8d0 --- /dev/null +++ b/src/main/kotlin/fr/shikkanime/entities/enums/Platform.kt @@ -0,0 +1,12 @@ +package fr.shikkanime.entities.enums + +enum class Platform( + val platformName: String, + var url: String, + var image: String +) { + ANIM("Animation Digital Network", "https://animationdigitalnetwork.fr/", "animation_digital_network.png"), + CRUN("Crunchyroll", "https://www.crunchyroll.com/", "crunchyroll.png"), + DISN("Disney+", "https://www.disneyplus.com/", "disneyplus.png"), + ; +} diff --git a/src/main/kotlin/fr/shikkanime/exceptions/AnimeException.kt b/src/main/kotlin/fr/shikkanime/exceptions/AnimeException.kt new file mode 100644 index 00000000..5fcb94e0 --- /dev/null +++ b/src/main/kotlin/fr/shikkanime/exceptions/AnimeException.kt @@ -0,0 +1,3 @@ +package fr.shikkanime.exceptions + +open class AnimeException(override val message: String? = null) : Exception(message) diff --git a/src/main/kotlin/fr/shikkanime/exceptions/AnimeNotSimulcastedException.kt b/src/main/kotlin/fr/shikkanime/exceptions/AnimeNotSimulcastedException.kt new file mode 100644 index 00000000..27ad3a20 --- /dev/null +++ b/src/main/kotlin/fr/shikkanime/exceptions/AnimeNotSimulcastedException.kt @@ -0,0 +1,3 @@ +package fr.shikkanime.exceptions + +data class AnimeNotSimulcastedException(override val message: String? = null) : AnimeException(message) diff --git a/src/main/kotlin/fr/shikkanime/exceptions/EpisodeAlreadyReleasedException.kt b/src/main/kotlin/fr/shikkanime/exceptions/EpisodeAlreadyReleasedException.kt new file mode 100644 index 00000000..31eebf93 --- /dev/null +++ b/src/main/kotlin/fr/shikkanime/exceptions/EpisodeAlreadyReleasedException.kt @@ -0,0 +1,3 @@ +package fr.shikkanime.exceptions + +data class EpisodeAlreadyReleasedException(override val message: String? = null) : EpisodeException(message) diff --git a/src/main/kotlin/fr/shikkanime/exceptions/EpisodeException.kt b/src/main/kotlin/fr/shikkanime/exceptions/EpisodeException.kt new file mode 100644 index 00000000..d7c65193 --- /dev/null +++ b/src/main/kotlin/fr/shikkanime/exceptions/EpisodeException.kt @@ -0,0 +1,3 @@ +package fr.shikkanime.exceptions + +open class EpisodeException(override val message: String? = null) : Exception(message) diff --git a/src/main/kotlin/fr/shikkanime/exceptions/EpisodeNoSubtitlesOrVoiceException.kt b/src/main/kotlin/fr/shikkanime/exceptions/EpisodeNoSubtitlesOrVoiceException.kt new file mode 100644 index 00000000..4250e0b3 --- /dev/null +++ b/src/main/kotlin/fr/shikkanime/exceptions/EpisodeNoSubtitlesOrVoiceException.kt @@ -0,0 +1,3 @@ +package fr.shikkanime.exceptions + +data class EpisodeNoSubtitlesOrVoiceException(override val message: String? = null) : EpisodeException(message) diff --git a/src/main/kotlin/fr/shikkanime/exceptions/EpisodeNotAvailableInCountryException.kt b/src/main/kotlin/fr/shikkanime/exceptions/EpisodeNotAvailableInCountryException.kt new file mode 100644 index 00000000..a333e7d7 --- /dev/null +++ b/src/main/kotlin/fr/shikkanime/exceptions/EpisodeNotAvailableInCountryException.kt @@ -0,0 +1,3 @@ +package fr.shikkanime.exceptions + +data class EpisodeNotAvailableInCountryException(override val message: String? = null) : EpisodeException(message) diff --git a/src/main/kotlin/fr/shikkanime/jobs/FetchEpisodesJob.kt b/src/main/kotlin/fr/shikkanime/jobs/FetchEpisodesJob.kt index 13d8b8fe..dbb23beb 100644 --- a/src/main/kotlin/fr/shikkanime/jobs/FetchEpisodesJob.kt +++ b/src/main/kotlin/fr/shikkanime/jobs/FetchEpisodesJob.kt @@ -2,15 +2,35 @@ package fr.shikkanime.jobs import fr.shikkanime.entities.Episode import fr.shikkanime.services.EpisodeService +import fr.shikkanime.services.ImageService import fr.shikkanime.utils.Constant import jakarta.inject.Inject import java.time.ZonedDateTime class FetchEpisodesJob : AbstractJob() { + private var isInitialized = false + private var isRunning = false + private val set = mutableSetOf() + @Inject private lateinit var episodeService: EpisodeService override fun run() { + if (isRunning) { + println("Job is already running") + return + } + + isRunning = true + + if (!isInitialized) { + val hashes = episodeService.findAllHashes() + + set.addAll(hashes) + Constant.abstractPlatforms.forEach { it.hashCache.addAll(hashes) } + isInitialized = true + } + val zonedDateTime = ZonedDateTime.now().withNano(0) val episodes = mutableListOf() @@ -25,6 +45,18 @@ class FetchEpisodesJob : AbstractJob() { } } - episodes.forEach { episodeService.saveOrUpdate(it) } + episodes + .filter { + (zonedDateTime.isEqual(it.releaseDate) || zonedDateTime.isAfter(it.releaseDate)) && !set.contains( + it.hash + ) + } + .forEach { + val savedEpisode = episodeService.save(it) + savedEpisode.hash?.let { hash -> set.add(hash) } + ImageService.add(savedEpisode.uuid!!, savedEpisode.image!!) + } + + isRunning = false } } \ No newline at end of file diff --git a/src/main/kotlin/fr/shikkanime/jobs/SavingImageCacheJob.kt b/src/main/kotlin/fr/shikkanime/jobs/SavingImageCacheJob.kt new file mode 100644 index 00000000..1b9e7b14 --- /dev/null +++ b/src/main/kotlin/fr/shikkanime/jobs/SavingImageCacheJob.kt @@ -0,0 +1,10 @@ +package fr.shikkanime.jobs + +import fr.shikkanime.services.ImageService + +class SavingImageCacheJob : AbstractJob() { + + override fun run() { + ImageService.saveCache() + } +} \ No newline at end of file diff --git a/src/main/kotlin/fr/shikkanime/modules/DefaultModule.kt b/src/main/kotlin/fr/shikkanime/modules/DefaultModule.kt index 7de3b090..7ee962cd 100644 --- a/src/main/kotlin/fr/shikkanime/modules/DefaultModule.kt +++ b/src/main/kotlin/fr/shikkanime/modules/DefaultModule.kt @@ -1,6 +1,7 @@ package fr.shikkanime.modules import com.google.inject.AbstractModule +import fr.shikkanime.jobs.AbstractJob import fr.shikkanime.platforms.AbstractPlatform import fr.shikkanime.repositories.AbstractRepository import fr.shikkanime.services.AbstractService @@ -24,6 +25,10 @@ class DefaultModule : AbstractModule() { bind(it).asEagerSingleton() } + Constant.reflections.getSubTypesOf(AbstractJob::class.java).forEach { + bind(it).asEagerSingleton() + } + Constant.reflections.getTypesAnnotatedWith(Controller::class.java).forEach { bind(it).asEagerSingleton() } diff --git a/src/main/kotlin/fr/shikkanime/platforms/AbstractPlatform.kt b/src/main/kotlin/fr/shikkanime/platforms/AbstractPlatform.kt index 4b4a5f90..f54da1d2 100644 --- a/src/main/kotlin/fr/shikkanime/platforms/AbstractPlatform.kt +++ b/src/main/kotlin/fr/shikkanime/platforms/AbstractPlatform.kt @@ -1,50 +1,37 @@ package fr.shikkanime.platforms -import fr.shikkanime.entities.Country import fr.shikkanime.entities.Episode -import fr.shikkanime.entities.Platform -import fr.shikkanime.services.CountryService -import fr.shikkanime.utils.Constant -import jakarta.inject.Inject +import fr.shikkanime.entities.enums.Platform +import fr.shikkanime.utils.ObjectParser import kotlinx.coroutines.runBlocking import java.io.File import java.time.ZonedDateTime -abstract class AbstractPlatform { - data class Api( - val lastCheck: ZonedDateTime, - val content: Map = emptyMap(), - ) - - @Inject - protected lateinit var countryService: CountryService - +abstract class AbstractPlatform { + val hashCache = mutableListOf() var configuration: C? = null - private var apiCache: Api? = null + private var apiCache = mutableMapOf, V>() protected abstract fun getConfigurationClass(): Class abstract fun getPlatform(): Platform - abstract suspend fun fetchApiContent(zonedDateTime: ZonedDateTime): Api - abstract fun fetchEpisodes(zonedDateTime: ZonedDateTime): List + abstract suspend fun fetchApiContent(key: K, zonedDateTime: ZonedDateTime): V + abstract fun fetchEpisodes(zonedDateTime: ZonedDateTime, bypassFileContent: File? = null): List abstract fun reset() - protected fun getCountries() = countryService.findAllByCode(configuration!!.availableCountries) - - fun getApiContent(country: Country, zonedDateTime: ZonedDateTime): String { - var hasFetch = false - - if (apiCache == null) { - apiCache = runBlocking { fetchApiContent(zonedDateTime) } - hasFetch = true + fun getApiContent(key: K, zonedDateTime: ZonedDateTime): V { + if (apiCache.none { it.key.first == key }) { + apiCache[Pair(key, zonedDateTime)] = runBlocking { fetchApiContent(key, zonedDateTime) } } - val plusHours = apiCache!!.lastCheck.plusMinutes(configuration!!.apiCheckDelayInMinutes) + val entry = apiCache.entries.firstOrNull { it.key.first == key }!! + val plusMinutes = entry.key.second.plusMinutes(configuration!!.apiCheckDelayInMinutes) - if (!hasFetch && zonedDateTime.isEqual(plusHours) || zonedDateTime.isAfter(plusHours)) { - apiCache = runBlocking { fetchApiContent(zonedDateTime) } + if (zonedDateTime.isEqual(plusMinutes) || zonedDateTime.isAfter(plusMinutes)) { + apiCache.remove(entry.key) + apiCache[Pair(key, zonedDateTime)] = runBlocking { fetchApiContent(key, zonedDateTime) } } - return apiCache!!.content[country.countryCode]!! + return apiCache.entries.firstOrNull { it.key.first == key }!!.value } fun loadConfiguration(): C { @@ -62,7 +49,7 @@ abstract class AbstractPlatform { try { println("Reading config file for ${getPlatform().name} platform") - val fromJson = Constant.gson.fromJson(file.readText(), getConfigurationClass()) + val fromJson = ObjectParser.fromJson(file.readText(), getConfigurationClass()) configuration = fromJson return fromJson } catch (e: Exception) { @@ -71,7 +58,7 @@ abstract class AbstractPlatform { } } - fun saveConfiguration() { + open fun saveConfiguration() { val file = getConfigurationFile() if (!file.exists()) { @@ -79,11 +66,10 @@ abstract class AbstractPlatform { file.createNewFile() } - file.writeText(Constant.gson.toJson(configuration)) - apiCache = null + file.writeText(ObjectParser.toJson(configuration)) } private fun getConfigurationFile(): File { - return File("config/${getPlatform().name!!.lowercase().replace(" ", "-")}.json") + return File("config/${getPlatform().platformName.lowercase().replace(" ", "-")}.json") } } diff --git a/src/main/kotlin/fr/shikkanime/platforms/AnimationDigitalNetworkPlatform.kt b/src/main/kotlin/fr/shikkanime/platforms/AnimationDigitalNetworkPlatform.kt index 8ae46c58..59804a8c 100644 --- a/src/main/kotlin/fr/shikkanime/platforms/AnimationDigitalNetworkPlatform.kt +++ b/src/main/kotlin/fr/shikkanime/platforms/AnimationDigitalNetworkPlatform.kt @@ -1,61 +1,55 @@ package fr.shikkanime.platforms +import com.google.gson.JsonArray import com.google.gson.JsonObject import fr.shikkanime.entities.Anime -import fr.shikkanime.entities.Country import fr.shikkanime.entities.Episode -import fr.shikkanime.entities.Platform +import fr.shikkanime.entities.enums.CountryCode import fr.shikkanime.entities.enums.EpisodeType import fr.shikkanime.entities.enums.LangType -import fr.shikkanime.utils.Constant +import fr.shikkanime.entities.enums.Platform +import fr.shikkanime.exceptions.AnimeException +import fr.shikkanime.exceptions.AnimeNotSimulcastedException import fr.shikkanime.utils.HttpRequest +import fr.shikkanime.utils.ObjectParser +import fr.shikkanime.utils.ObjectParser.getAsBoolean +import fr.shikkanime.utils.ObjectParser.getAsInt +import fr.shikkanime.utils.ObjectParser.getAsLong +import fr.shikkanime.utils.ObjectParser.getAsString import io.ktor.client.statement.* import io.ktor.http.* +import java.io.File import java.time.ZonedDateTime -class AnimationDigitalNetworkPlatform : AbstractPlatform() { +class AnimationDigitalNetworkPlatform : AbstractPlatform() { override fun getConfigurationClass(): Class = PlatformConfiguration::class.java - override fun getPlatform(): Platform { - return Platform( - name = "Animation Digital Network", - url = "https://animationdigitalnetwork.fr/", - image = "animation_digital_network.png", - ) - } + override fun getPlatform(): Platform = Platform.ANIM - override suspend fun fetchApiContent(zonedDateTime: ZonedDateTime): Api { - val map = mutableMapOf() - val countries = getCountries() + override suspend fun fetchApiContent(key: CountryCode, zonedDateTime: ZonedDateTime): JsonArray { val toDateString = zonedDateTime.toLocalDate().toString() + val url = "https://gw.api.animationdigitalnetwork.${key.name.lowercase()}/video/calendar?date=$toDateString" + val response = HttpRequest().get(url) - countries.forEach { country -> - val url = "https://gw.api.animationdigitalnetwork.${country.countryCode!!.lowercase()}/video/calendar?date=$toDateString" - val response = HttpRequest().get(url) - - if (response.status != HttpStatusCode.OK) { - return@forEach - } - - map[country.countryCode] = response.bodyAsText() + if (response.status != HttpStatusCode.OK) { + return JsonArray() } - return Api(zonedDateTime, map) + return ObjectParser.fromJson(response.bodyAsText(), JsonObject::class.java).getAsJsonArray("videos")!! } - override fun fetchEpisodes(zonedDateTime: ZonedDateTime): List { + override fun fetchEpisodes(zonedDateTime: ZonedDateTime, bypassFileContent: File?): List { val list = mutableListOf() - val countries = getCountries() - countries.forEach { country -> - val api = getApiContent(country, zonedDateTime) - val array = Constant.gson.fromJson(api, JsonObject::class.java).getAsJsonArray("videos") + configuration!!.availableCountries.forEach { countryCode -> + val api = getApiContent(countryCode, zonedDateTime) - array.forEach { + api.forEach { try { - list.add(convertEpisode(country, it.asJsonObject, zonedDateTime)) - } catch (e: Exception) { + list.add(convertEpisode(countryCode, it.getAsJsonObject(), zonedDateTime)) + } catch (_: AnimeException) { // Ignore + } catch (e: Exception) { e.printStackTrace() } } @@ -68,22 +62,27 @@ class AnimationDigitalNetworkPlatform : AbstractPlatform( TODO("Not yet implemented") } - private fun convertEpisode(country: Country, jsonObject: JsonObject, zonedDateTime: ZonedDateTime): Episode { + private fun convertEpisode( + countryCode: CountryCode, + jsonObject: JsonObject, + zonedDateTime: ZonedDateTime + ): Episode { val show = jsonObject.getAsJsonObject("show") ?: throw Exception("Show is null") - var animeName = show["shortTitle"]?.asString ?: show["title"]?.asString ?: throw Exception("Anime name is null") + var animeName = + show.getAsString("shortTitle") ?: show.getAsString("title") ?: throw Exception("Anime name is null") animeName = animeName.replace(Regex("Saison \\d"), "").trim('-').trim() - val animeImage = show["image2x"]?.asString ?: throw Exception("Anime image is null") - val animeDescription = show["summary"]?.asString ?: "" - val genres = show.getAsJsonArray("genres")?.toList() ?: emptyList() + val animeImage = show.getAsString("image2x") ?: throw Exception("Anime image is null") + val animeDescription = show.getAsString("summary")?.replace('\n', ' ') ?: "" + val genres = show.getAsJsonArray("genres") ?: JsonArray() - if (genres.isEmpty() || !genres.any { it.asString.startsWith("Animation ", true) }) { + if (genres.isEmpty || !genres.any { it.asString.startsWith("Animation ", true) }) { throw Exception("Anime is not an animation") } - var isSimulcasted = show["simulcast"]?.asBoolean == true || - show["firstReleaseYear"]?.asString == zonedDateTime.toLocalDate().year.toString() || + var isSimulcasted = show.getAsBoolean("simulcast") == true || + show.getAsString("firstReleaseYear") == zonedDateTime.toLocalDate().year.toString() || configuration?.simulcasts?.contains(animeName) == true val descriptionLowercase = animeDescription.lowercase() @@ -94,16 +93,15 @@ class AnimationDigitalNetworkPlatform : AbstractPlatform( descriptionLowercase.startsWith("(diffusion de l'épisode 1 le") if (!isSimulcasted) { - throw Exception("Anime is not simulcasted") + throw AnimeNotSimulcastedException("Anime is not simulcasted") } - val releaseDateString = jsonObject["releaseDate"]?.asString ?: throw Exception("Release date is null") - // Example : 2023-12-08T13:30:00Z + val releaseDateString = jsonObject.getAsString("releaseDate") ?: throw Exception("Release date is null") val releaseDate = ZonedDateTime.parse(releaseDateString) - val season = show["season"]?.asString?.toIntOrNull() ?: 1 + val season = jsonObject.getAsString("season")?.toIntOrNull() ?: 1 - val numberAsString = jsonObject["shortNumber"]?.asString + val numberAsString = jsonObject.getAsString("shortNumber") if (numberAsString?.startsWith("Bande-annonce") == true) { throw Exception("Anime is a trailer") @@ -117,26 +115,26 @@ class AnimationDigitalNetworkPlatform : AbstractPlatform( else -> EpisodeType.EPISODE } - val langType = when (jsonObject.getAsJsonArray("languages").lastOrNull()?.asString) { + val langType = when (jsonObject.getAsJsonArray("languages")?.lastOrNull()?.asString) { "vostf" -> LangType.SUBTITLES "vf" -> LangType.VOICE else -> throw Exception("Language is null") } - val id = jsonObject["id"]?.asLong ?: throw Exception("Id is null") + val id = jsonObject.getAsInt("id") - val title = jsonObject["name"]?.asString?.ifBlank { null } + val title = jsonObject.getAsString("name")?.ifBlank { null } - val url = jsonObject["url"]?.asString ?: throw Exception("Url is null") + val url = jsonObject.getAsString("url") ?: throw Exception("Url is null") - val image = jsonObject["image2x"]?.asString ?: throw Exception("Image is null") + val image = jsonObject.getAsString("image2x") ?: throw Exception("Image is null") - val duration = jsonObject["duration"]?.asLong ?: -1 + val duration = jsonObject.getAsLong("duration", -1) return Episode( platform = getPlatform(), anime = Anime( - country = country, + countryCode = countryCode, name = animeName, releaseDate = releaseDate, image = animeImage, @@ -144,7 +142,7 @@ class AnimationDigitalNetworkPlatform : AbstractPlatform( ), episodeType = episodeType, langType = langType, - hash = id.toString(), + hash = "${countryCode}-${getPlatform()}-$id-$langType", releaseDate = releaseDate, season = season, number = number, diff --git a/src/main/kotlin/fr/shikkanime/platforms/CrunchyrollPlatform.kt b/src/main/kotlin/fr/shikkanime/platforms/CrunchyrollPlatform.kt index 874c64b1..1794e035 100644 --- a/src/main/kotlin/fr/shikkanime/platforms/CrunchyrollPlatform.kt +++ b/src/main/kotlin/fr/shikkanime/platforms/CrunchyrollPlatform.kt @@ -1,11 +1,30 @@ package fr.shikkanime.platforms +import com.google.gson.JsonObject +import fr.shikkanime.caches.CountryCodeAnimeIdKeyCache +import fr.shikkanime.entities.Anime import fr.shikkanime.entities.Episode -import fr.shikkanime.entities.Platform +import fr.shikkanime.entities.enums.CountryCode +import fr.shikkanime.entities.enums.EpisodeType +import fr.shikkanime.entities.enums.LangType +import fr.shikkanime.entities.enums.Platform +import fr.shikkanime.exceptions.* +import fr.shikkanime.utils.HttpRequest +import fr.shikkanime.utils.MapCache +import fr.shikkanime.utils.ObjectParser +import fr.shikkanime.utils.ObjectParser.getAsInt +import fr.shikkanime.utils.ObjectParser.getAsLong +import fr.shikkanime.utils.ObjectParser.getAsString +import io.ktor.client.statement.* import io.ktor.http.* +import java.io.File +import java.time.Duration import java.time.ZonedDateTime +import java.time.format.DateTimeFormatter -class CrunchyrollPlatform : AbstractPlatform() { +private const val IMAGE_NULL_ERROR = "Image is null" + +class CrunchyrollPlatform : AbstractPlatform() { data class CrunchyrollConfiguration( var simulcastCheckDelayInMinutes: Long = 60, ) : PlatformConfiguration() { @@ -28,25 +47,248 @@ class CrunchyrollPlatform : AbstractPlatform>(Duration.ofHours(1)) { + fun getSimulcastCode(name: String): String { + val simulcastCodeTmp = name.lowercase().replace(" ", "-") + val simulcastYear = simulcastCodeTmp.split("-").last() - override fun getPlatform(): Platform { - return Platform( - name = "Crunchyroll", - url = "https://www.crunchyroll.com/", - image = "crunchyroll.png", - ) + val simulcastSeasonCode = when (simulcastCodeTmp.split("-").first()) { + "printemps" -> "spring" + "été" -> "summer" + "automne" -> "fall" + "hiver" -> "winter" + else -> throw Exception("Simulcast season not found") + } + + return "$simulcastSeasonCode-$simulcastYear" + } + + fun getPreviousSimulcastCode(currentSimulcastCode: String): String { + return when (currentSimulcastCode.split("-").first()) { + "spring" -> "winter-${currentSimulcastCode.split("-").last()}" + "winter" -> "fall-${currentSimulcastCode.split("-").last().toInt() - 1}" + "fall" -> "summer-${currentSimulcastCode.split("-").last().toInt()}" + "summer" -> "spring-${currentSimulcastCode.split("-").last().toInt()}" + else -> throw Exception("Simulcast season not found") + } + } + + val httpRequest = HttpRequest() + val simulcasts = mutableSetOf() + + val simulcastSelector = + "#content > div > div.app-body-wrapper > div > div > div.erc-browse-collection > div > div:nth-child(1) > div > div > h4 > a" + val simulcastAnimesSelector = ".erc-browse-cards-collection > .browse-card > div > div > h4 > a" + + try { + val currentSimulcastContent = httpRequest.getBrowser( + "https://www.crunchyroll.com/${it.name.lowercase()}/simulcasts", + simulcastSelector + ) + val currentSimulcast = + currentSimulcastContent.select("#content > div > div.app-body-wrapper > div > div > div.header > div > div > span.call-to-action--PEidl.call-to-action--is-m--RVdkI.select-trigger__title-cta--C5-uH.select-trigger__title-cta--is-displayed-on-mobile--6oNk1") + .text() ?: return@MapCache simulcasts + val currentSimulcastCode = getSimulcastCode(currentSimulcast) + println("Current simulcast code for $it: $currentSimulcast > $currentSimulcastCode") + val currentSimulcastAnimes = + currentSimulcastContent.select(simulcastAnimesSelector).map { a -> a.text().lowercase() }.toSet() + println("Found ${currentSimulcastAnimes.size} animes for the current simulcast") + + val previousSimulcastCode = getPreviousSimulcastCode(currentSimulcastCode) + println("Previous simulcast code for $it: $previousSimulcastCode") + + val previousSimulcastContent = httpRequest.getBrowser( + "https://www.crunchyroll.com/${it.name.lowercase()}/simulcasts/seasons/$previousSimulcastCode", + simulcastSelector + ) + val previousSimulcastAnimes = + previousSimulcastContent.select(simulcastAnimesSelector).map { a -> a.text().lowercase() }.toSet() + println("Found ${previousSimulcastAnimes.size} animes for the previous simulcast") + + val combinedSimulcasts = (currentSimulcastAnimes + previousSimulcastAnimes).toSet() + simulcasts.addAll(combinedSimulcasts) + } catch (e: Exception) { + e.printStackTrace() + } finally { + httpRequest.closeBrowser() + } + + println(simulcasts) + return@MapCache simulcasts } - override suspend fun fetchApiContent(zonedDateTime: ZonedDateTime): Api { - TODO("Not yet implemented") + val animeInfoCache = MapCache>(Duration.ofDays(1)) { + val httpRequest = HttpRequest() + var image: String? = null + var description: String? = null + + try { + val content = httpRequest.getBrowser( + "https://www.crunchyroll.com/${it.countryCode.name.lowercase()}/${it.animeId}", + "div.undefined:nth-child(1) > figure:nth-child(1) > picture:nth-child(1) > img:nth-child(2)" + ) + image = + content.selectXpath("//*[@id=\"content\"]/div/div[2]/div/div[1]/div[2]/div/div/div[2]/div[2]/figure/picture/img") + .attr("src") + description = + content.selectXpath("//*[@id=\"content\"]/div/div[2]/div/div[2]/div[1]/div[1]/div[5]/div/div/div/p") + .text() + } catch (e: Exception) { + e.printStackTrace() + } finally { + httpRequest.closeBrowser() + } + + if (image.isNullOrEmpty()) { + throw Exception("Image is null or empty") + } + + return@MapCache image to description } - override fun fetchEpisodes(zonedDateTime: ZonedDateTime): List { - return emptyList() + override fun getConfigurationClass() = CrunchyrollConfiguration::class.java + + override fun getPlatform(): Platform = Platform.CRUN + + override suspend fun fetchApiContent(key: CountryCode, zonedDateTime: ZonedDateTime): String { + val url = "https://www.crunchyroll.com/rss/anime?lang=${key.locale?.replace("-", "")}" + val response = HttpRequest().get(url) + + if (response.status != HttpStatusCode.OK) { + return "" + } + + return response.bodyAsText() + } + + override fun fetchEpisodes(zonedDateTime: ZonedDateTime, bypassFileContent: File?): List { + val list = mutableListOf() + + configuration!!.availableCountries.forEach { countryCode -> + var api = + if (bypassFileContent != null && bypassFileContent.exists()) bypassFileContent.readText() else getApiContent( + countryCode, + zonedDateTime + ) + api = api.replace(System.lineSeparator(), "").replace("\n", "") + val array = "(.*?)".toRegex().findAll(api).map { it.value } + + array.forEach { + try { + val xml = ObjectParser.fromXml(it, JsonObject::class.java) + list.add(convertEpisode(countryCode, xml)) + } catch (_: EpisodeException) { + // Ignore + } catch (_: AnimeException) { + // Ignore + } catch (e: Exception) { + e.printStackTrace() + } + } + } + + return list } override fun reset() { TODO("Not yet implemented") } + + override fun saveConfiguration() { + super.saveConfiguration() + simulcasts.resetWithNewDuration(Duration.ofMinutes(configuration!!.simulcastCheckDelayInMinutes)) + } + + private fun convertEpisode(countryCode: CountryCode, jsonObject: JsonObject): Episode { + val animeName = jsonObject.getAsString("crunchyroll:seriesTitle") ?: throw Exception("Anime name is null") + val restrictions = jsonObject.getAsJsonObject("media:restriction")?.getAsString("")?.split(" ") ?: emptyList() + + if (restrictions.isEmpty() || !restrictions.contains(countryCode.name.lowercase())) { + throw EpisodeNotAvailableInCountryException("Episode of $animeName is not available in ${countryCode.name}") + } + + val fullName = jsonObject.getAsString("title") ?: throw Exception("Episode title is null") + val isDubbed = fullName.contains("(${countryCode.voice})", true) + val isMovie = fullName.contains("movie", true) || fullName.contains("film", true) + + val subtitles = jsonObject.getAsString("crunchyroll:subtitleLanguages")?.split(",") ?: emptyList() + + if (!isDubbed && (subtitles.isEmpty() || !subtitles.contains( + countryCode.locale?.replace("-", " - ")?.lowercase() + )) + ) { + throw EpisodeNoSubtitlesOrVoiceException("Episode is not available in ${countryCode.name} with subtitles or voice") + } + + val langType = if (isDubbed) LangType.VOICE else LangType.SUBTITLES + + val id = jsonObject.getAsString("crunchyroll:mediaId") ?: throw Exception("Id is null") + val hash = "${countryCode}-${getPlatform()}-$id-$langType" + + if (hashCache.contains(hash)) { + throw EpisodeAlreadyReleasedException() + } + + val releaseDate = + jsonObject.getAsString("pubDate")?.let { ZonedDateTime.parse(it, DateTimeFormatter.RFC_1123_DATE_TIME) } + ?: throw Exception("Release date is null") + + val season = jsonObject.getAsString("crunchyroll:season")?.toIntOrNull() ?: 1 + + val number = jsonObject.getAsString("crunchyroll:episodeNumber")?.toIntOrNull() ?: -1 + + val episodeType = + if (isMovie) EpisodeType.FILM else if (number == -1) EpisodeType.SPECIAL else EpisodeType.EPISODE + + val title = jsonObject.getAsString("crunchyroll:episodeTitle")?.ifBlank { null } + + val url = jsonObject.getAsString("link")?.ifBlank { null } ?: throw Exception("Url is null") + + val images = jsonObject.getAsJsonArray("media:thumbnail") ?: throw Exception(IMAGE_NULL_ERROR) + val biggestImage = + images.maxByOrNull { it.asJsonObject.getAsInt("width")!! } ?: throw Exception(IMAGE_NULL_ERROR) + val image = biggestImage.asJsonObject.getAsString("url")?.ifBlank { null } ?: throw Exception(IMAGE_NULL_ERROR) + + val duration = jsonObject.getAsLong("crunchyroll:duration", -1) + + var isSimulcasted = + simulcasts[countryCode].contains(animeName.lowercase()) || configuration!!.simulcasts.contains(animeName) + isSimulcasted = isSimulcasted || isMovie + + if (!isSimulcasted) { + throw AnimeNotSimulcastedException("\"$animeName\" is not simulcasted") + } + + val splitted = url.split("/") + + if (splitted.size < 2) { + throw Exception("Anime id is null") + } + + val animeId = splitted[splitted.size - 2] + val (animeImage, animeDescription) = animeInfoCache[CountryCodeAnimeIdKeyCache(countryCode, animeId)] + + hashCache.add(hash) + + return Episode( + platform = getPlatform(), + anime = Anime( + countryCode = countryCode, + name = animeName, + releaseDate = releaseDate, + image = animeImage, + description = animeDescription + ), + episodeType = episodeType, + langType = langType, + hash = hash, + releaseDate = releaseDate, + season = season, + number = number, + title = title, + url = url, + image = image, + duration = duration + ) + } } \ No newline at end of file diff --git a/src/main/kotlin/fr/shikkanime/platforms/DisneyPlusPlatform.kt b/src/main/kotlin/fr/shikkanime/platforms/DisneyPlusPlatform.kt new file mode 100644 index 00000000..6041b910 --- /dev/null +++ b/src/main/kotlin/fr/shikkanime/platforms/DisneyPlusPlatform.kt @@ -0,0 +1,207 @@ +package fr.shikkanime.platforms + +import com.google.gson.JsonArray +import com.google.gson.JsonObject +import fr.shikkanime.caches.CountryCodeAnimeIdKeyCache +import fr.shikkanime.entities.Anime +import fr.shikkanime.entities.Episode +import fr.shikkanime.entities.enums.CountryCode +import fr.shikkanime.entities.enums.EpisodeType +import fr.shikkanime.entities.enums.LangType +import fr.shikkanime.entities.enums.Platform +import fr.shikkanime.exceptions.AnimeException +import fr.shikkanime.utils.HttpRequest +import fr.shikkanime.utils.ObjectParser +import fr.shikkanime.utils.ObjectParser.getAsBoolean +import fr.shikkanime.utils.ObjectParser.getAsInt +import fr.shikkanime.utils.ObjectParser.getAsLong +import fr.shikkanime.utils.ObjectParser.getAsString +import io.ktor.client.statement.* +import io.ktor.http.* +import java.io.File +import java.time.ZonedDateTime + +class DisneyPlusPlatform : + AbstractPlatform() { + data class DisneyPlusConfiguration( + var authorization: String = "", + var refreshToken: String = "", + ) : PlatformConfiguration() { + override fun of(parameters: Parameters) { + super.of(parameters) + parameters["authorization"]?.let { authorization = it } + parameters["refreshToken"]?.let { refreshToken = it } + } + + override fun toConfigurationFields(): MutableSet { + return super.toConfigurationFields().apply { + add( + ConfigurationField( + label = "Authorization", + name = "authorization", + type = "text", + value = authorization + ) + ) + add( + ConfigurationField( + label = "Refresh token", + name = "refreshToken", + type = "text", + value = refreshToken + ) + ) + } + } + } + + override fun getConfigurationClass() = DisneyPlusConfiguration::class.java + + override fun getPlatform(): Platform = Platform.DISN + + override suspend fun fetchApiContent(key: CountryCodeAnimeIdKeyCache, zonedDateTime: ZonedDateTime): JsonArray { + check(configuration!!.authorization.isNotBlank()) { "Authorization is null" } + check(configuration!!.refreshToken.isNotBlank()) { "Refresh token is null" } + val httpRequest = HttpRequest() + + val loginDevice = httpRequest.post( + "https://disney.api.edge.bamgrid.com/graph/v1/device/graphql", + mapOf( + "Authorization" to configuration!!.authorization, + ), + ObjectParser.toJson( + mapOf( + "operationName" to "refreshToken", + "query" to "mutation refreshToken(\$input:RefreshTokenInput!){refreshToken(refreshToken:\$input){activeSession{sessionId}}}", + "variables" to mapOf( + "input" to mapOf( + "refreshToken" to configuration!!.refreshToken + ) + ), + ) + ) + ) + + check(loginDevice.status.value == 200) { "Failed to login to Disney+" } + val loginDeviceJson = ObjectParser.fromJson(loginDevice.bodyAsText(), JsonObject::class.java) + val accessToken = loginDeviceJson.getAsJsonObject("extensions").getAsJsonObject("sdk").getAsJsonObject("token") + .getAsString("accessToken") + + val seasonsResponse = httpRequest.get( + "https://disney.content.edge.bamgrid.com/svc/content/DmcSeriesBundle/version/5.1/region/${key.countryCode.name}/audience/k-false,l-true/maturity/1850/language/${key.countryCode.locale}/encodedSeriesId/${key.animeId}", + mapOf("Authorization" to "Bearer $accessToken") + ) + check(seasonsResponse.status.value == 200) { "Failed to fetch Disney+ content" } + val seasonsJson = ObjectParser.fromJson(seasonsResponse.bodyAsText(), JsonObject::class.java) + val seasons = seasonsJson.getAsJsonObject("data").getAsJsonObject("DmcSeriesBundle").getAsJsonObject("seasons") + .getAsJsonArray("seasons").mapNotNull { it.asJsonObject.getAsString("seasonId") } + val episodes = JsonArray() + + seasons.forEach { season -> + var page = 1 + var hasMore: Boolean + + do { + val url = + "https://disney.content.edge.bamgrid.com/svc/content/DmcEpisodes/version/5.1/region/${key.countryCode.name}/audience/k-false,l-true/maturity/1850/language/${key.countryCode.locale}/seasonId/${season}/pageSize/15/page/${page++}" + val response = httpRequest.get(url, mapOf("Authorization" to "Bearer $accessToken")) + check(response.status.value == 200) { "Failed to fetch Disney+ content" } + val json = ObjectParser.fromJson(response.bodyAsText(), JsonObject::class.java) + + val dmcEpisodesMeta = json.getAsJsonObject("data").getAsJsonObject("DmcEpisodes") + hasMore = dmcEpisodesMeta.getAsJsonObject("meta").getAsBoolean("hasMore") ?: false + dmcEpisodesMeta.getAsJsonArray("videos").forEach { episodes.add(it) } + } while (hasMore) + } + + return episodes + } + + override fun fetchEpisodes(zonedDateTime: ZonedDateTime, bypassFileContent: File?): List { + val list = mutableListOf() + + configuration!!.availableCountries.forEach { countryCode -> + configuration!!.simulcasts.forEach { simulcast -> + val api = getApiContent(CountryCodeAnimeIdKeyCache(countryCode, simulcast), zonedDateTime) + + api.forEach { + try { + list.add(convertEpisode(countryCode, it.getAsJsonObject(), zonedDateTime)) + } catch (_: AnimeException) { + // Ignore + } catch (e: Exception) { + e.printStackTrace() + } + } + } + } + + return list + } + + override fun reset() { + TODO("Not yet implemented") + } + + private fun convertEpisode( + countryCode: CountryCode, + jsonObject: JsonObject, + zonedDateTime: ZonedDateTime + ): Episode { + val texts = jsonObject.getAsJsonObject("text") + val titles = texts?.getAsJsonObject("title")?.getAsJsonObject("full") + val animeName = titles?.getAsJsonObject("series")?.getAsJsonObject("default")?.getAsString("content") + ?: throw Exception("Anime name is null") + + val animeImage = jsonObject.getAsJsonObject("image")?.getAsJsonObject("tile")?.getAsJsonObject("0.71") + ?.getAsJsonObject("series")?.getAsJsonObject("default")?.getAsString("url") + ?: throw Exception("Anime image is null") + val animeDescription = + texts.getAsJsonObject("description")?.getAsJsonObject("medium")?.getAsJsonObject("series") + ?.getAsJsonObject("default")?.getAsString("content")?.replace('\n', ' ') ?: "" + + val season = jsonObject.getAsInt("seasonSequenceNumber") + + val number = jsonObject.getAsInt("episodeSequenceNumber") + + val id = jsonObject.getAsString("contentId") + + val title = + titles.getAsJsonObject("program")?.getAsJsonObject("default")?.getAsString("content")?.ifBlank { null } + + val url = "https://www.disneyplus.com/${countryCode.locale?.lowercase()}/video/$id" + + val image = jsonObject.getAsJsonObject("image")?.getAsJsonObject("thumbnail")?.getAsJsonObject("1.78") + ?.getAsJsonObject("program")?.getAsJsonObject("default")?.getAsString("url") + ?: throw Exception("Image is null") + + var duration = jsonObject.getAsJsonObject("mediaMetadata")?.getAsLong("runtimeMillis", -1) ?: -1 + + if (duration != -1L) { + duration /= 1000 + } + + val langType = LangType.SUBTITLES + + return Episode( + platform = getPlatform(), + anime = Anime( + countryCode = countryCode, + name = animeName, + releaseDate = zonedDateTime, + image = animeImage, + description = animeDescription, + ), + episodeType = EpisodeType.EPISODE, + langType = langType, + hash = "${countryCode}-${getPlatform()}-$id-$langType", + releaseDate = zonedDateTime, + season = season, + number = number, + title = title, + url = url, + image = image, + duration = duration, + ) + } +} \ No newline at end of file diff --git a/src/main/kotlin/fr/shikkanime/platforms/PlatformConfiguration.kt b/src/main/kotlin/fr/shikkanime/platforms/PlatformConfiguration.kt index 3d32d278..66b9bae1 100644 --- a/src/main/kotlin/fr/shikkanime/platforms/PlatformConfiguration.kt +++ b/src/main/kotlin/fr/shikkanime/platforms/PlatformConfiguration.kt @@ -1,5 +1,6 @@ package fr.shikkanime.platforms +import fr.shikkanime.entities.enums.CountryCode import io.ktor.http.* data class ConfigurationField( @@ -10,7 +11,7 @@ data class ConfigurationField( ) open class PlatformConfiguration( - var availableCountries: MutableSet = mutableSetOf(), + var availableCountries: MutableSet = mutableSetOf(), var apiCheckDelayInMinutes: Long = 0, var simulcasts: MutableSet = mutableSetOf(), ) { @@ -21,10 +22,12 @@ open class PlatformConfiguration( return@let } - availableCountries = it.split(",").toMutableSet() + availableCountries = CountryCode.from(it.split(",")) as MutableSet } - parameters["apiCheckDelayInMinutes"]?.let { apiCheckDelayInMinutes = it.toLong() } + parameters["apiCheckDelayInMinutes"]?.let { + apiCheckDelayInMinutes = it.toLong() + } (parameters.getAll("simulcasts") ?: emptyList()).let { val toMutableSet = it.toMutableSet() diff --git a/src/main/kotlin/fr/shikkanime/plugins/HTTP.kt b/src/main/kotlin/fr/shikkanime/plugins/HTTP.kt index 480cb7f5..a8a3baeb 100644 --- a/src/main/kotlin/fr/shikkanime/plugins/HTTP.kt +++ b/src/main/kotlin/fr/shikkanime/plugins/HTTP.kt @@ -2,7 +2,7 @@ package fr.shikkanime.plugins import freemarker.cache.ClassTemplateLoader import io.ktor.http.* -import io.ktor.serialization.gson.* +import io.ktor.serialization.jackson.* import io.ktor.server.application.* import io.ktor.server.freemarker.* import io.ktor.server.plugins.compression.* @@ -36,7 +36,7 @@ fun Application.configureHTTP() { } } install(ContentNegotiation) { - gson { + jackson { } } install(FreeMarker) { diff --git a/src/main/kotlin/fr/shikkanime/plugins/Routing.kt b/src/main/kotlin/fr/shikkanime/plugins/Routing.kt index 0d1b880c..75a5d97f 100644 --- a/src/main/kotlin/fr/shikkanime/plugins/Routing.kt +++ b/src/main/kotlin/fr/shikkanime/plugins/Routing.kt @@ -1,6 +1,5 @@ package fr.shikkanime.plugins -import fr.shikkanime.dtos.PlatformDto import fr.shikkanime.entities.Link import fr.shikkanime.utils.Constant import fr.shikkanime.utils.routes.* @@ -42,7 +41,8 @@ private fun Routing.createRoutes() { } fun Routing.createControllerRoutes(controller: Any) { - val prefix = if (controller::class.hasAnnotation()) controller::class.findAnnotation()!!.value else "" + val prefix = + if (controller::class.hasAnnotation()) controller::class.findAnnotation()!!.value else "" val kMethods = controller::class.declaredFunctions.filter { it.hasAnnotation() }.toMutableSet() route(prefix) { @@ -170,7 +170,6 @@ private suspend fun callMethodWithParameters( kParameter.hasAnnotation() -> { when (kParameter.type.javaType) { Array::class.java -> call.receive>() - PlatformDto::class.java -> call.receive() Parameters::class.java -> call.receiveParameters() else -> call.receive() } diff --git a/src/main/kotlin/fr/shikkanime/repositories/AnimeRepository.kt b/src/main/kotlin/fr/shikkanime/repositories/AnimeRepository.kt index e63e84df..26e8415c 100644 --- a/src/main/kotlin/fr/shikkanime/repositories/AnimeRepository.kt +++ b/src/main/kotlin/fr/shikkanime/repositories/AnimeRepository.kt @@ -1,11 +1,13 @@ package fr.shikkanime.repositories import fr.shikkanime.entities.Anime +import fr.shikkanime.entities.enums.CountryCode class AnimeRepository : AbstractRepository() { - fun findByName(name: String?): Anime? { + fun findByName(countryCode: CountryCode, name: String?): Anime? { return inTransaction { - it.createQuery("FROM Anime WHERE LOWER(name) = :name", getEntityClass()) + it.createQuery("FROM Anime WHERE countryCode = :countryCode AND LOWER(name) = :name", getEntityClass()) + .setParameter("countryCode", countryCode) .setParameter("name", name?.lowercase()) .resultList .firstOrNull() diff --git a/src/main/kotlin/fr/shikkanime/repositories/CountryRepository.kt b/src/main/kotlin/fr/shikkanime/repositories/CountryRepository.kt deleted file mode 100644 index 4767f00a..00000000 --- a/src/main/kotlin/fr/shikkanime/repositories/CountryRepository.kt +++ /dev/null @@ -1,31 +0,0 @@ -package fr.shikkanime.repositories - -import fr.shikkanime.entities.Country - -class CountryRepository : AbstractRepository() { - fun findByName(name: String): Country? { - return inTransaction { - it.createQuery("FROM Country WHERE name = :name", getEntityClass()) - .setParameter("name", name) - .resultList - .firstOrNull() - } - } - - fun findByCode(code: String): Country? { - return inTransaction { - it.createQuery("FROM Country WHERE countryCode = :code", getEntityClass()) - .setParameter("code", code) - .resultList - .firstOrNull() - } - } - - fun findAllByCode(codes: List): List { - return inTransaction { - it.createQuery("FROM Country WHERE countryCode IN :codes", getEntityClass()) - .setParameter("codes", codes) - .resultList - } - } -} \ No newline at end of file diff --git a/src/main/kotlin/fr/shikkanime/repositories/EpisodeRepository.kt b/src/main/kotlin/fr/shikkanime/repositories/EpisodeRepository.kt index ff5ea3dc..a735a382 100644 --- a/src/main/kotlin/fr/shikkanime/repositories/EpisodeRepository.kt +++ b/src/main/kotlin/fr/shikkanime/repositories/EpisodeRepository.kt @@ -3,6 +3,13 @@ package fr.shikkanime.repositories import fr.shikkanime.entities.Episode class EpisodeRepository : AbstractRepository() { + fun findAllHashes(): List { + return inTransaction { + it.createQuery("SELECT hash FROM Episode", String::class.java) + .resultList + } + } + fun findByHash(hash: String?): Episode? { return inTransaction { it.createQuery("FROM Episode WHERE LOWER(hash) = :hash", getEntityClass()) diff --git a/src/main/kotlin/fr/shikkanime/repositories/PlatformRepository.kt b/src/main/kotlin/fr/shikkanime/repositories/PlatformRepository.kt deleted file mode 100644 index 0732621b..00000000 --- a/src/main/kotlin/fr/shikkanime/repositories/PlatformRepository.kt +++ /dev/null @@ -1,14 +0,0 @@ -package fr.shikkanime.repositories - -import fr.shikkanime.entities.Platform - -class PlatformRepository : AbstractRepository() { - fun findByName(name: String): Platform? { - return inTransaction { - it.createQuery("FROM Platform WHERE name = :name", getEntityClass()) - .setParameter("name", name) - .resultList - .firstOrNull() - } - } -} \ No newline at end of file diff --git a/src/main/kotlin/fr/shikkanime/services/AbstractService.kt b/src/main/kotlin/fr/shikkanime/services/AbstractService.kt index 9b600c01..f0501cec 100644 --- a/src/main/kotlin/fr/shikkanime/services/AbstractService.kt +++ b/src/main/kotlin/fr/shikkanime/services/AbstractService.kt @@ -15,7 +15,5 @@ abstract class AbstractService> { protected fun update(entity: E) = getRepository().update(entity) - open fun saveOrUpdate(entity: E) = entity - fun delete(entity: E) = getRepository().delete(entity) } \ No newline at end of file diff --git a/src/main/kotlin/fr/shikkanime/services/AnimeService.kt b/src/main/kotlin/fr/shikkanime/services/AnimeService.kt index c6e28c03..60790f47 100644 --- a/src/main/kotlin/fr/shikkanime/services/AnimeService.kt +++ b/src/main/kotlin/fr/shikkanime/services/AnimeService.kt @@ -2,6 +2,7 @@ package fr.shikkanime.services import com.google.inject.Inject import fr.shikkanime.entities.Anime +import fr.shikkanime.entities.enums.CountryCode import fr.shikkanime.repositories.AnimeRepository class AnimeService : AbstractService() { @@ -12,21 +13,7 @@ class AnimeService : AbstractService() { return animeRepository } - fun findByName(name: String?): Anime? { - return animeRepository.findByName(name) - } - - override fun saveOrUpdate(entity: Anime): Anime { - val entityFromDb = findByName(entity.name) ?: return save(entity) - - if (entityFromDb.image != entity.image) { - entityFromDb.image = entity.image - } - - if (entityFromDb.description != entity.description) { - entityFromDb.description = entity.description - } - - return super.update(entityFromDb) + fun findByName(countryCode: CountryCode, name: String?): Anime? { + return animeRepository.findByName(countryCode, name) } } \ No newline at end of file diff --git a/src/main/kotlin/fr/shikkanime/services/CountryService.kt b/src/main/kotlin/fr/shikkanime/services/CountryService.kt deleted file mode 100644 index d8c6e207..00000000 --- a/src/main/kotlin/fr/shikkanime/services/CountryService.kt +++ /dev/null @@ -1,31 +0,0 @@ -package fr.shikkanime.services - -import com.google.inject.Inject -import fr.shikkanime.entities.Country -import fr.shikkanime.repositories.CountryRepository -import fr.shikkanime.utils.MapCache - -class CountryService : AbstractService() { - @Inject - private lateinit var countryRepository: CountryRepository - - private val codesCache: MapCache> = MapCache { - countryRepository.findAllByCode(it.split(",")) - } - - override fun getRepository(): CountryRepository { - return countryRepository - } - - fun findByName(name: String): Country? { - return countryRepository.findByName(name) - } - - fun findByCode(code: String): Country? { - return countryRepository.findByCode(code) - } - - fun findAllByCode(codes: Collection): List { - return codesCache[codes.joinToString(",")] - } -} \ No newline at end of file diff --git a/src/main/kotlin/fr/shikkanime/services/EpisodeService.kt b/src/main/kotlin/fr/shikkanime/services/EpisodeService.kt index fd5d6742..dd6601a6 100644 --- a/src/main/kotlin/fr/shikkanime/services/EpisodeService.kt +++ b/src/main/kotlin/fr/shikkanime/services/EpisodeService.kt @@ -8,9 +8,6 @@ class EpisodeService : AbstractService() { @Inject private lateinit var episodeRepository: EpisodeRepository - @Inject - private lateinit var platformService: PlatformService - @Inject private lateinit var animeService: AnimeService @@ -18,40 +15,17 @@ class EpisodeService : AbstractService() { return episodeRepository } + fun findAllHashes(): List { + return episodeRepository.findAllHashes() + } + fun findByHash(hash: String?): Episode? { return episodeRepository.findByHash(hash) } - override fun saveOrUpdate(entity: Episode): Episode { - entity.platform = platformService.saveOrUpdate(entity.platform!!) - entity.anime = animeService.saveOrUpdate(entity.anime!!) - - val entityFromDb = findByHash(entity.hash) ?: return save(entity) - - if (entityFromDb.season != entity.season) { - entityFromDb.season = entity.season - } - - if (entityFromDb.number != entity.number) { - entityFromDb.number = entity.number - } - - if (entityFromDb.title != entity.title) { - entityFromDb.title = entity.title - } - - if (entityFromDb.url != entity.url) { - entityFromDb.url = entity.url - } - - if (entityFromDb.image != entity.image) { - entityFromDb.image = entity.image - } - - if (entityFromDb.duration != entity.duration) { - entityFromDb.duration = entity.duration - } - - return super.update(entityFromDb) + override fun save(entity: Episode): Episode { + entity.anime = animeService.findByName(entity.anime!!.countryCode!!, entity.anime!!.name!!) + ?: animeService.save(entity.anime!!) + return super.save(entity) } } \ No newline at end of file diff --git a/src/main/kotlin/fr/shikkanime/services/ImageService.kt b/src/main/kotlin/fr/shikkanime/services/ImageService.kt new file mode 100644 index 00000000..3e372a37 --- /dev/null +++ b/src/main/kotlin/fr/shikkanime/services/ImageService.kt @@ -0,0 +1,121 @@ +package fr.shikkanime.services + +import fr.shikkanime.utils.CompressionManager +import fr.shikkanime.utils.HttpRequest +import fr.shikkanime.utils.ObjectParser +import io.ktor.client.statement.* +import io.ktor.http.* +import kotlinx.coroutines.runBlocking +import java.io.File +import java.util.* +import java.util.concurrent.Executors +import kotlin.system.measureTimeMillis + +object ImageService { + data class Image( + val url: String, + var bytes: ByteArray = byteArrayOf(), + var ratio: Double = 0.0, + ) { + override fun equals(other: Any?): Boolean { + if (this === other) return true + if (javaClass != other?.javaClass) return false + + other as Image + + if (url != other.url) return false + if (!bytes.contentEquals(other.bytes)) return false + if (ratio != other.ratio) return false + + return true + } + + override fun hashCode(): Int { + var result = url.hashCode() + result = 31 * result + bytes.contentHashCode() + result = 31 * result + ratio.hashCode() + return result + } + } + + private val threadPool = Executors.newFixedThreadPool(Runtime.getRuntime().availableProcessors() - 1) + private val file = File("images-cache.shikk") + private var cache = mutableMapOf() + + fun loadCache() { + if (!file.exists()) { + saveCache() + return + } + + println("Loading images cache...") + + val take = measureTimeMillis { + val json = String(CompressionManager.fromGzip(file.readBytes())) + val map = ObjectParser.fromJson(json, mutableMapOf().javaClass) + cache = map + } + + println("Loaded images cache in $take ms") + } + + fun saveCache() { + if (!file.exists()) { + file.createNewFile() + } + + println("Saving images cache...") + + val take = measureTimeMillis { + file.writeBytes(CompressionManager.toGzip(ObjectParser.toJson(cache).toByteArray())) + } + + println("Saved images cache in $take ms") + } + + fun contains(uuid: UUID) = cache.containsKey(uuid) + + fun add(uuid: UUID, url: String) { + val image = Image(url) + cache[uuid] = image + + threadPool.submit { + val take = measureTimeMillis { + try { + println("Loading image $url...") + val httpResponse = runBlocking { + HttpRequest().get(url) + } + + if (httpResponse.status != HttpStatusCode.OK) { + println("Failed to load image $url") + cache.remove(uuid) + return@measureTimeMillis + } + + val bytes = runBlocking { + httpResponse.readBytes() + } + + if (bytes.isEmpty()) { + println("Failed to load image $url") + cache.remove(uuid) + return@measureTimeMillis + } + + println("Encoding image to WebP...") + val webp = CompressionManager.encodeToWebP(bytes) + image.bytes = webp + image.ratio = bytes.size.toDouble() / webp.size.toDouble() + cache[uuid] = image + } catch (e: Exception) { + println("Failed to load image $url: ${e.message}") + cache.remove(uuid) + } + } + + println("Ratio for $url: ${String.format("%.2f", image.ratio * 100)}%, took $take ms") + println("Encoded image to WebP in ${take}ms") + } + } +} \ No newline at end of file diff --git a/src/main/kotlin/fr/shikkanime/services/MetricService.kt b/src/main/kotlin/fr/shikkanime/services/MetricService.kt index 9aa5cfb8..f1d61483 100644 --- a/src/main/kotlin/fr/shikkanime/services/MetricService.kt +++ b/src/main/kotlin/fr/shikkanime/services/MetricService.kt @@ -7,21 +7,16 @@ import fr.shikkanime.repositories.MetricRepository import fr.shikkanime.utils.MapCache import java.time.Duration import java.time.ZonedDateTime -import java.time.temporal.ChronoUnit class MetricService : AbstractService() { @Inject private lateinit var metricRepository: MetricRepository - private val averageCpuLoadCache = MapCache( - Duration.of(1, ChronoUnit.HOURS) - ) { + private val averageCpuLoadCache = MapCache(Duration.ofHours(1)) { metricRepository.getAverageCpuLoad(it.from, it.to) } - private val averageMemoryUsageCache = MapCache( - Duration.of(1, ChronoUnit.HOURS) - ) { + private val averageMemoryUsageCache = MapCache(Duration.ofHours(1)) { metricRepository.getAverageMemoryUsage(it.from, it.to) } diff --git a/src/main/kotlin/fr/shikkanime/services/PlatformService.kt b/src/main/kotlin/fr/shikkanime/services/PlatformService.kt deleted file mode 100644 index 9f35db22..00000000 --- a/src/main/kotlin/fr/shikkanime/services/PlatformService.kt +++ /dev/null @@ -1,36 +0,0 @@ -package fr.shikkanime.services - -import com.google.inject.Inject -import fr.shikkanime.entities.Platform -import fr.shikkanime.repositories.PlatformRepository - -class PlatformService : AbstractService() { - @Inject - private lateinit var platformRepository: PlatformRepository - - override fun getRepository(): PlatformRepository { - return platformRepository - } - - fun findByName(name: String?): Platform? { - if (name.isNullOrBlank()) { - return null - } - - return platformRepository.findByName(name) - } - - override fun saveOrUpdate(entity: Platform): Platform { - val entityFromDb = findByName(entity.name) ?: return save(entity) - - if (entityFromDb.url != entity.url) { - entityFromDb.url = entity.url - } - - if (entityFromDb.image != entity.image) { - entityFromDb.image = entity.image - } - - return super.update(entityFromDb) - } -} \ No newline at end of file diff --git a/src/main/kotlin/fr/shikkanime/utils/CompressionManager.kt b/src/main/kotlin/fr/shikkanime/utils/CompressionManager.kt new file mode 100644 index 00000000..ac039348 --- /dev/null +++ b/src/main/kotlin/fr/shikkanime/utils/CompressionManager.kt @@ -0,0 +1,43 @@ +package fr.shikkanime.utils + +import nu.pattern.OpenCV +import org.opencv.core.MatOfByte +import org.opencv.core.MatOfInt +import org.opencv.imgcodecs.Imgcodecs +import java.io.ByteArrayInputStream +import java.io.ByteArrayOutputStream +import java.util.zip.GZIPInputStream +import java.util.zip.GZIPOutputStream + +object CompressionManager { + init { + OpenCV.loadLocally() + } + + fun toGzip(bytes: ByteArray): ByteArray { + val byteArrayOutputStream = ByteArrayOutputStream() + val gzip = GZIPOutputStream(byteArrayOutputStream) + gzip.write(bytes) + gzip.close() + return byteArrayOutputStream.toByteArray() + } + + fun fromGzip(bytes: ByteArray): ByteArray { + val gzip = GZIPInputStream(ByteArrayInputStream(bytes)) + val compressed = gzip.readBytes() + gzip.close() + return compressed + } + + fun encodeToWebP(image: ByteArray): ByteArray { + val matImage = Imgcodecs.imdecode(MatOfByte(*image), Imgcodecs.IMREAD_UNCHANGED) + val parameters = MatOfInt(Imgcodecs.IMWRITE_WEBP_QUALITY, 75) + val output = MatOfByte() + + if (Imgcodecs.imencode(".webp", matImage, output, parameters)) { + return output.toArray() + } else { + throw Exception("Failed to encode image to WebP") + } + } +} \ No newline at end of file diff --git a/src/main/kotlin/fr/shikkanime/utils/Constant.kt b/src/main/kotlin/fr/shikkanime/utils/Constant.kt index cd05d2c6..33aeb7c9 100644 --- a/src/main/kotlin/fr/shikkanime/utils/Constant.kt +++ b/src/main/kotlin/fr/shikkanime/utils/Constant.kt @@ -1,6 +1,5 @@ package fr.shikkanime.utils -import com.google.gson.Gson import com.google.inject.Guice import com.google.inject.Injector import fr.shikkanime.modules.DefaultModule @@ -8,7 +7,6 @@ import fr.shikkanime.platforms.AbstractPlatform import org.reflections.Reflections object Constant { - val gson = Gson() val reflections = Reflections("fr.shikkanime") val injector: Injector = Guice.createInjector(DefaultModule()) diff --git a/src/main/kotlin/fr/shikkanime/utils/Database.kt b/src/main/kotlin/fr/shikkanime/utils/Database.kt index cc9f12c9..8dc978d9 100644 --- a/src/main/kotlin/fr/shikkanime/utils/Database.kt +++ b/src/main/kotlin/fr/shikkanime/utils/Database.kt @@ -48,7 +48,11 @@ class Database { } } - constructor() : this(File("hibernate.cfg.xml")) + constructor() : this( + File( + ClassLoader.getSystemClassLoader().getResource("hibernate.cfg.xml")?.file ?: "hibernate.cfg.xml" + ) + ) fun getEntityManager(): EntityManager { return sessionFactory.createEntityManager() diff --git a/src/main/kotlin/fr/shikkanime/utils/HttpRequest.kt b/src/main/kotlin/fr/shikkanime/utils/HttpRequest.kt index bbc8c7f3..c780ee56 100644 --- a/src/main/kotlin/fr/shikkanime/utils/HttpRequest.kt +++ b/src/main/kotlin/fr/shikkanime/utils/HttpRequest.kt @@ -1,19 +1,98 @@ package fr.shikkanime.utils +import com.microsoft.playwright.Browser +import com.microsoft.playwright.BrowserType +import com.microsoft.playwright.Page +import com.microsoft.playwright.Playwright import io.ktor.client.* import io.ktor.client.engine.cio.* import io.ktor.client.request.* import io.ktor.client.statement.* +import org.jsoup.Jsoup +import org.jsoup.nodes.Document +import kotlin.system.measureTimeMillis class HttpRequest { - suspend fun get(url: String): HttpResponse { + private var isBrowserInitialized = false + private var playwright: Playwright? = null + private var browser: Browser? = null + private var page: Page? = null + + suspend fun get(url: String, headers: Map = emptyMap()): HttpResponse { val httpClient = HttpClient(CIO) println("Making request to $url... (GET)") val start = System.currentTimeMillis() - val response = httpClient.get(url) + val response = httpClient.get(url) { + headers.forEach { (key, value) -> + header(key, value) + } + } httpClient.close() println("Request to $url done in ${System.currentTimeMillis() - start}ms (GET)") return response } + + suspend fun post(url: String, headers: Map = emptyMap(), body: String): HttpResponse { + val httpClient = HttpClient(CIO) + println("Making request to $url... (POST)") + val start = System.currentTimeMillis() + val response = httpClient.post(url) { + headers.forEach { (key, value) -> + header(key, value) + } + + setBody(body) + } + httpClient.close() + println("Request to $url done in ${System.currentTimeMillis() - start}ms (POST)") + return response + } + + private fun initBrowser() { + if (isBrowserInitialized) { + return + } + + playwright = Playwright.create() + browser = playwright?.firefox()?.launch(BrowserType.LaunchOptions().setHeadless(true)) + page = browser?.newPage() + page?.setDefaultTimeout(15_000.0) + page?.setDefaultNavigationTimeout(15_000.0) + isBrowserInitialized = true + } + + fun getBrowser(url: String, selector: String? = null, retry: Int = 3): Document { + initBrowser() + println("Making request to $url... (BROWSER)") + + val takeMs = measureTimeMillis { + try { + page?.navigate(url) + + if (selector != null) { + page?.waitForSelector(selector) + } else { + page?.waitForLoadState() + } + } catch (e: Exception) { + if (retry > 0) { + println("Retrying...") + return getBrowser(url, selector, retry - 1) + } + + throw e + } + } + + val content = page?.content() + println("Request to $url done in ${takeMs}ms (BROWSER)") + return Jsoup.parse(content ?: throw Exception("Content is null")) + } + + fun closeBrowser() { + page?.close() + browser?.close() + playwright?.close() + } } \ No newline at end of file diff --git a/src/main/kotlin/fr/shikkanime/utils/MapCache.kt b/src/main/kotlin/fr/shikkanime/utils/MapCache.kt index faf499b5..92d29a35 100644 --- a/src/main/kotlin/fr/shikkanime/utils/MapCache.kt +++ b/src/main/kotlin/fr/shikkanime/utils/MapCache.kt @@ -6,17 +6,22 @@ import com.google.common.cache.LoadingCache import java.time.Duration class MapCache( - duration: Duration? = null, + private var duration: Duration? = null, private val classes: List> = listOf(), private val block: (K) -> V, ) { - private val cache: LoadingCache + private lateinit var cache: LoadingCache init { + setCache() + globalCaches.add(this) + } + + private fun MapCache.setCache() { val builder = CacheBuilder.newBuilder() if (duration != null) { - builder.expireAfterWrite(duration) + builder.expireAfterWrite(duration!!) } cache = builder @@ -25,20 +30,31 @@ class MapCache( return block(key) } }) + } - caches.add(this) + fun has(key: K): Boolean { + return cache.getIfPresent(key) != null } operator fun get(key: K): V { return cache.getUnchecked(key) } + operator fun set(key: K, value: V & Any) { + cache.put(key, value) + } + + fun resetWithNewDuration(duration: Duration) { + this.duration = duration + setCache() + } + companion object { - private val caches: MutableList> = mutableListOf() + private val globalCaches: MutableList> = mutableListOf() fun invalidate(vararg classes: Class<*>) { classes.forEach { clazz -> - caches.filter { it.classes.contains(clazz) }.forEach { it.cache.invalidateAll() } + globalCaches.filter { it.classes.contains(clazz) }.forEach { it.cache.invalidateAll() } } } } diff --git a/src/main/kotlin/fr/shikkanime/utils/ObjectParser.kt b/src/main/kotlin/fr/shikkanime/utils/ObjectParser.kt new file mode 100644 index 00000000..ec1a3ac9 --- /dev/null +++ b/src/main/kotlin/fr/shikkanime/utils/ObjectParser.kt @@ -0,0 +1,48 @@ +package fr.shikkanime.utils + +import com.ctc.wstx.stax.WstxInputFactory +import com.fasterxml.jackson.databind.ObjectMapper +import com.fasterxml.jackson.dataformat.xml.XmlMapper +import com.google.gson.Gson +import com.google.gson.JsonObject + +object ObjectParser { + private val gson = Gson() + private val objectMapper = ObjectMapper() + private val xmlInputFactory = WstxInputFactory() + private val xmlMapper: XmlMapper + + init { + xmlInputFactory.configureForSpeed() + xmlInputFactory.setProperty(WstxInputFactory.IS_NAMESPACE_AWARE, false) + xmlMapper = XmlMapper(xmlInputFactory) + } + + fun fromJson(json: String, clazz: Class): T { + return gson.fromJson(json, clazz) + } + + fun toJson(obj: T): String { + return gson.toJson(obj) + } + + fun JsonObject.getAsString(key: String): String? { + return this[key]?.asString + } + + fun JsonObject.getAsBoolean(key: String): Boolean? { + return this[key]?.asBoolean + } + + fun JsonObject.getAsInt(key: String): Int? { + return this[key]?.asInt + } + + fun JsonObject.getAsLong(key: String, default: Long): Long { + return this[key]?.asLong ?: default + } + + fun fromXml(xml: String, clazz: Class): T { + return objectMapper.writeValueAsString(xmlMapper.readTree(xml)).let { gson.fromJson(it, clazz) } + } +} \ No newline at end of file diff --git a/src/main/resources/db/changelog/2023/12/01-changelog.xml b/src/main/resources/db/changelog/2023/12/01-changelog.xml index d2f3449d..4153631f 100644 --- a/src/main/resources/db/changelog/2023/12/01-changelog.xml +++ b/src/main/resources/db/changelog/2023/12/01-changelog.xml @@ -3,7 +3,8 @@ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns="http://www.liquibase.org/xml/ns/dbchangelog" xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog - http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-4.25.xsd" objectQuotingStrategy="QUOTE_ONLY_RESERVED_WORDS"> + http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-4.25.xsd" + objectQuotingStrategy="QUOTE_ONLY_RESERVED_WORDS"> @@ -38,74 +39,6 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - SELECT COUNT(*) - FROM country - WHERE country_code = 'FR' - - - - - France - FR - - - - @@ -119,8 +52,8 @@ primaryKey="true" primaryKeyName="pk_anime"/> - + + type="VARCHAR(1000)"> + type="VARCHAR(1000)"/> - - - - - - - - - - - + @@ -168,8 +87,8 @@ primaryKey="true" primaryKeyName="pk_episode"/> - + - + @@ -201,13 +120,13 @@ + type="VARCHAR(1000)"/> + type="VARCHAR(1000)"> + type="VARCHAR(1000)"> - + @@ -230,18 +149,4 @@ referencedColumnNames="uuid" referencedTableName="anime"/> - - - - - - - - - - \ No newline at end of file diff --git a/src/main/resources/db/changelog/db.changelog-master.xml b/src/main/resources/db/changelog/db.changelog-master.xml index a1282c8f..3c4a299a 100644 --- a/src/main/resources/db/changelog/db.changelog-master.xml +++ b/src/main/resources/db/changelog/db.changelog-master.xml @@ -3,6 +3,7 @@ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns="http://www.liquibase.org/xml/ns/dbchangelog" xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog - http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-4.25.xsd" objectQuotingStrategy="QUOTE_ONLY_RESERVED_WORDS"> + http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-4.25.xsd" + objectQuotingStrategy="QUOTE_ONLY_RESERVED_WORDS"> \ No newline at end of file diff --git a/src/main/resources/templates/admin/_layout.ftl b/src/main/resources/templates/admin/_layout.ftl index 15fd11d6..269dad6d 100644 --- a/src/main/resources/templates/admin/_layout.ftl +++ b/src/main/resources/templates/admin/_layout.ftl @@ -10,9 +10,10 @@ - - + +