diff --git a/.gitignore b/.gitignore
index e1451812..807c3ff1 100644
--- a/.gitignore
+++ b/.gitignore
@@ -37,3 +37,4 @@ out/
/metrics.json
/.jpb/
/config/
+/images-cache.shikk
diff --git a/build.gradle.kts b/build.gradle.kts
index 80b9c2d6..4a59f49f 100644
--- a/build.gradle.kts
+++ b/build.gradle.kts
@@ -30,7 +30,7 @@ dependencies {
implementation("io.ktor:ktor-server-compression-jvm:$ktor_version")
implementation("io.ktor:ktor-server-cors-jvm:$ktor_version")
implementation("io.ktor:ktor-server-content-negotiation-jvm:$ktor_version")
- implementation("io.ktor:ktor-serialization-gson:$ktor_version")
+ implementation("io.ktor:ktor-serialization-jackson:$ktor_version")
implementation("io.ktor:ktor-server-freemarker-jvm:$ktor_version")
implementation("io.ktor:ktor-server-netty-jvm:$ktor_version")
implementation("io.ktor:ktor-client-core:$ktor_version")
@@ -42,6 +42,12 @@ dependencies {
implementation("org.liquibase:liquibase-core:4.25.0")
implementation("org.quartz-scheduler:quartz:2.5.0-rc1")
implementation("com.google.guava:guava:32.1.3-jre")
+ implementation("com.fasterxml.jackson.dataformat:jackson-dataformat-xml:2.16.0")
+ implementation("com.microsoft.playwright:playwright:1.40.0")
+ implementation("org.jsoup:jsoup:1.17.1")
+ implementation("com.google.code.gson:gson:2.10.1")
+ implementation("org.openpnp:opencv:4.7.0-0")
testImplementation("io.ktor:ktor-server-tests-jvm:$ktor_version")
testImplementation("org.jetbrains.kotlin:kotlin-test-junit:$kotlin_version")
+ testImplementation("com.h2database:h2:2.2.224")
}
diff --git a/hibernate.cfg.xml b/hibernate.cfg.xml
index 6e5bd88f..425cd7fa 100644
--- a/hibernate.cfg.xml
+++ b/hibernate.cfg.xml
@@ -8,7 +8,7 @@
jdbc:postgresql://localhost:5432/shikkanime
postgres
mysecretpassword
- true
+ false
org.hibernate.context.internal.ThreadLocalSessionContext
\ No newline at end of file
diff --git a/src/main/kotlin/fr/shikkanime/Application.kt b/src/main/kotlin/fr/shikkanime/Application.kt
index bcc41d47..e231459c 100644
--- a/src/main/kotlin/fr/shikkanime/Application.kt
+++ b/src/main/kotlin/fr/shikkanime/Application.kt
@@ -3,18 +3,23 @@ package fr.shikkanime
import fr.shikkanime.jobs.FetchEpisodesJob
import fr.shikkanime.jobs.GCJob
import fr.shikkanime.jobs.MetricJob
+import fr.shikkanime.jobs.SavingImageCacheJob
import fr.shikkanime.plugins.configureHTTP
import fr.shikkanime.plugins.configureRouting
import fr.shikkanime.plugins.configureSecurity
+import fr.shikkanime.services.ImageService
import fr.shikkanime.utils.JobManager
import io.ktor.server.application.*
import io.ktor.server.engine.*
import io.ktor.server.netty.*
fun main() {
+ ImageService.loadCache()
+
JobManager.scheduleJob("*/10 * * * * ?", MetricJob::class.java)
JobManager.scheduleJob("0 */5 * * * ?", GCJob::class.java)
JobManager.scheduleJob("0 * * * * ?", FetchEpisodesJob::class.java)
+ JobManager.scheduleJob("0 */5 * * * ?", SavingImageCacheJob::class.java)
JobManager.start()
embeddedServer(
diff --git a/src/main/kotlin/fr/shikkanime/caches/CountryCodeAnimeIdKeyCache.kt b/src/main/kotlin/fr/shikkanime/caches/CountryCodeAnimeIdKeyCache.kt
new file mode 100644
index 00000000..57f8d266
--- /dev/null
+++ b/src/main/kotlin/fr/shikkanime/caches/CountryCodeAnimeIdKeyCache.kt
@@ -0,0 +1,26 @@
+package fr.shikkanime.caches
+
+import fr.shikkanime.entities.enums.CountryCode
+
+data class CountryCodeAnimeIdKeyCache(
+ val countryCode: CountryCode,
+ val animeId: String,
+) {
+ override fun equals(other: Any?): Boolean {
+ if (this === other) return true
+ if (javaClass != other?.javaClass) return false
+
+ other as CountryCodeAnimeIdKeyCache
+
+ if (countryCode != other.countryCode) return false
+ if (animeId != other.animeId) return false
+
+ return true
+ }
+
+ override fun hashCode(): Int {
+ var result = countryCode.hashCode()
+ result = 31 * result + animeId.hashCode()
+ return result
+ }
+}
diff --git a/src/main/kotlin/fr/shikkanime/controllers/AdminController.kt b/src/main/kotlin/fr/shikkanime/controllers/AdminController.kt
index b564eb78..2a78e62d 100644
--- a/src/main/kotlin/fr/shikkanime/controllers/AdminController.kt
+++ b/src/main/kotlin/fr/shikkanime/controllers/AdminController.kt
@@ -23,7 +23,8 @@ class AdminController {
return TemplateResponse(
"admin/platforms.ftl",
"Platforms",
- mutableMapOf("platforms" to Constant.abstractPlatforms)
+ mutableMapOf(
+ "platforms" to Constant.abstractPlatforms.toList().sortedBy { it.getPlatform().name.lowercase() })
)
}
@@ -34,7 +35,8 @@ class AdminController {
val redirectResponse = RedirectResponse("/admin/platforms")
val platformName = parameters["platform"] ?: return redirectResponse
- val abstractPlatform = Constant.abstractPlatforms.find { it.getPlatform().name == platformName } ?: return redirectResponse
+ val abstractPlatform =
+ Constant.abstractPlatforms.find { it.getPlatform().name == platformName } ?: return redirectResponse
abstractPlatform.configuration?.of(parameters)
abstractPlatform.saveConfiguration()
return redirectResponse
diff --git a/src/main/kotlin/fr/shikkanime/controllers/api/PlatformController.kt b/src/main/kotlin/fr/shikkanime/controllers/api/PlatformController.kt
deleted file mode 100644
index 96eb9c60..00000000
--- a/src/main/kotlin/fr/shikkanime/controllers/api/PlatformController.kt
+++ /dev/null
@@ -1,43 +0,0 @@
-package fr.shikkanime.controllers.api
-
-import com.google.inject.Inject
-import fr.shikkanime.converters.AbstractConverter
-import fr.shikkanime.dtos.PlatformDto
-import fr.shikkanime.entities.Platform
-import fr.shikkanime.services.PlatformService
-import fr.shikkanime.utils.routes.BodyParam
-import fr.shikkanime.utils.routes.Controller
-import fr.shikkanime.utils.routes.Path
-import fr.shikkanime.utils.routes.Response
-import fr.ziedelth.utils.routes.method.Get
-import fr.ziedelth.utils.routes.method.Post
-import java.util.*
-
-@Controller("/api/platforms")
-class PlatformController {
- @Inject
- private lateinit var platformService: PlatformService
-
- @Path
- @Get
- private fun getPlatforms(): Response {
- return Response.ok(AbstractConverter.convert(platformService.findAll(), PlatformDto::class.java))
- }
-
- @Path("/{uuid}")
- @Get
- private fun getPlatform(uuid: UUID): Response {
- return Response.ok(AbstractConverter.convert(platformService.find(uuid), PlatformDto::class.java))
- }
-
- @Path
- @Post
- private fun createPlatform(@BodyParam platformDto: PlatformDto): Response {
- return Response.ok(
- AbstractConverter.convert(
- platformService.save(AbstractConverter.convert(platformDto, Platform::class.java)),
- PlatformDto::class.java
- )
- )
- }
-}
\ No newline at end of file
diff --git a/src/main/kotlin/fr/shikkanime/converters/AbstractConverter.kt b/src/main/kotlin/fr/shikkanime/converters/AbstractConverter.kt
index 162357f2..4dd6868c 100644
--- a/src/main/kotlin/fr/shikkanime/converters/AbstractConverter.kt
+++ b/src/main/kotlin/fr/shikkanime/converters/AbstractConverter.kt
@@ -36,7 +36,10 @@ abstract class AbstractConverter {
return try {
method.invoke(abstractConverter, `object`) as T
} catch (e: Exception) {
- throw IllegalStateException("Can not convert \"${`object`.javaClass.simpleName}\" to \"${to.simpleName}\"", e)
+ throw IllegalStateException(
+ "Can not convert \"${`object`.javaClass.simpleName}\" to \"${to.simpleName}\"",
+ e
+ )
}
}
diff --git a/src/main/kotlin/fr/shikkanime/converters/metric/MetricToMetricDtoConverter.kt b/src/main/kotlin/fr/shikkanime/converters/metric/MetricToMetricDtoConverter.kt
index 69a8e9db..3446cc73 100644
--- a/src/main/kotlin/fr/shikkanime/converters/metric/MetricToMetricDtoConverter.kt
+++ b/src/main/kotlin/fr/shikkanime/converters/metric/MetricToMetricDtoConverter.kt
@@ -23,9 +23,11 @@ class MetricToMetricDtoConverter : AbstractConverter() {
return MetricDto(
uuid = from.uuid,
cpuLoad = (from.cpuLoad * 100).toString().replace(',', '.'),
- averageCpuLoad = metricService.getAverageCpuLoad(minusHours, from.date)?.times(100)?.toString()?.replace(',', '.') ?: "0",
+ averageCpuLoad = metricService.getAverageCpuLoad(minusHours, from.date)?.times(100)?.toString()
+ ?.replace(',', '.') ?: "0",
memoryUsage = (from.memoryUsage / 1024.0 / 1024.0).toString().replace(',', '.'),
- averageMemoryUsage = metricService.getAverageMemoryUsage(minusHours, from.date)?.div(1024)?.div(1024)?.toString()?.replace(',', '.') ?: "0",
+ averageMemoryUsage = metricService.getAverageMemoryUsage(minusHours, from.date)?.div(1024)?.div(1024)
+ ?.toString()?.replace(',', '.') ?: "0",
databaseSize = (from.databaseSize / 1024.0 / 1024.0).toDoublePoint(),
date = from.date.withZoneSameInstant(europeParisZone).format(dateFormatter)
)
diff --git a/src/main/kotlin/fr/shikkanime/converters/platform/PlatformDtoToPlatformConverter.kt b/src/main/kotlin/fr/shikkanime/converters/platform/PlatformDtoToPlatformConverter.kt
deleted file mode 100644
index 58125a09..00000000
--- a/src/main/kotlin/fr/shikkanime/converters/platform/PlatformDtoToPlatformConverter.kt
+++ /dev/null
@@ -1,28 +0,0 @@
-package fr.shikkanime.converters.platform
-
-import com.google.inject.Inject
-import fr.shikkanime.converters.AbstractConverter
-import fr.shikkanime.dtos.PlatformDto
-import fr.shikkanime.entities.Platform
-import fr.shikkanime.services.PlatformService
-
-class PlatformDtoToPlatformConverter : AbstractConverter() {
- @Inject
- private lateinit var platformService: PlatformService
-
- override fun convert(from: PlatformDto): Platform {
- if (from.uuid != null) {
- val find = platformService.find(from.uuid)
-
- if (find != null) {
- return find
- }
- }
-
- return Platform(
- name = from.name,
- url = from.url,
- image = from.image
- )
- }
-}
\ No newline at end of file
diff --git a/src/main/kotlin/fr/shikkanime/converters/platform/PlatformToPlatformDtoConverter.kt b/src/main/kotlin/fr/shikkanime/converters/platform/PlatformToPlatformDtoConverter.kt
deleted file mode 100644
index 8b962248..00000000
--- a/src/main/kotlin/fr/shikkanime/converters/platform/PlatformToPlatformDtoConverter.kt
+++ /dev/null
@@ -1,16 +0,0 @@
-package fr.shikkanime.converters.platform
-
-import fr.shikkanime.converters.AbstractConverter
-import fr.shikkanime.dtos.PlatformDto
-import fr.shikkanime.entities.Platform
-
-class PlatformToPlatformDtoConverter : AbstractConverter() {
- override fun convert(from: Platform): PlatformDto {
- return PlatformDto(
- uuid = from.uuid,
- name = from.name!!,
- url = from.url!!,
- image = from.image!!
- )
- }
-}
\ No newline at end of file
diff --git a/src/main/kotlin/fr/shikkanime/dtos/PlatformDto.kt b/src/main/kotlin/fr/shikkanime/dtos/PlatformDto.kt
deleted file mode 100644
index 1825410e..00000000
--- a/src/main/kotlin/fr/shikkanime/dtos/PlatformDto.kt
+++ /dev/null
@@ -1,11 +0,0 @@
-package fr.shikkanime.dtos
-
-import java.io.Serializable
-import java.util.*
-
-data class PlatformDto(
- val uuid: UUID?,
- val name: String,
- val url: String,
- val image: String
-) : Serializable
diff --git a/src/main/kotlin/fr/shikkanime/entities/Anime.kt b/src/main/kotlin/fr/shikkanime/entities/Anime.kt
index 6ee184ec..f83f50c0 100644
--- a/src/main/kotlin/fr/shikkanime/entities/Anime.kt
+++ b/src/main/kotlin/fr/shikkanime/entities/Anime.kt
@@ -1,5 +1,6 @@
package fr.shikkanime.entities
+import fr.shikkanime.entities.enums.CountryCode
import jakarta.persistence.*
import java.time.ZonedDateTime
import java.util.*
@@ -8,14 +9,15 @@ import java.util.*
@Table(name = "anime")
data class Anime(
override val uuid: UUID? = null,
- @ManyToOne(optional = false, fetch = FetchType.LAZY)
- val country: Country? = null,
+ @Column(nullable = false, name = "country_code")
+ @Enumerated(EnumType.STRING)
+ val countryCode: CountryCode? = null,
@Column(nullable = false)
val name: String? = null,
@Column(nullable = false, name = "release_date")
val releaseDate: ZonedDateTime = ZonedDateTime.now(),
- @Column(nullable = false)
+ @Column(nullable = false, columnDefinition = "VARCHAR(1000)")
var image: String? = null,
- @Column(nullable = true, columnDefinition = "TEXT")
+ @Column(nullable = true, columnDefinition = "VARCHAR(1000)")
var description: String? = null,
) : ShikkEntity(uuid)
diff --git a/src/main/kotlin/fr/shikkanime/entities/Country.kt b/src/main/kotlin/fr/shikkanime/entities/Country.kt
deleted file mode 100644
index b0264e56..00000000
--- a/src/main/kotlin/fr/shikkanime/entities/Country.kt
+++ /dev/null
@@ -1,16 +0,0 @@
-package fr.shikkanime.entities
-
-import jakarta.persistence.Column
-import jakarta.persistence.Entity
-import jakarta.persistence.Table
-import java.util.*
-
-@Entity
-@Table(name = "country")
-data class Country(
- override val uuid: UUID? = null,
- @Column(nullable = false, unique = true)
- val name: String? = null,
- @Column(nullable = false, unique = true, name = "country_code")
- val countryCode: String? = null,
-) : ShikkEntity(uuid)
\ No newline at end of file
diff --git a/src/main/kotlin/fr/shikkanime/entities/Episode.kt b/src/main/kotlin/fr/shikkanime/entities/Episode.kt
index 9058e71d..06181bf1 100644
--- a/src/main/kotlin/fr/shikkanime/entities/Episode.kt
+++ b/src/main/kotlin/fr/shikkanime/entities/Episode.kt
@@ -2,6 +2,7 @@ package fr.shikkanime.entities
import fr.shikkanime.entities.enums.EpisodeType
import fr.shikkanime.entities.enums.LangType
+import fr.shikkanime.entities.enums.Platform
import jakarta.persistence.*
import java.time.ZonedDateTime
import java.util.*
@@ -10,7 +11,8 @@ import java.util.*
@Table(name = "episode")
data class Episode(
override val uuid: UUID? = null,
- @ManyToOne(optional = false, fetch = FetchType.LAZY)
+ @Column(nullable = false)
+ @Enumerated(EnumType.STRING)
var platform: Platform? = null,
@ManyToOne(optional = false, fetch = FetchType.LAZY)
var anime: Anime? = null,
@@ -20,7 +22,7 @@ data class Episode(
@Column(nullable = false, name = "lang_type")
@Enumerated(EnumType.STRING)
val langType: LangType? = null,
- @Column(nullable = false)
+ @Column(nullable = false, unique = true)
val hash: String? = null,
@Column(nullable = false, name = "release_date")
val releaseDate: ZonedDateTime = ZonedDateTime.now(),
@@ -28,11 +30,11 @@ data class Episode(
var season: Int? = null,
@Column(nullable = false)
var number: Int? = null,
- @Column(nullable = true, columnDefinition = "TEXT")
+ @Column(nullable = true, columnDefinition = "VARCHAR(1000)")
var title: String? = null,
- @Column(nullable = false, columnDefinition = "TEXT")
+ @Column(nullable = false, columnDefinition = "VARCHAR(1000)")
var url: String? = null,
- @Column(nullable = false, columnDefinition = "TEXT")
+ @Column(nullable = false, columnDefinition = "VARCHAR(1000)")
var image: String? = null,
@Column(nullable = false)
var duration: Long = -1
diff --git a/src/main/kotlin/fr/shikkanime/entities/Platform.kt b/src/main/kotlin/fr/shikkanime/entities/Platform.kt
deleted file mode 100644
index 9146a6dd..00000000
--- a/src/main/kotlin/fr/shikkanime/entities/Platform.kt
+++ /dev/null
@@ -1,18 +0,0 @@
-package fr.shikkanime.entities
-
-import jakarta.persistence.Column
-import jakarta.persistence.Entity
-import jakarta.persistence.Table
-import java.util.*
-
-@Entity
-@Table(name = "platform")
-data class Platform(
- override val uuid: UUID? = null,
- @Column(nullable = false, unique = true)
- val name: String? = null,
- @Column(nullable = false)
- var url: String? = null,
- @Column(nullable = false)
- var image: String? = null
-) : ShikkEntity(uuid)
diff --git a/src/main/kotlin/fr/shikkanime/entities/enums/CountryCode.kt b/src/main/kotlin/fr/shikkanime/entities/enums/CountryCode.kt
new file mode 100644
index 00000000..7f9af5a0
--- /dev/null
+++ b/src/main/kotlin/fr/shikkanime/entities/enums/CountryCode.kt
@@ -0,0 +1,13 @@
+package fr.shikkanime.entities.enums
+
+
+enum class CountryCode(val locale: String? = null, val voice: String? = null) {
+ FR("fr-FR", "VF"),
+ ;
+
+ companion object {
+ fun from(collection: Collection): Set {
+ return collection.map { valueOf(it.uppercase()) }.toSet()
+ }
+ }
+}
\ No newline at end of file
diff --git a/src/main/kotlin/fr/shikkanime/entities/enums/Platform.kt b/src/main/kotlin/fr/shikkanime/entities/enums/Platform.kt
new file mode 100644
index 00000000..cf40b8d0
--- /dev/null
+++ b/src/main/kotlin/fr/shikkanime/entities/enums/Platform.kt
@@ -0,0 +1,12 @@
+package fr.shikkanime.entities.enums
+
+enum class Platform(
+ val platformName: String,
+ var url: String,
+ var image: String
+) {
+ ANIM("Animation Digital Network", "https://animationdigitalnetwork.fr/", "animation_digital_network.png"),
+ CRUN("Crunchyroll", "https://www.crunchyroll.com/", "crunchyroll.png"),
+ DISN("Disney+", "https://www.disneyplus.com/", "disneyplus.png"),
+ ;
+}
diff --git a/src/main/kotlin/fr/shikkanime/exceptions/AnimeException.kt b/src/main/kotlin/fr/shikkanime/exceptions/AnimeException.kt
new file mode 100644
index 00000000..5fcb94e0
--- /dev/null
+++ b/src/main/kotlin/fr/shikkanime/exceptions/AnimeException.kt
@@ -0,0 +1,3 @@
+package fr.shikkanime.exceptions
+
+open class AnimeException(override val message: String? = null) : Exception(message)
diff --git a/src/main/kotlin/fr/shikkanime/exceptions/AnimeNotSimulcastedException.kt b/src/main/kotlin/fr/shikkanime/exceptions/AnimeNotSimulcastedException.kt
new file mode 100644
index 00000000..27ad3a20
--- /dev/null
+++ b/src/main/kotlin/fr/shikkanime/exceptions/AnimeNotSimulcastedException.kt
@@ -0,0 +1,3 @@
+package fr.shikkanime.exceptions
+
+data class AnimeNotSimulcastedException(override val message: String? = null) : AnimeException(message)
diff --git a/src/main/kotlin/fr/shikkanime/exceptions/EpisodeAlreadyReleasedException.kt b/src/main/kotlin/fr/shikkanime/exceptions/EpisodeAlreadyReleasedException.kt
new file mode 100644
index 00000000..31eebf93
--- /dev/null
+++ b/src/main/kotlin/fr/shikkanime/exceptions/EpisodeAlreadyReleasedException.kt
@@ -0,0 +1,3 @@
+package fr.shikkanime.exceptions
+
+data class EpisodeAlreadyReleasedException(override val message: String? = null) : EpisodeException(message)
diff --git a/src/main/kotlin/fr/shikkanime/exceptions/EpisodeException.kt b/src/main/kotlin/fr/shikkanime/exceptions/EpisodeException.kt
new file mode 100644
index 00000000..d7c65193
--- /dev/null
+++ b/src/main/kotlin/fr/shikkanime/exceptions/EpisodeException.kt
@@ -0,0 +1,3 @@
+package fr.shikkanime.exceptions
+
+open class EpisodeException(override val message: String? = null) : Exception(message)
diff --git a/src/main/kotlin/fr/shikkanime/exceptions/EpisodeNoSubtitlesOrVoiceException.kt b/src/main/kotlin/fr/shikkanime/exceptions/EpisodeNoSubtitlesOrVoiceException.kt
new file mode 100644
index 00000000..4250e0b3
--- /dev/null
+++ b/src/main/kotlin/fr/shikkanime/exceptions/EpisodeNoSubtitlesOrVoiceException.kt
@@ -0,0 +1,3 @@
+package fr.shikkanime.exceptions
+
+data class EpisodeNoSubtitlesOrVoiceException(override val message: String? = null) : EpisodeException(message)
diff --git a/src/main/kotlin/fr/shikkanime/exceptions/EpisodeNotAvailableInCountryException.kt b/src/main/kotlin/fr/shikkanime/exceptions/EpisodeNotAvailableInCountryException.kt
new file mode 100644
index 00000000..a333e7d7
--- /dev/null
+++ b/src/main/kotlin/fr/shikkanime/exceptions/EpisodeNotAvailableInCountryException.kt
@@ -0,0 +1,3 @@
+package fr.shikkanime.exceptions
+
+data class EpisodeNotAvailableInCountryException(override val message: String? = null) : EpisodeException(message)
diff --git a/src/main/kotlin/fr/shikkanime/jobs/FetchEpisodesJob.kt b/src/main/kotlin/fr/shikkanime/jobs/FetchEpisodesJob.kt
index 13d8b8fe..dbb23beb 100644
--- a/src/main/kotlin/fr/shikkanime/jobs/FetchEpisodesJob.kt
+++ b/src/main/kotlin/fr/shikkanime/jobs/FetchEpisodesJob.kt
@@ -2,15 +2,35 @@ package fr.shikkanime.jobs
import fr.shikkanime.entities.Episode
import fr.shikkanime.services.EpisodeService
+import fr.shikkanime.services.ImageService
import fr.shikkanime.utils.Constant
import jakarta.inject.Inject
import java.time.ZonedDateTime
class FetchEpisodesJob : AbstractJob() {
+ private var isInitialized = false
+ private var isRunning = false
+ private val set = mutableSetOf()
+
@Inject
private lateinit var episodeService: EpisodeService
override fun run() {
+ if (isRunning) {
+ println("Job is already running")
+ return
+ }
+
+ isRunning = true
+
+ if (!isInitialized) {
+ val hashes = episodeService.findAllHashes()
+
+ set.addAll(hashes)
+ Constant.abstractPlatforms.forEach { it.hashCache.addAll(hashes) }
+ isInitialized = true
+ }
+
val zonedDateTime = ZonedDateTime.now().withNano(0)
val episodes = mutableListOf()
@@ -25,6 +45,18 @@ class FetchEpisodesJob : AbstractJob() {
}
}
- episodes.forEach { episodeService.saveOrUpdate(it) }
+ episodes
+ .filter {
+ (zonedDateTime.isEqual(it.releaseDate) || zonedDateTime.isAfter(it.releaseDate)) && !set.contains(
+ it.hash
+ )
+ }
+ .forEach {
+ val savedEpisode = episodeService.save(it)
+ savedEpisode.hash?.let { hash -> set.add(hash) }
+ ImageService.add(savedEpisode.uuid!!, savedEpisode.image!!)
+ }
+
+ isRunning = false
}
}
\ No newline at end of file
diff --git a/src/main/kotlin/fr/shikkanime/jobs/SavingImageCacheJob.kt b/src/main/kotlin/fr/shikkanime/jobs/SavingImageCacheJob.kt
new file mode 100644
index 00000000..1b9e7b14
--- /dev/null
+++ b/src/main/kotlin/fr/shikkanime/jobs/SavingImageCacheJob.kt
@@ -0,0 +1,10 @@
+package fr.shikkanime.jobs
+
+import fr.shikkanime.services.ImageService
+
+class SavingImageCacheJob : AbstractJob() {
+
+ override fun run() {
+ ImageService.saveCache()
+ }
+}
\ No newline at end of file
diff --git a/src/main/kotlin/fr/shikkanime/modules/DefaultModule.kt b/src/main/kotlin/fr/shikkanime/modules/DefaultModule.kt
index 7de3b090..7ee962cd 100644
--- a/src/main/kotlin/fr/shikkanime/modules/DefaultModule.kt
+++ b/src/main/kotlin/fr/shikkanime/modules/DefaultModule.kt
@@ -1,6 +1,7 @@
package fr.shikkanime.modules
import com.google.inject.AbstractModule
+import fr.shikkanime.jobs.AbstractJob
import fr.shikkanime.platforms.AbstractPlatform
import fr.shikkanime.repositories.AbstractRepository
import fr.shikkanime.services.AbstractService
@@ -24,6 +25,10 @@ class DefaultModule : AbstractModule() {
bind(it).asEagerSingleton()
}
+ Constant.reflections.getSubTypesOf(AbstractJob::class.java).forEach {
+ bind(it).asEagerSingleton()
+ }
+
Constant.reflections.getTypesAnnotatedWith(Controller::class.java).forEach {
bind(it).asEagerSingleton()
}
diff --git a/src/main/kotlin/fr/shikkanime/platforms/AbstractPlatform.kt b/src/main/kotlin/fr/shikkanime/platforms/AbstractPlatform.kt
index 4b4a5f90..f54da1d2 100644
--- a/src/main/kotlin/fr/shikkanime/platforms/AbstractPlatform.kt
+++ b/src/main/kotlin/fr/shikkanime/platforms/AbstractPlatform.kt
@@ -1,50 +1,37 @@
package fr.shikkanime.platforms
-import fr.shikkanime.entities.Country
import fr.shikkanime.entities.Episode
-import fr.shikkanime.entities.Platform
-import fr.shikkanime.services.CountryService
-import fr.shikkanime.utils.Constant
-import jakarta.inject.Inject
+import fr.shikkanime.entities.enums.Platform
+import fr.shikkanime.utils.ObjectParser
import kotlinx.coroutines.runBlocking
import java.io.File
import java.time.ZonedDateTime
-abstract class AbstractPlatform {
- data class Api(
- val lastCheck: ZonedDateTime,
- val content: Map = emptyMap(),
- )
-
- @Inject
- protected lateinit var countryService: CountryService
-
+abstract class AbstractPlatform {
+ val hashCache = mutableListOf()
var configuration: C? = null
- private var apiCache: Api? = null
+ private var apiCache = mutableMapOf, V>()
protected abstract fun getConfigurationClass(): Class
abstract fun getPlatform(): Platform
- abstract suspend fun fetchApiContent(zonedDateTime: ZonedDateTime): Api
- abstract fun fetchEpisodes(zonedDateTime: ZonedDateTime): List
+ abstract suspend fun fetchApiContent(key: K, zonedDateTime: ZonedDateTime): V
+ abstract fun fetchEpisodes(zonedDateTime: ZonedDateTime, bypassFileContent: File? = null): List
abstract fun reset()
- protected fun getCountries() = countryService.findAllByCode(configuration!!.availableCountries)
-
- fun getApiContent(country: Country, zonedDateTime: ZonedDateTime): String {
- var hasFetch = false
-
- if (apiCache == null) {
- apiCache = runBlocking { fetchApiContent(zonedDateTime) }
- hasFetch = true
+ fun getApiContent(key: K, zonedDateTime: ZonedDateTime): V {
+ if (apiCache.none { it.key.first == key }) {
+ apiCache[Pair(key, zonedDateTime)] = runBlocking { fetchApiContent(key, zonedDateTime) }
}
- val plusHours = apiCache!!.lastCheck.plusMinutes(configuration!!.apiCheckDelayInMinutes)
+ val entry = apiCache.entries.firstOrNull { it.key.first == key }!!
+ val plusMinutes = entry.key.second.plusMinutes(configuration!!.apiCheckDelayInMinutes)
- if (!hasFetch && zonedDateTime.isEqual(plusHours) || zonedDateTime.isAfter(plusHours)) {
- apiCache = runBlocking { fetchApiContent(zonedDateTime) }
+ if (zonedDateTime.isEqual(plusMinutes) || zonedDateTime.isAfter(plusMinutes)) {
+ apiCache.remove(entry.key)
+ apiCache[Pair(key, zonedDateTime)] = runBlocking { fetchApiContent(key, zonedDateTime) }
}
- return apiCache!!.content[country.countryCode]!!
+ return apiCache.entries.firstOrNull { it.key.first == key }!!.value
}
fun loadConfiguration(): C {
@@ -62,7 +49,7 @@ abstract class AbstractPlatform {
try {
println("Reading config file for ${getPlatform().name} platform")
- val fromJson = Constant.gson.fromJson(file.readText(), getConfigurationClass())
+ val fromJson = ObjectParser.fromJson(file.readText(), getConfigurationClass())
configuration = fromJson
return fromJson
} catch (e: Exception) {
@@ -71,7 +58,7 @@ abstract class AbstractPlatform {
}
}
- fun saveConfiguration() {
+ open fun saveConfiguration() {
val file = getConfigurationFile()
if (!file.exists()) {
@@ -79,11 +66,10 @@ abstract class AbstractPlatform {
file.createNewFile()
}
- file.writeText(Constant.gson.toJson(configuration))
- apiCache = null
+ file.writeText(ObjectParser.toJson(configuration))
}
private fun getConfigurationFile(): File {
- return File("config/${getPlatform().name!!.lowercase().replace(" ", "-")}.json")
+ return File("config/${getPlatform().platformName.lowercase().replace(" ", "-")}.json")
}
}
diff --git a/src/main/kotlin/fr/shikkanime/platforms/AnimationDigitalNetworkPlatform.kt b/src/main/kotlin/fr/shikkanime/platforms/AnimationDigitalNetworkPlatform.kt
index 8ae46c58..59804a8c 100644
--- a/src/main/kotlin/fr/shikkanime/platforms/AnimationDigitalNetworkPlatform.kt
+++ b/src/main/kotlin/fr/shikkanime/platforms/AnimationDigitalNetworkPlatform.kt
@@ -1,61 +1,55 @@
package fr.shikkanime.platforms
+import com.google.gson.JsonArray
import com.google.gson.JsonObject
import fr.shikkanime.entities.Anime
-import fr.shikkanime.entities.Country
import fr.shikkanime.entities.Episode
-import fr.shikkanime.entities.Platform
+import fr.shikkanime.entities.enums.CountryCode
import fr.shikkanime.entities.enums.EpisodeType
import fr.shikkanime.entities.enums.LangType
-import fr.shikkanime.utils.Constant
+import fr.shikkanime.entities.enums.Platform
+import fr.shikkanime.exceptions.AnimeException
+import fr.shikkanime.exceptions.AnimeNotSimulcastedException
import fr.shikkanime.utils.HttpRequest
+import fr.shikkanime.utils.ObjectParser
+import fr.shikkanime.utils.ObjectParser.getAsBoolean
+import fr.shikkanime.utils.ObjectParser.getAsInt
+import fr.shikkanime.utils.ObjectParser.getAsLong
+import fr.shikkanime.utils.ObjectParser.getAsString
import io.ktor.client.statement.*
import io.ktor.http.*
+import java.io.File
import java.time.ZonedDateTime
-class AnimationDigitalNetworkPlatform : AbstractPlatform() {
+class AnimationDigitalNetworkPlatform : AbstractPlatform() {
override fun getConfigurationClass(): Class = PlatformConfiguration::class.java
- override fun getPlatform(): Platform {
- return Platform(
- name = "Animation Digital Network",
- url = "https://animationdigitalnetwork.fr/",
- image = "animation_digital_network.png",
- )
- }
+ override fun getPlatform(): Platform = Platform.ANIM
- override suspend fun fetchApiContent(zonedDateTime: ZonedDateTime): Api {
- val map = mutableMapOf()
- val countries = getCountries()
+ override suspend fun fetchApiContent(key: CountryCode, zonedDateTime: ZonedDateTime): JsonArray {
val toDateString = zonedDateTime.toLocalDate().toString()
+ val url = "https://gw.api.animationdigitalnetwork.${key.name.lowercase()}/video/calendar?date=$toDateString"
+ val response = HttpRequest().get(url)
- countries.forEach { country ->
- val url = "https://gw.api.animationdigitalnetwork.${country.countryCode!!.lowercase()}/video/calendar?date=$toDateString"
- val response = HttpRequest().get(url)
-
- if (response.status != HttpStatusCode.OK) {
- return@forEach
- }
-
- map[country.countryCode] = response.bodyAsText()
+ if (response.status != HttpStatusCode.OK) {
+ return JsonArray()
}
- return Api(zonedDateTime, map)
+ return ObjectParser.fromJson(response.bodyAsText(), JsonObject::class.java).getAsJsonArray("videos")!!
}
- override fun fetchEpisodes(zonedDateTime: ZonedDateTime): List {
+ override fun fetchEpisodes(zonedDateTime: ZonedDateTime, bypassFileContent: File?): List {
val list = mutableListOf()
- val countries = getCountries()
- countries.forEach { country ->
- val api = getApiContent(country, zonedDateTime)
- val array = Constant.gson.fromJson(api, JsonObject::class.java).getAsJsonArray("videos")
+ configuration!!.availableCountries.forEach { countryCode ->
+ val api = getApiContent(countryCode, zonedDateTime)
- array.forEach {
+ api.forEach {
try {
- list.add(convertEpisode(country, it.asJsonObject, zonedDateTime))
- } catch (e: Exception) {
+ list.add(convertEpisode(countryCode, it.getAsJsonObject(), zonedDateTime))
+ } catch (_: AnimeException) {
// Ignore
+ } catch (e: Exception) {
e.printStackTrace()
}
}
@@ -68,22 +62,27 @@ class AnimationDigitalNetworkPlatform : AbstractPlatform(
TODO("Not yet implemented")
}
- private fun convertEpisode(country: Country, jsonObject: JsonObject, zonedDateTime: ZonedDateTime): Episode {
+ private fun convertEpisode(
+ countryCode: CountryCode,
+ jsonObject: JsonObject,
+ zonedDateTime: ZonedDateTime
+ ): Episode {
val show = jsonObject.getAsJsonObject("show") ?: throw Exception("Show is null")
- var animeName = show["shortTitle"]?.asString ?: show["title"]?.asString ?: throw Exception("Anime name is null")
+ var animeName =
+ show.getAsString("shortTitle") ?: show.getAsString("title") ?: throw Exception("Anime name is null")
animeName = animeName.replace(Regex("Saison \\d"), "").trim('-').trim()
- val animeImage = show["image2x"]?.asString ?: throw Exception("Anime image is null")
- val animeDescription = show["summary"]?.asString ?: ""
- val genres = show.getAsJsonArray("genres")?.toList() ?: emptyList()
+ val animeImage = show.getAsString("image2x") ?: throw Exception("Anime image is null")
+ val animeDescription = show.getAsString("summary")?.replace('\n', ' ') ?: ""
+ val genres = show.getAsJsonArray("genres") ?: JsonArray()
- if (genres.isEmpty() || !genres.any { it.asString.startsWith("Animation ", true) }) {
+ if (genres.isEmpty || !genres.any { it.asString.startsWith("Animation ", true) }) {
throw Exception("Anime is not an animation")
}
- var isSimulcasted = show["simulcast"]?.asBoolean == true ||
- show["firstReleaseYear"]?.asString == zonedDateTime.toLocalDate().year.toString() ||
+ var isSimulcasted = show.getAsBoolean("simulcast") == true ||
+ show.getAsString("firstReleaseYear") == zonedDateTime.toLocalDate().year.toString() ||
configuration?.simulcasts?.contains(animeName) == true
val descriptionLowercase = animeDescription.lowercase()
@@ -94,16 +93,15 @@ class AnimationDigitalNetworkPlatform : AbstractPlatform(
descriptionLowercase.startsWith("(diffusion de l'épisode 1 le")
if (!isSimulcasted) {
- throw Exception("Anime is not simulcasted")
+ throw AnimeNotSimulcastedException("Anime is not simulcasted")
}
- val releaseDateString = jsonObject["releaseDate"]?.asString ?: throw Exception("Release date is null")
- // Example : 2023-12-08T13:30:00Z
+ val releaseDateString = jsonObject.getAsString("releaseDate") ?: throw Exception("Release date is null")
val releaseDate = ZonedDateTime.parse(releaseDateString)
- val season = show["season"]?.asString?.toIntOrNull() ?: 1
+ val season = jsonObject.getAsString("season")?.toIntOrNull() ?: 1
- val numberAsString = jsonObject["shortNumber"]?.asString
+ val numberAsString = jsonObject.getAsString("shortNumber")
if (numberAsString?.startsWith("Bande-annonce") == true) {
throw Exception("Anime is a trailer")
@@ -117,26 +115,26 @@ class AnimationDigitalNetworkPlatform : AbstractPlatform(
else -> EpisodeType.EPISODE
}
- val langType = when (jsonObject.getAsJsonArray("languages").lastOrNull()?.asString) {
+ val langType = when (jsonObject.getAsJsonArray("languages")?.lastOrNull()?.asString) {
"vostf" -> LangType.SUBTITLES
"vf" -> LangType.VOICE
else -> throw Exception("Language is null")
}
- val id = jsonObject["id"]?.asLong ?: throw Exception("Id is null")
+ val id = jsonObject.getAsInt("id")
- val title = jsonObject["name"]?.asString?.ifBlank { null }
+ val title = jsonObject.getAsString("name")?.ifBlank { null }
- val url = jsonObject["url"]?.asString ?: throw Exception("Url is null")
+ val url = jsonObject.getAsString("url") ?: throw Exception("Url is null")
- val image = jsonObject["image2x"]?.asString ?: throw Exception("Image is null")
+ val image = jsonObject.getAsString("image2x") ?: throw Exception("Image is null")
- val duration = jsonObject["duration"]?.asLong ?: -1
+ val duration = jsonObject.getAsLong("duration", -1)
return Episode(
platform = getPlatform(),
anime = Anime(
- country = country,
+ countryCode = countryCode,
name = animeName,
releaseDate = releaseDate,
image = animeImage,
@@ -144,7 +142,7 @@ class AnimationDigitalNetworkPlatform : AbstractPlatform(
),
episodeType = episodeType,
langType = langType,
- hash = id.toString(),
+ hash = "${countryCode}-${getPlatform()}-$id-$langType",
releaseDate = releaseDate,
season = season,
number = number,
diff --git a/src/main/kotlin/fr/shikkanime/platforms/CrunchyrollPlatform.kt b/src/main/kotlin/fr/shikkanime/platforms/CrunchyrollPlatform.kt
index 874c64b1..1794e035 100644
--- a/src/main/kotlin/fr/shikkanime/platforms/CrunchyrollPlatform.kt
+++ b/src/main/kotlin/fr/shikkanime/platforms/CrunchyrollPlatform.kt
@@ -1,11 +1,30 @@
package fr.shikkanime.platforms
+import com.google.gson.JsonObject
+import fr.shikkanime.caches.CountryCodeAnimeIdKeyCache
+import fr.shikkanime.entities.Anime
import fr.shikkanime.entities.Episode
-import fr.shikkanime.entities.Platform
+import fr.shikkanime.entities.enums.CountryCode
+import fr.shikkanime.entities.enums.EpisodeType
+import fr.shikkanime.entities.enums.LangType
+import fr.shikkanime.entities.enums.Platform
+import fr.shikkanime.exceptions.*
+import fr.shikkanime.utils.HttpRequest
+import fr.shikkanime.utils.MapCache
+import fr.shikkanime.utils.ObjectParser
+import fr.shikkanime.utils.ObjectParser.getAsInt
+import fr.shikkanime.utils.ObjectParser.getAsLong
+import fr.shikkanime.utils.ObjectParser.getAsString
+import io.ktor.client.statement.*
import io.ktor.http.*
+import java.io.File
+import java.time.Duration
import java.time.ZonedDateTime
+import java.time.format.DateTimeFormatter
-class CrunchyrollPlatform : AbstractPlatform() {
+private const val IMAGE_NULL_ERROR = "Image is null"
+
+class CrunchyrollPlatform : AbstractPlatform() {
data class CrunchyrollConfiguration(
var simulcastCheckDelayInMinutes: Long = 60,
) : PlatformConfiguration() {
@@ -28,25 +47,248 @@ class CrunchyrollPlatform : AbstractPlatform>(Duration.ofHours(1)) {
+ fun getSimulcastCode(name: String): String {
+ val simulcastCodeTmp = name.lowercase().replace(" ", "-")
+ val simulcastYear = simulcastCodeTmp.split("-").last()
- override fun getPlatform(): Platform {
- return Platform(
- name = "Crunchyroll",
- url = "https://www.crunchyroll.com/",
- image = "crunchyroll.png",
- )
+ val simulcastSeasonCode = when (simulcastCodeTmp.split("-").first()) {
+ "printemps" -> "spring"
+ "été" -> "summer"
+ "automne" -> "fall"
+ "hiver" -> "winter"
+ else -> throw Exception("Simulcast season not found")
+ }
+
+ return "$simulcastSeasonCode-$simulcastYear"
+ }
+
+ fun getPreviousSimulcastCode(currentSimulcastCode: String): String {
+ return when (currentSimulcastCode.split("-").first()) {
+ "spring" -> "winter-${currentSimulcastCode.split("-").last()}"
+ "winter" -> "fall-${currentSimulcastCode.split("-").last().toInt() - 1}"
+ "fall" -> "summer-${currentSimulcastCode.split("-").last().toInt()}"
+ "summer" -> "spring-${currentSimulcastCode.split("-").last().toInt()}"
+ else -> throw Exception("Simulcast season not found")
+ }
+ }
+
+ val httpRequest = HttpRequest()
+ val simulcasts = mutableSetOf()
+
+ val simulcastSelector =
+ "#content > div > div.app-body-wrapper > div > div > div.erc-browse-collection > div > div:nth-child(1) > div > div > h4 > a"
+ val simulcastAnimesSelector = ".erc-browse-cards-collection > .browse-card > div > div > h4 > a"
+
+ try {
+ val currentSimulcastContent = httpRequest.getBrowser(
+ "https://www.crunchyroll.com/${it.name.lowercase()}/simulcasts",
+ simulcastSelector
+ )
+ val currentSimulcast =
+ currentSimulcastContent.select("#content > div > div.app-body-wrapper > div > div > div.header > div > div > span.call-to-action--PEidl.call-to-action--is-m--RVdkI.select-trigger__title-cta--C5-uH.select-trigger__title-cta--is-displayed-on-mobile--6oNk1")
+ .text() ?: return@MapCache simulcasts
+ val currentSimulcastCode = getSimulcastCode(currentSimulcast)
+ println("Current simulcast code for $it: $currentSimulcast > $currentSimulcastCode")
+ val currentSimulcastAnimes =
+ currentSimulcastContent.select(simulcastAnimesSelector).map { a -> a.text().lowercase() }.toSet()
+ println("Found ${currentSimulcastAnimes.size} animes for the current simulcast")
+
+ val previousSimulcastCode = getPreviousSimulcastCode(currentSimulcastCode)
+ println("Previous simulcast code for $it: $previousSimulcastCode")
+
+ val previousSimulcastContent = httpRequest.getBrowser(
+ "https://www.crunchyroll.com/${it.name.lowercase()}/simulcasts/seasons/$previousSimulcastCode",
+ simulcastSelector
+ )
+ val previousSimulcastAnimes =
+ previousSimulcastContent.select(simulcastAnimesSelector).map { a -> a.text().lowercase() }.toSet()
+ println("Found ${previousSimulcastAnimes.size} animes for the previous simulcast")
+
+ val combinedSimulcasts = (currentSimulcastAnimes + previousSimulcastAnimes).toSet()
+ simulcasts.addAll(combinedSimulcasts)
+ } catch (e: Exception) {
+ e.printStackTrace()
+ } finally {
+ httpRequest.closeBrowser()
+ }
+
+ println(simulcasts)
+ return@MapCache simulcasts
}
- override suspend fun fetchApiContent(zonedDateTime: ZonedDateTime): Api {
- TODO("Not yet implemented")
+ val animeInfoCache = MapCache>(Duration.ofDays(1)) {
+ val httpRequest = HttpRequest()
+ var image: String? = null
+ var description: String? = null
+
+ try {
+ val content = httpRequest.getBrowser(
+ "https://www.crunchyroll.com/${it.countryCode.name.lowercase()}/${it.animeId}",
+ "div.undefined:nth-child(1) > figure:nth-child(1) > picture:nth-child(1) > img:nth-child(2)"
+ )
+ image =
+ content.selectXpath("//*[@id=\"content\"]/div/div[2]/div/div[1]/div[2]/div/div/div[2]/div[2]/figure/picture/img")
+ .attr("src")
+ description =
+ content.selectXpath("//*[@id=\"content\"]/div/div[2]/div/div[2]/div[1]/div[1]/div[5]/div/div/div/p")
+ .text()
+ } catch (e: Exception) {
+ e.printStackTrace()
+ } finally {
+ httpRequest.closeBrowser()
+ }
+
+ if (image.isNullOrEmpty()) {
+ throw Exception("Image is null or empty")
+ }
+
+ return@MapCache image to description
}
- override fun fetchEpisodes(zonedDateTime: ZonedDateTime): List {
- return emptyList()
+ override fun getConfigurationClass() = CrunchyrollConfiguration::class.java
+
+ override fun getPlatform(): Platform = Platform.CRUN
+
+ override suspend fun fetchApiContent(key: CountryCode, zonedDateTime: ZonedDateTime): String {
+ val url = "https://www.crunchyroll.com/rss/anime?lang=${key.locale?.replace("-", "")}"
+ val response = HttpRequest().get(url)
+
+ if (response.status != HttpStatusCode.OK) {
+ return ""
+ }
+
+ return response.bodyAsText()
+ }
+
+ override fun fetchEpisodes(zonedDateTime: ZonedDateTime, bypassFileContent: File?): List {
+ val list = mutableListOf()
+
+ configuration!!.availableCountries.forEach { countryCode ->
+ var api =
+ if (bypassFileContent != null && bypassFileContent.exists()) bypassFileContent.readText() else getApiContent(
+ countryCode,
+ zonedDateTime
+ )
+ api = api.replace(System.lineSeparator(), "").replace("\n", "")
+ val array = "- (.*?)
".toRegex().findAll(api).map { it.value }
+
+ array.forEach {
+ try {
+ val xml = ObjectParser.fromXml(it, JsonObject::class.java)
+ list.add(convertEpisode(countryCode, xml))
+ } catch (_: EpisodeException) {
+ // Ignore
+ } catch (_: AnimeException) {
+ // Ignore
+ } catch (e: Exception) {
+ e.printStackTrace()
+ }
+ }
+ }
+
+ return list
}
override fun reset() {
TODO("Not yet implemented")
}
+
+ override fun saveConfiguration() {
+ super.saveConfiguration()
+ simulcasts.resetWithNewDuration(Duration.ofMinutes(configuration!!.simulcastCheckDelayInMinutes))
+ }
+
+ private fun convertEpisode(countryCode: CountryCode, jsonObject: JsonObject): Episode {
+ val animeName = jsonObject.getAsString("crunchyroll:seriesTitle") ?: throw Exception("Anime name is null")
+ val restrictions = jsonObject.getAsJsonObject("media:restriction")?.getAsString("")?.split(" ") ?: emptyList()
+
+ if (restrictions.isEmpty() || !restrictions.contains(countryCode.name.lowercase())) {
+ throw EpisodeNotAvailableInCountryException("Episode of $animeName is not available in ${countryCode.name}")
+ }
+
+ val fullName = jsonObject.getAsString("title") ?: throw Exception("Episode title is null")
+ val isDubbed = fullName.contains("(${countryCode.voice})", true)
+ val isMovie = fullName.contains("movie", true) || fullName.contains("film", true)
+
+ val subtitles = jsonObject.getAsString("crunchyroll:subtitleLanguages")?.split(",") ?: emptyList()
+
+ if (!isDubbed && (subtitles.isEmpty() || !subtitles.contains(
+ countryCode.locale?.replace("-", " - ")?.lowercase()
+ ))
+ ) {
+ throw EpisodeNoSubtitlesOrVoiceException("Episode is not available in ${countryCode.name} with subtitles or voice")
+ }
+
+ val langType = if (isDubbed) LangType.VOICE else LangType.SUBTITLES
+
+ val id = jsonObject.getAsString("crunchyroll:mediaId") ?: throw Exception("Id is null")
+ val hash = "${countryCode}-${getPlatform()}-$id-$langType"
+
+ if (hashCache.contains(hash)) {
+ throw EpisodeAlreadyReleasedException()
+ }
+
+ val releaseDate =
+ jsonObject.getAsString("pubDate")?.let { ZonedDateTime.parse(it, DateTimeFormatter.RFC_1123_DATE_TIME) }
+ ?: throw Exception("Release date is null")
+
+ val season = jsonObject.getAsString("crunchyroll:season")?.toIntOrNull() ?: 1
+
+ val number = jsonObject.getAsString("crunchyroll:episodeNumber")?.toIntOrNull() ?: -1
+
+ val episodeType =
+ if (isMovie) EpisodeType.FILM else if (number == -1) EpisodeType.SPECIAL else EpisodeType.EPISODE
+
+ val title = jsonObject.getAsString("crunchyroll:episodeTitle")?.ifBlank { null }
+
+ val url = jsonObject.getAsString("link")?.ifBlank { null } ?: throw Exception("Url is null")
+
+ val images = jsonObject.getAsJsonArray("media:thumbnail") ?: throw Exception(IMAGE_NULL_ERROR)
+ val biggestImage =
+ images.maxByOrNull { it.asJsonObject.getAsInt("width")!! } ?: throw Exception(IMAGE_NULL_ERROR)
+ val image = biggestImage.asJsonObject.getAsString("url")?.ifBlank { null } ?: throw Exception(IMAGE_NULL_ERROR)
+
+ val duration = jsonObject.getAsLong("crunchyroll:duration", -1)
+
+ var isSimulcasted =
+ simulcasts[countryCode].contains(animeName.lowercase()) || configuration!!.simulcasts.contains(animeName)
+ isSimulcasted = isSimulcasted || isMovie
+
+ if (!isSimulcasted) {
+ throw AnimeNotSimulcastedException("\"$animeName\" is not simulcasted")
+ }
+
+ val splitted = url.split("/")
+
+ if (splitted.size < 2) {
+ throw Exception("Anime id is null")
+ }
+
+ val animeId = splitted[splitted.size - 2]
+ val (animeImage, animeDescription) = animeInfoCache[CountryCodeAnimeIdKeyCache(countryCode, animeId)]
+
+ hashCache.add(hash)
+
+ return Episode(
+ platform = getPlatform(),
+ anime = Anime(
+ countryCode = countryCode,
+ name = animeName,
+ releaseDate = releaseDate,
+ image = animeImage,
+ description = animeDescription
+ ),
+ episodeType = episodeType,
+ langType = langType,
+ hash = hash,
+ releaseDate = releaseDate,
+ season = season,
+ number = number,
+ title = title,
+ url = url,
+ image = image,
+ duration = duration
+ )
+ }
}
\ No newline at end of file
diff --git a/src/main/kotlin/fr/shikkanime/platforms/DisneyPlusPlatform.kt b/src/main/kotlin/fr/shikkanime/platforms/DisneyPlusPlatform.kt
new file mode 100644
index 00000000..6041b910
--- /dev/null
+++ b/src/main/kotlin/fr/shikkanime/platforms/DisneyPlusPlatform.kt
@@ -0,0 +1,207 @@
+package fr.shikkanime.platforms
+
+import com.google.gson.JsonArray
+import com.google.gson.JsonObject
+import fr.shikkanime.caches.CountryCodeAnimeIdKeyCache
+import fr.shikkanime.entities.Anime
+import fr.shikkanime.entities.Episode
+import fr.shikkanime.entities.enums.CountryCode
+import fr.shikkanime.entities.enums.EpisodeType
+import fr.shikkanime.entities.enums.LangType
+import fr.shikkanime.entities.enums.Platform
+import fr.shikkanime.exceptions.AnimeException
+import fr.shikkanime.utils.HttpRequest
+import fr.shikkanime.utils.ObjectParser
+import fr.shikkanime.utils.ObjectParser.getAsBoolean
+import fr.shikkanime.utils.ObjectParser.getAsInt
+import fr.shikkanime.utils.ObjectParser.getAsLong
+import fr.shikkanime.utils.ObjectParser.getAsString
+import io.ktor.client.statement.*
+import io.ktor.http.*
+import java.io.File
+import java.time.ZonedDateTime
+
+class DisneyPlusPlatform :
+ AbstractPlatform() {
+ data class DisneyPlusConfiguration(
+ var authorization: String = "",
+ var refreshToken: String = "",
+ ) : PlatformConfiguration() {
+ override fun of(parameters: Parameters) {
+ super.of(parameters)
+ parameters["authorization"]?.let { authorization = it }
+ parameters["refreshToken"]?.let { refreshToken = it }
+ }
+
+ override fun toConfigurationFields(): MutableSet {
+ return super.toConfigurationFields().apply {
+ add(
+ ConfigurationField(
+ label = "Authorization",
+ name = "authorization",
+ type = "text",
+ value = authorization
+ )
+ )
+ add(
+ ConfigurationField(
+ label = "Refresh token",
+ name = "refreshToken",
+ type = "text",
+ value = refreshToken
+ )
+ )
+ }
+ }
+ }
+
+ override fun getConfigurationClass() = DisneyPlusConfiguration::class.java
+
+ override fun getPlatform(): Platform = Platform.DISN
+
+ override suspend fun fetchApiContent(key: CountryCodeAnimeIdKeyCache, zonedDateTime: ZonedDateTime): JsonArray {
+ check(configuration!!.authorization.isNotBlank()) { "Authorization is null" }
+ check(configuration!!.refreshToken.isNotBlank()) { "Refresh token is null" }
+ val httpRequest = HttpRequest()
+
+ val loginDevice = httpRequest.post(
+ "https://disney.api.edge.bamgrid.com/graph/v1/device/graphql",
+ mapOf(
+ "Authorization" to configuration!!.authorization,
+ ),
+ ObjectParser.toJson(
+ mapOf(
+ "operationName" to "refreshToken",
+ "query" to "mutation refreshToken(\$input:RefreshTokenInput!){refreshToken(refreshToken:\$input){activeSession{sessionId}}}",
+ "variables" to mapOf(
+ "input" to mapOf(
+ "refreshToken" to configuration!!.refreshToken
+ )
+ ),
+ )
+ )
+ )
+
+ check(loginDevice.status.value == 200) { "Failed to login to Disney+" }
+ val loginDeviceJson = ObjectParser.fromJson(loginDevice.bodyAsText(), JsonObject::class.java)
+ val accessToken = loginDeviceJson.getAsJsonObject("extensions").getAsJsonObject("sdk").getAsJsonObject("token")
+ .getAsString("accessToken")
+
+ val seasonsResponse = httpRequest.get(
+ "https://disney.content.edge.bamgrid.com/svc/content/DmcSeriesBundle/version/5.1/region/${key.countryCode.name}/audience/k-false,l-true/maturity/1850/language/${key.countryCode.locale}/encodedSeriesId/${key.animeId}",
+ mapOf("Authorization" to "Bearer $accessToken")
+ )
+ check(seasonsResponse.status.value == 200) { "Failed to fetch Disney+ content" }
+ val seasonsJson = ObjectParser.fromJson(seasonsResponse.bodyAsText(), JsonObject::class.java)
+ val seasons = seasonsJson.getAsJsonObject("data").getAsJsonObject("DmcSeriesBundle").getAsJsonObject("seasons")
+ .getAsJsonArray("seasons").mapNotNull { it.asJsonObject.getAsString("seasonId") }
+ val episodes = JsonArray()
+
+ seasons.forEach { season ->
+ var page = 1
+ var hasMore: Boolean
+
+ do {
+ val url =
+ "https://disney.content.edge.bamgrid.com/svc/content/DmcEpisodes/version/5.1/region/${key.countryCode.name}/audience/k-false,l-true/maturity/1850/language/${key.countryCode.locale}/seasonId/${season}/pageSize/15/page/${page++}"
+ val response = httpRequest.get(url, mapOf("Authorization" to "Bearer $accessToken"))
+ check(response.status.value == 200) { "Failed to fetch Disney+ content" }
+ val json = ObjectParser.fromJson(response.bodyAsText(), JsonObject::class.java)
+
+ val dmcEpisodesMeta = json.getAsJsonObject("data").getAsJsonObject("DmcEpisodes")
+ hasMore = dmcEpisodesMeta.getAsJsonObject("meta").getAsBoolean("hasMore") ?: false
+ dmcEpisodesMeta.getAsJsonArray("videos").forEach { episodes.add(it) }
+ } while (hasMore)
+ }
+
+ return episodes
+ }
+
+ override fun fetchEpisodes(zonedDateTime: ZonedDateTime, bypassFileContent: File?): List {
+ val list = mutableListOf()
+
+ configuration!!.availableCountries.forEach { countryCode ->
+ configuration!!.simulcasts.forEach { simulcast ->
+ val api = getApiContent(CountryCodeAnimeIdKeyCache(countryCode, simulcast), zonedDateTime)
+
+ api.forEach {
+ try {
+ list.add(convertEpisode(countryCode, it.getAsJsonObject(), zonedDateTime))
+ } catch (_: AnimeException) {
+ // Ignore
+ } catch (e: Exception) {
+ e.printStackTrace()
+ }
+ }
+ }
+ }
+
+ return list
+ }
+
+ override fun reset() {
+ TODO("Not yet implemented")
+ }
+
+ private fun convertEpisode(
+ countryCode: CountryCode,
+ jsonObject: JsonObject,
+ zonedDateTime: ZonedDateTime
+ ): Episode {
+ val texts = jsonObject.getAsJsonObject("text")
+ val titles = texts?.getAsJsonObject("title")?.getAsJsonObject("full")
+ val animeName = titles?.getAsJsonObject("series")?.getAsJsonObject("default")?.getAsString("content")
+ ?: throw Exception("Anime name is null")
+
+ val animeImage = jsonObject.getAsJsonObject("image")?.getAsJsonObject("tile")?.getAsJsonObject("0.71")
+ ?.getAsJsonObject("series")?.getAsJsonObject("default")?.getAsString("url")
+ ?: throw Exception("Anime image is null")
+ val animeDescription =
+ texts.getAsJsonObject("description")?.getAsJsonObject("medium")?.getAsJsonObject("series")
+ ?.getAsJsonObject("default")?.getAsString("content")?.replace('\n', ' ') ?: ""
+
+ val season = jsonObject.getAsInt("seasonSequenceNumber")
+
+ val number = jsonObject.getAsInt("episodeSequenceNumber")
+
+ val id = jsonObject.getAsString("contentId")
+
+ val title =
+ titles.getAsJsonObject("program")?.getAsJsonObject("default")?.getAsString("content")?.ifBlank { null }
+
+ val url = "https://www.disneyplus.com/${countryCode.locale?.lowercase()}/video/$id"
+
+ val image = jsonObject.getAsJsonObject("image")?.getAsJsonObject("thumbnail")?.getAsJsonObject("1.78")
+ ?.getAsJsonObject("program")?.getAsJsonObject("default")?.getAsString("url")
+ ?: throw Exception("Image is null")
+
+ var duration = jsonObject.getAsJsonObject("mediaMetadata")?.getAsLong("runtimeMillis", -1) ?: -1
+
+ if (duration != -1L) {
+ duration /= 1000
+ }
+
+ val langType = LangType.SUBTITLES
+
+ return Episode(
+ platform = getPlatform(),
+ anime = Anime(
+ countryCode = countryCode,
+ name = animeName,
+ releaseDate = zonedDateTime,
+ image = animeImage,
+ description = animeDescription,
+ ),
+ episodeType = EpisodeType.EPISODE,
+ langType = langType,
+ hash = "${countryCode}-${getPlatform()}-$id-$langType",
+ releaseDate = zonedDateTime,
+ season = season,
+ number = number,
+ title = title,
+ url = url,
+ image = image,
+ duration = duration,
+ )
+ }
+}
\ No newline at end of file
diff --git a/src/main/kotlin/fr/shikkanime/platforms/PlatformConfiguration.kt b/src/main/kotlin/fr/shikkanime/platforms/PlatformConfiguration.kt
index 3d32d278..66b9bae1 100644
--- a/src/main/kotlin/fr/shikkanime/platforms/PlatformConfiguration.kt
+++ b/src/main/kotlin/fr/shikkanime/platforms/PlatformConfiguration.kt
@@ -1,5 +1,6 @@
package fr.shikkanime.platforms
+import fr.shikkanime.entities.enums.CountryCode
import io.ktor.http.*
data class ConfigurationField(
@@ -10,7 +11,7 @@ data class ConfigurationField(
)
open class PlatformConfiguration(
- var availableCountries: MutableSet = mutableSetOf(),
+ var availableCountries: MutableSet = mutableSetOf(),
var apiCheckDelayInMinutes: Long = 0,
var simulcasts: MutableSet = mutableSetOf(),
) {
@@ -21,10 +22,12 @@ open class PlatformConfiguration(
return@let
}
- availableCountries = it.split(",").toMutableSet()
+ availableCountries = CountryCode.from(it.split(",")) as MutableSet
}
- parameters["apiCheckDelayInMinutes"]?.let { apiCheckDelayInMinutes = it.toLong() }
+ parameters["apiCheckDelayInMinutes"]?.let {
+ apiCheckDelayInMinutes = it.toLong()
+ }
(parameters.getAll("simulcasts") ?: emptyList()).let {
val toMutableSet = it.toMutableSet()
diff --git a/src/main/kotlin/fr/shikkanime/plugins/HTTP.kt b/src/main/kotlin/fr/shikkanime/plugins/HTTP.kt
index 480cb7f5..a8a3baeb 100644
--- a/src/main/kotlin/fr/shikkanime/plugins/HTTP.kt
+++ b/src/main/kotlin/fr/shikkanime/plugins/HTTP.kt
@@ -2,7 +2,7 @@ package fr.shikkanime.plugins
import freemarker.cache.ClassTemplateLoader
import io.ktor.http.*
-import io.ktor.serialization.gson.*
+import io.ktor.serialization.jackson.*
import io.ktor.server.application.*
import io.ktor.server.freemarker.*
import io.ktor.server.plugins.compression.*
@@ -36,7 +36,7 @@ fun Application.configureHTTP() {
}
}
install(ContentNegotiation) {
- gson {
+ jackson {
}
}
install(FreeMarker) {
diff --git a/src/main/kotlin/fr/shikkanime/plugins/Routing.kt b/src/main/kotlin/fr/shikkanime/plugins/Routing.kt
index 0d1b880c..75a5d97f 100644
--- a/src/main/kotlin/fr/shikkanime/plugins/Routing.kt
+++ b/src/main/kotlin/fr/shikkanime/plugins/Routing.kt
@@ -1,6 +1,5 @@
package fr.shikkanime.plugins
-import fr.shikkanime.dtos.PlatformDto
import fr.shikkanime.entities.Link
import fr.shikkanime.utils.Constant
import fr.shikkanime.utils.routes.*
@@ -42,7 +41,8 @@ private fun Routing.createRoutes() {
}
fun Routing.createControllerRoutes(controller: Any) {
- val prefix = if (controller::class.hasAnnotation()) controller::class.findAnnotation()!!.value else ""
+ val prefix =
+ if (controller::class.hasAnnotation()) controller::class.findAnnotation()!!.value else ""
val kMethods = controller::class.declaredFunctions.filter { it.hasAnnotation() }.toMutableSet()
route(prefix) {
@@ -170,7 +170,6 @@ private suspend fun callMethodWithParameters(
kParameter.hasAnnotation() -> {
when (kParameter.type.javaType) {
Array::class.java -> call.receive>()
- PlatformDto::class.java -> call.receive()
Parameters::class.java -> call.receiveParameters()
else -> call.receive()
}
diff --git a/src/main/kotlin/fr/shikkanime/repositories/AnimeRepository.kt b/src/main/kotlin/fr/shikkanime/repositories/AnimeRepository.kt
index e63e84df..26e8415c 100644
--- a/src/main/kotlin/fr/shikkanime/repositories/AnimeRepository.kt
+++ b/src/main/kotlin/fr/shikkanime/repositories/AnimeRepository.kt
@@ -1,11 +1,13 @@
package fr.shikkanime.repositories
import fr.shikkanime.entities.Anime
+import fr.shikkanime.entities.enums.CountryCode
class AnimeRepository : AbstractRepository() {
- fun findByName(name: String?): Anime? {
+ fun findByName(countryCode: CountryCode, name: String?): Anime? {
return inTransaction {
- it.createQuery("FROM Anime WHERE LOWER(name) = :name", getEntityClass())
+ it.createQuery("FROM Anime WHERE countryCode = :countryCode AND LOWER(name) = :name", getEntityClass())
+ .setParameter("countryCode", countryCode)
.setParameter("name", name?.lowercase())
.resultList
.firstOrNull()
diff --git a/src/main/kotlin/fr/shikkanime/repositories/CountryRepository.kt b/src/main/kotlin/fr/shikkanime/repositories/CountryRepository.kt
deleted file mode 100644
index 4767f00a..00000000
--- a/src/main/kotlin/fr/shikkanime/repositories/CountryRepository.kt
+++ /dev/null
@@ -1,31 +0,0 @@
-package fr.shikkanime.repositories
-
-import fr.shikkanime.entities.Country
-
-class CountryRepository : AbstractRepository() {
- fun findByName(name: String): Country? {
- return inTransaction {
- it.createQuery("FROM Country WHERE name = :name", getEntityClass())
- .setParameter("name", name)
- .resultList
- .firstOrNull()
- }
- }
-
- fun findByCode(code: String): Country? {
- return inTransaction {
- it.createQuery("FROM Country WHERE countryCode = :code", getEntityClass())
- .setParameter("code", code)
- .resultList
- .firstOrNull()
- }
- }
-
- fun findAllByCode(codes: List): List {
- return inTransaction {
- it.createQuery("FROM Country WHERE countryCode IN :codes", getEntityClass())
- .setParameter("codes", codes)
- .resultList
- }
- }
-}
\ No newline at end of file
diff --git a/src/main/kotlin/fr/shikkanime/repositories/EpisodeRepository.kt b/src/main/kotlin/fr/shikkanime/repositories/EpisodeRepository.kt
index ff5ea3dc..a735a382 100644
--- a/src/main/kotlin/fr/shikkanime/repositories/EpisodeRepository.kt
+++ b/src/main/kotlin/fr/shikkanime/repositories/EpisodeRepository.kt
@@ -3,6 +3,13 @@ package fr.shikkanime.repositories
import fr.shikkanime.entities.Episode
class EpisodeRepository : AbstractRepository() {
+ fun findAllHashes(): List {
+ return inTransaction {
+ it.createQuery("SELECT hash FROM Episode", String::class.java)
+ .resultList
+ }
+ }
+
fun findByHash(hash: String?): Episode? {
return inTransaction {
it.createQuery("FROM Episode WHERE LOWER(hash) = :hash", getEntityClass())
diff --git a/src/main/kotlin/fr/shikkanime/repositories/PlatformRepository.kt b/src/main/kotlin/fr/shikkanime/repositories/PlatformRepository.kt
deleted file mode 100644
index 0732621b..00000000
--- a/src/main/kotlin/fr/shikkanime/repositories/PlatformRepository.kt
+++ /dev/null
@@ -1,14 +0,0 @@
-package fr.shikkanime.repositories
-
-import fr.shikkanime.entities.Platform
-
-class PlatformRepository : AbstractRepository() {
- fun findByName(name: String): Platform? {
- return inTransaction {
- it.createQuery("FROM Platform WHERE name = :name", getEntityClass())
- .setParameter("name", name)
- .resultList
- .firstOrNull()
- }
- }
-}
\ No newline at end of file
diff --git a/src/main/kotlin/fr/shikkanime/services/AbstractService.kt b/src/main/kotlin/fr/shikkanime/services/AbstractService.kt
index 9b600c01..f0501cec 100644
--- a/src/main/kotlin/fr/shikkanime/services/AbstractService.kt
+++ b/src/main/kotlin/fr/shikkanime/services/AbstractService.kt
@@ -15,7 +15,5 @@ abstract class AbstractService> {
protected fun update(entity: E) = getRepository().update(entity)
- open fun saveOrUpdate(entity: E) = entity
-
fun delete(entity: E) = getRepository().delete(entity)
}
\ No newline at end of file
diff --git a/src/main/kotlin/fr/shikkanime/services/AnimeService.kt b/src/main/kotlin/fr/shikkanime/services/AnimeService.kt
index c6e28c03..60790f47 100644
--- a/src/main/kotlin/fr/shikkanime/services/AnimeService.kt
+++ b/src/main/kotlin/fr/shikkanime/services/AnimeService.kt
@@ -2,6 +2,7 @@ package fr.shikkanime.services
import com.google.inject.Inject
import fr.shikkanime.entities.Anime
+import fr.shikkanime.entities.enums.CountryCode
import fr.shikkanime.repositories.AnimeRepository
class AnimeService : AbstractService() {
@@ -12,21 +13,7 @@ class AnimeService : AbstractService() {
return animeRepository
}
- fun findByName(name: String?): Anime? {
- return animeRepository.findByName(name)
- }
-
- override fun saveOrUpdate(entity: Anime): Anime {
- val entityFromDb = findByName(entity.name) ?: return save(entity)
-
- if (entityFromDb.image != entity.image) {
- entityFromDb.image = entity.image
- }
-
- if (entityFromDb.description != entity.description) {
- entityFromDb.description = entity.description
- }
-
- return super.update(entityFromDb)
+ fun findByName(countryCode: CountryCode, name: String?): Anime? {
+ return animeRepository.findByName(countryCode, name)
}
}
\ No newline at end of file
diff --git a/src/main/kotlin/fr/shikkanime/services/CountryService.kt b/src/main/kotlin/fr/shikkanime/services/CountryService.kt
deleted file mode 100644
index d8c6e207..00000000
--- a/src/main/kotlin/fr/shikkanime/services/CountryService.kt
+++ /dev/null
@@ -1,31 +0,0 @@
-package fr.shikkanime.services
-
-import com.google.inject.Inject
-import fr.shikkanime.entities.Country
-import fr.shikkanime.repositories.CountryRepository
-import fr.shikkanime.utils.MapCache
-
-class CountryService : AbstractService() {
- @Inject
- private lateinit var countryRepository: CountryRepository
-
- private val codesCache: MapCache> = MapCache {
- countryRepository.findAllByCode(it.split(","))
- }
-
- override fun getRepository(): CountryRepository {
- return countryRepository
- }
-
- fun findByName(name: String): Country? {
- return countryRepository.findByName(name)
- }
-
- fun findByCode(code: String): Country? {
- return countryRepository.findByCode(code)
- }
-
- fun findAllByCode(codes: Collection): List {
- return codesCache[codes.joinToString(",")]
- }
-}
\ No newline at end of file
diff --git a/src/main/kotlin/fr/shikkanime/services/EpisodeService.kt b/src/main/kotlin/fr/shikkanime/services/EpisodeService.kt
index fd5d6742..dd6601a6 100644
--- a/src/main/kotlin/fr/shikkanime/services/EpisodeService.kt
+++ b/src/main/kotlin/fr/shikkanime/services/EpisodeService.kt
@@ -8,9 +8,6 @@ class EpisodeService : AbstractService() {
@Inject
private lateinit var episodeRepository: EpisodeRepository
- @Inject
- private lateinit var platformService: PlatformService
-
@Inject
private lateinit var animeService: AnimeService
@@ -18,40 +15,17 @@ class EpisodeService : AbstractService() {
return episodeRepository
}
+ fun findAllHashes(): List {
+ return episodeRepository.findAllHashes()
+ }
+
fun findByHash(hash: String?): Episode? {
return episodeRepository.findByHash(hash)
}
- override fun saveOrUpdate(entity: Episode): Episode {
- entity.platform = platformService.saveOrUpdate(entity.platform!!)
- entity.anime = animeService.saveOrUpdate(entity.anime!!)
-
- val entityFromDb = findByHash(entity.hash) ?: return save(entity)
-
- if (entityFromDb.season != entity.season) {
- entityFromDb.season = entity.season
- }
-
- if (entityFromDb.number != entity.number) {
- entityFromDb.number = entity.number
- }
-
- if (entityFromDb.title != entity.title) {
- entityFromDb.title = entity.title
- }
-
- if (entityFromDb.url != entity.url) {
- entityFromDb.url = entity.url
- }
-
- if (entityFromDb.image != entity.image) {
- entityFromDb.image = entity.image
- }
-
- if (entityFromDb.duration != entity.duration) {
- entityFromDb.duration = entity.duration
- }
-
- return super.update(entityFromDb)
+ override fun save(entity: Episode): Episode {
+ entity.anime = animeService.findByName(entity.anime!!.countryCode!!, entity.anime!!.name!!)
+ ?: animeService.save(entity.anime!!)
+ return super.save(entity)
}
}
\ No newline at end of file
diff --git a/src/main/kotlin/fr/shikkanime/services/ImageService.kt b/src/main/kotlin/fr/shikkanime/services/ImageService.kt
new file mode 100644
index 00000000..3e372a37
--- /dev/null
+++ b/src/main/kotlin/fr/shikkanime/services/ImageService.kt
@@ -0,0 +1,121 @@
+package fr.shikkanime.services
+
+import fr.shikkanime.utils.CompressionManager
+import fr.shikkanime.utils.HttpRequest
+import fr.shikkanime.utils.ObjectParser
+import io.ktor.client.statement.*
+import io.ktor.http.*
+import kotlinx.coroutines.runBlocking
+import java.io.File
+import java.util.*
+import java.util.concurrent.Executors
+import kotlin.system.measureTimeMillis
+
+object ImageService {
+ data class Image(
+ val url: String,
+ var bytes: ByteArray = byteArrayOf(),
+ var ratio: Double = 0.0,
+ ) {
+ override fun equals(other: Any?): Boolean {
+ if (this === other) return true
+ if (javaClass != other?.javaClass) return false
+
+ other as Image
+
+ if (url != other.url) return false
+ if (!bytes.contentEquals(other.bytes)) return false
+ if (ratio != other.ratio) return false
+
+ return true
+ }
+
+ override fun hashCode(): Int {
+ var result = url.hashCode()
+ result = 31 * result + bytes.contentHashCode()
+ result = 31 * result + ratio.hashCode()
+ return result
+ }
+ }
+
+ private val threadPool = Executors.newFixedThreadPool(Runtime.getRuntime().availableProcessors() - 1)
+ private val file = File("images-cache.shikk")
+ private var cache = mutableMapOf()
+
+ fun loadCache() {
+ if (!file.exists()) {
+ saveCache()
+ return
+ }
+
+ println("Loading images cache...")
+
+ val take = measureTimeMillis {
+ val json = String(CompressionManager.fromGzip(file.readBytes()))
+ val map = ObjectParser.fromJson(json, mutableMapOf().javaClass)
+ cache = map
+ }
+
+ println("Loaded images cache in $take ms")
+ }
+
+ fun saveCache() {
+ if (!file.exists()) {
+ file.createNewFile()
+ }
+
+ println("Saving images cache...")
+
+ val take = measureTimeMillis {
+ file.writeBytes(CompressionManager.toGzip(ObjectParser.toJson(cache).toByteArray()))
+ }
+
+ println("Saved images cache in $take ms")
+ }
+
+ fun contains(uuid: UUID) = cache.containsKey(uuid)
+
+ fun add(uuid: UUID, url: String) {
+ val image = Image(url)
+ cache[uuid] = image
+
+ threadPool.submit {
+ val take = measureTimeMillis {
+ try {
+ println("Loading image $url...")
+ val httpResponse = runBlocking {
+ HttpRequest().get(url)
+ }
+
+ if (httpResponse.status != HttpStatusCode.OK) {
+ println("Failed to load image $url")
+ cache.remove(uuid)
+ return@measureTimeMillis
+ }
+
+ val bytes = runBlocking {
+ httpResponse.readBytes()
+ }
+
+ if (bytes.isEmpty()) {
+ println("Failed to load image $url")
+ cache.remove(uuid)
+ return@measureTimeMillis
+ }
+
+ println("Encoding image to WebP...")
+ val webp = CompressionManager.encodeToWebP(bytes)
+ image.bytes = webp
+ image.ratio = bytes.size.toDouble() / webp.size.toDouble()
+ cache[uuid] = image
+ } catch (e: Exception) {
+ println("Failed to load image $url: ${e.message}")
+ cache.remove(uuid)
+ }
+ }
+
+ println("Ratio for $url: ${String.format("%.2f", image.ratio * 100)}%, took $take ms")
+ println("Encoded image to WebP in ${take}ms")
+ }
+ }
+}
\ No newline at end of file
diff --git a/src/main/kotlin/fr/shikkanime/services/MetricService.kt b/src/main/kotlin/fr/shikkanime/services/MetricService.kt
index 9aa5cfb8..f1d61483 100644
--- a/src/main/kotlin/fr/shikkanime/services/MetricService.kt
+++ b/src/main/kotlin/fr/shikkanime/services/MetricService.kt
@@ -7,21 +7,16 @@ import fr.shikkanime.repositories.MetricRepository
import fr.shikkanime.utils.MapCache
import java.time.Duration
import java.time.ZonedDateTime
-import java.time.temporal.ChronoUnit
class MetricService : AbstractService() {
@Inject
private lateinit var metricRepository: MetricRepository
- private val averageCpuLoadCache = MapCache(
- Duration.of(1, ChronoUnit.HOURS)
- ) {
+ private val averageCpuLoadCache = MapCache(Duration.ofHours(1)) {
metricRepository.getAverageCpuLoad(it.from, it.to)
}
- private val averageMemoryUsageCache = MapCache(
- Duration.of(1, ChronoUnit.HOURS)
- ) {
+ private val averageMemoryUsageCache = MapCache(Duration.ofHours(1)) {
metricRepository.getAverageMemoryUsage(it.from, it.to)
}
diff --git a/src/main/kotlin/fr/shikkanime/services/PlatformService.kt b/src/main/kotlin/fr/shikkanime/services/PlatformService.kt
deleted file mode 100644
index 9f35db22..00000000
--- a/src/main/kotlin/fr/shikkanime/services/PlatformService.kt
+++ /dev/null
@@ -1,36 +0,0 @@
-package fr.shikkanime.services
-
-import com.google.inject.Inject
-import fr.shikkanime.entities.Platform
-import fr.shikkanime.repositories.PlatformRepository
-
-class PlatformService : AbstractService() {
- @Inject
- private lateinit var platformRepository: PlatformRepository
-
- override fun getRepository(): PlatformRepository {
- return platformRepository
- }
-
- fun findByName(name: String?): Platform? {
- if (name.isNullOrBlank()) {
- return null
- }
-
- return platformRepository.findByName(name)
- }
-
- override fun saveOrUpdate(entity: Platform): Platform {
- val entityFromDb = findByName(entity.name) ?: return save(entity)
-
- if (entityFromDb.url != entity.url) {
- entityFromDb.url = entity.url
- }
-
- if (entityFromDb.image != entity.image) {
- entityFromDb.image = entity.image
- }
-
- return super.update(entityFromDb)
- }
-}
\ No newline at end of file
diff --git a/src/main/kotlin/fr/shikkanime/utils/CompressionManager.kt b/src/main/kotlin/fr/shikkanime/utils/CompressionManager.kt
new file mode 100644
index 00000000..ac039348
--- /dev/null
+++ b/src/main/kotlin/fr/shikkanime/utils/CompressionManager.kt
@@ -0,0 +1,43 @@
+package fr.shikkanime.utils
+
+import nu.pattern.OpenCV
+import org.opencv.core.MatOfByte
+import org.opencv.core.MatOfInt
+import org.opencv.imgcodecs.Imgcodecs
+import java.io.ByteArrayInputStream
+import java.io.ByteArrayOutputStream
+import java.util.zip.GZIPInputStream
+import java.util.zip.GZIPOutputStream
+
+object CompressionManager {
+ init {
+ OpenCV.loadLocally()
+ }
+
+ fun toGzip(bytes: ByteArray): ByteArray {
+ val byteArrayOutputStream = ByteArrayOutputStream()
+ val gzip = GZIPOutputStream(byteArrayOutputStream)
+ gzip.write(bytes)
+ gzip.close()
+ return byteArrayOutputStream.toByteArray()
+ }
+
+ fun fromGzip(bytes: ByteArray): ByteArray {
+ val gzip = GZIPInputStream(ByteArrayInputStream(bytes))
+ val compressed = gzip.readBytes()
+ gzip.close()
+ return compressed
+ }
+
+ fun encodeToWebP(image: ByteArray): ByteArray {
+ val matImage = Imgcodecs.imdecode(MatOfByte(*image), Imgcodecs.IMREAD_UNCHANGED)
+ val parameters = MatOfInt(Imgcodecs.IMWRITE_WEBP_QUALITY, 75)
+ val output = MatOfByte()
+
+ if (Imgcodecs.imencode(".webp", matImage, output, parameters)) {
+ return output.toArray()
+ } else {
+ throw Exception("Failed to encode image to WebP")
+ }
+ }
+}
\ No newline at end of file
diff --git a/src/main/kotlin/fr/shikkanime/utils/Constant.kt b/src/main/kotlin/fr/shikkanime/utils/Constant.kt
index cd05d2c6..33aeb7c9 100644
--- a/src/main/kotlin/fr/shikkanime/utils/Constant.kt
+++ b/src/main/kotlin/fr/shikkanime/utils/Constant.kt
@@ -1,6 +1,5 @@
package fr.shikkanime.utils
-import com.google.gson.Gson
import com.google.inject.Guice
import com.google.inject.Injector
import fr.shikkanime.modules.DefaultModule
@@ -8,7 +7,6 @@ import fr.shikkanime.platforms.AbstractPlatform
import org.reflections.Reflections
object Constant {
- val gson = Gson()
val reflections = Reflections("fr.shikkanime")
val injector: Injector = Guice.createInjector(DefaultModule())
diff --git a/src/main/kotlin/fr/shikkanime/utils/Database.kt b/src/main/kotlin/fr/shikkanime/utils/Database.kt
index cc9f12c9..8dc978d9 100644
--- a/src/main/kotlin/fr/shikkanime/utils/Database.kt
+++ b/src/main/kotlin/fr/shikkanime/utils/Database.kt
@@ -48,7 +48,11 @@ class Database {
}
}
- constructor() : this(File("hibernate.cfg.xml"))
+ constructor() : this(
+ File(
+ ClassLoader.getSystemClassLoader().getResource("hibernate.cfg.xml")?.file ?: "hibernate.cfg.xml"
+ )
+ )
fun getEntityManager(): EntityManager {
return sessionFactory.createEntityManager()
diff --git a/src/main/kotlin/fr/shikkanime/utils/HttpRequest.kt b/src/main/kotlin/fr/shikkanime/utils/HttpRequest.kt
index bbc8c7f3..c780ee56 100644
--- a/src/main/kotlin/fr/shikkanime/utils/HttpRequest.kt
+++ b/src/main/kotlin/fr/shikkanime/utils/HttpRequest.kt
@@ -1,19 +1,98 @@
package fr.shikkanime.utils
+import com.microsoft.playwright.Browser
+import com.microsoft.playwright.BrowserType
+import com.microsoft.playwright.Page
+import com.microsoft.playwright.Playwright
import io.ktor.client.*
import io.ktor.client.engine.cio.*
import io.ktor.client.request.*
import io.ktor.client.statement.*
+import org.jsoup.Jsoup
+import org.jsoup.nodes.Document
+import kotlin.system.measureTimeMillis
class HttpRequest {
- suspend fun get(url: String): HttpResponse {
+ private var isBrowserInitialized = false
+ private var playwright: Playwright? = null
+ private var browser: Browser? = null
+ private var page: Page? = null
+
+ suspend fun get(url: String, headers: Map = emptyMap()): HttpResponse {
val httpClient = HttpClient(CIO)
println("Making request to $url... (GET)")
val start = System.currentTimeMillis()
- val response = httpClient.get(url)
+ val response = httpClient.get(url) {
+ headers.forEach { (key, value) ->
+ header(key, value)
+ }
+ }
httpClient.close()
println("Request to $url done in ${System.currentTimeMillis() - start}ms (GET)")
return response
}
+
+ suspend fun post(url: String, headers: Map = emptyMap(), body: String): HttpResponse {
+ val httpClient = HttpClient(CIO)
+ println("Making request to $url... (POST)")
+ val start = System.currentTimeMillis()
+ val response = httpClient.post(url) {
+ headers.forEach { (key, value) ->
+ header(key, value)
+ }
+
+ setBody(body)
+ }
+ httpClient.close()
+ println("Request to $url done in ${System.currentTimeMillis() - start}ms (POST)")
+ return response
+ }
+
+ private fun initBrowser() {
+ if (isBrowserInitialized) {
+ return
+ }
+
+ playwright = Playwright.create()
+ browser = playwright?.firefox()?.launch(BrowserType.LaunchOptions().setHeadless(true))
+ page = browser?.newPage()
+ page?.setDefaultTimeout(15_000.0)
+ page?.setDefaultNavigationTimeout(15_000.0)
+ isBrowserInitialized = true
+ }
+
+ fun getBrowser(url: String, selector: String? = null, retry: Int = 3): Document {
+ initBrowser()
+ println("Making request to $url... (BROWSER)")
+
+ val takeMs = measureTimeMillis {
+ try {
+ page?.navigate(url)
+
+ if (selector != null) {
+ page?.waitForSelector(selector)
+ } else {
+ page?.waitForLoadState()
+ }
+ } catch (e: Exception) {
+ if (retry > 0) {
+ println("Retrying...")
+ return getBrowser(url, selector, retry - 1)
+ }
+
+ throw e
+ }
+ }
+
+ val content = page?.content()
+ println("Request to $url done in ${takeMs}ms (BROWSER)")
+ return Jsoup.parse(content ?: throw Exception("Content is null"))
+ }
+
+ fun closeBrowser() {
+ page?.close()
+ browser?.close()
+ playwright?.close()
+ }
}
\ No newline at end of file
diff --git a/src/main/kotlin/fr/shikkanime/utils/MapCache.kt b/src/main/kotlin/fr/shikkanime/utils/MapCache.kt
index faf499b5..92d29a35 100644
--- a/src/main/kotlin/fr/shikkanime/utils/MapCache.kt
+++ b/src/main/kotlin/fr/shikkanime/utils/MapCache.kt
@@ -6,17 +6,22 @@ import com.google.common.cache.LoadingCache
import java.time.Duration
class MapCache(
- duration: Duration? = null,
+ private var duration: Duration? = null,
private val classes: List> = listOf(),
private val block: (K) -> V,
) {
- private val cache: LoadingCache
+ private lateinit var cache: LoadingCache
init {
+ setCache()
+ globalCaches.add(this)
+ }
+
+ private fun MapCache.setCache() {
val builder = CacheBuilder.newBuilder()
if (duration != null) {
- builder.expireAfterWrite(duration)
+ builder.expireAfterWrite(duration!!)
}
cache = builder
@@ -25,20 +30,31 @@ class MapCache(
return block(key)
}
})
+ }
- caches.add(this)
+ fun has(key: K): Boolean {
+ return cache.getIfPresent(key) != null
}
operator fun get(key: K): V {
return cache.getUnchecked(key)
}
+ operator fun set(key: K, value: V & Any) {
+ cache.put(key, value)
+ }
+
+ fun resetWithNewDuration(duration: Duration) {
+ this.duration = duration
+ setCache()
+ }
+
companion object {
- private val caches: MutableList> = mutableListOf()
+ private val globalCaches: MutableList> = mutableListOf()
fun invalidate(vararg classes: Class<*>) {
classes.forEach { clazz ->
- caches.filter { it.classes.contains(clazz) }.forEach { it.cache.invalidateAll() }
+ globalCaches.filter { it.classes.contains(clazz) }.forEach { it.cache.invalidateAll() }
}
}
}
diff --git a/src/main/kotlin/fr/shikkanime/utils/ObjectParser.kt b/src/main/kotlin/fr/shikkanime/utils/ObjectParser.kt
new file mode 100644
index 00000000..ec1a3ac9
--- /dev/null
+++ b/src/main/kotlin/fr/shikkanime/utils/ObjectParser.kt
@@ -0,0 +1,48 @@
+package fr.shikkanime.utils
+
+import com.ctc.wstx.stax.WstxInputFactory
+import com.fasterxml.jackson.databind.ObjectMapper
+import com.fasterxml.jackson.dataformat.xml.XmlMapper
+import com.google.gson.Gson
+import com.google.gson.JsonObject
+
+object ObjectParser {
+ private val gson = Gson()
+ private val objectMapper = ObjectMapper()
+ private val xmlInputFactory = WstxInputFactory()
+ private val xmlMapper: XmlMapper
+
+ init {
+ xmlInputFactory.configureForSpeed()
+ xmlInputFactory.setProperty(WstxInputFactory.IS_NAMESPACE_AWARE, false)
+ xmlMapper = XmlMapper(xmlInputFactory)
+ }
+
+ fun fromJson(json: String, clazz: Class): T {
+ return gson.fromJson(json, clazz)
+ }
+
+ fun toJson(obj: T): String {
+ return gson.toJson(obj)
+ }
+
+ fun JsonObject.getAsString(key: String): String? {
+ return this[key]?.asString
+ }
+
+ fun JsonObject.getAsBoolean(key: String): Boolean? {
+ return this[key]?.asBoolean
+ }
+
+ fun JsonObject.getAsInt(key: String): Int? {
+ return this[key]?.asInt
+ }
+
+ fun JsonObject.getAsLong(key: String, default: Long): Long {
+ return this[key]?.asLong ?: default
+ }
+
+ fun fromXml(xml: String, clazz: Class): T {
+ return objectMapper.writeValueAsString(xmlMapper.readTree(xml)).let { gson.fromJson(it, clazz) }
+ }
+}
\ No newline at end of file
diff --git a/src/main/resources/db/changelog/2023/12/01-changelog.xml b/src/main/resources/db/changelog/2023/12/01-changelog.xml
index d2f3449d..4153631f 100644
--- a/src/main/resources/db/changelog/2023/12/01-changelog.xml
+++ b/src/main/resources/db/changelog/2023/12/01-changelog.xml
@@ -3,7 +3,8 @@
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xmlns="http://www.liquibase.org/xml/ns/dbchangelog"
xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog
- http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-4.25.xsd" objectQuotingStrategy="QUOTE_ONLY_RESERVED_WORDS">
+ http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-4.25.xsd"
+ objectQuotingStrategy="QUOTE_ONLY_RESERVED_WORDS">
@@ -38,74 +39,6 @@
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- SELECT COUNT(*)
- FROM country
- WHERE country_code = 'FR'
-
-
-
-
- France
- FR
-
-
-
-
@@ -119,8 +52,8 @@
primaryKey="true"
primaryKeyName="pk_anime"/>
-
+
+ type="VARCHAR(1000)">
+ type="VARCHAR(1000)"/>
-
-
-
-
-
-
-
-
-
-
-
+
@@ -168,8 +87,8 @@
primaryKey="true"
primaryKeyName="pk_episode"/>
-
+
-
+
@@ -201,13 +120,13 @@
+ type="VARCHAR(1000)"/>
+ type="VARCHAR(1000)">
+ type="VARCHAR(1000)">
-
+
@@ -230,18 +149,4 @@
referencedColumnNames="uuid"
referencedTableName="anime"/>
-
-
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/src/main/resources/db/changelog/db.changelog-master.xml b/src/main/resources/db/changelog/db.changelog-master.xml
index a1282c8f..3c4a299a 100644
--- a/src/main/resources/db/changelog/db.changelog-master.xml
+++ b/src/main/resources/db/changelog/db.changelog-master.xml
@@ -3,6 +3,7 @@
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xmlns="http://www.liquibase.org/xml/ns/dbchangelog"
xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog
- http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-4.25.xsd" objectQuotingStrategy="QUOTE_ONLY_RESERVED_WORDS">
+ http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-4.25.xsd"
+ objectQuotingStrategy="QUOTE_ONLY_RESERVED_WORDS">
\ No newline at end of file
diff --git a/src/main/resources/templates/admin/_layout.ftl b/src/main/resources/templates/admin/_layout.ftl
index 15fd11d6..269dad6d 100644
--- a/src/main/resources/templates/admin/_layout.ftl
+++ b/src/main/resources/templates/admin/_layout.ftl
@@ -10,9 +10,10 @@
-
-
+
+