Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Dev #317

Merged
merged 3 commits into from
Mar 27, 2024
Merged

Dev #317

Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 1 addition & 3 deletions build.gradle.kts
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,6 @@ val guiceVersion = "7.0.0"
val liquibaseCoreVersion = "4.26.0"
val quartzVersion = "2.5.0-rc1"
val guavaVersion = "33.1.0-jre"
val jacksonVersion = "2.17.0"
val playwrightVersion = "1.42.0"
val jsoupVersion = "1.17.2"
val gsonVersion = "2.10.1"
Expand All @@ -29,7 +28,7 @@ plugins {
kotlin("jvm") version "2.0.0-Beta5"
id("io.ktor.plugin") version "2.3.9"
jacoco
id("org.sonarqube") version "4.4.1.3373"
id("org.sonarqube") version "5.0.0.4638"
}

group = "fr.shikkanime"
Expand Down Expand Up @@ -79,7 +78,6 @@ dependencies {
implementation("org.liquibase:liquibase-core:$liquibaseCoreVersion")
implementation("org.quartz-scheduler:quartz:$quartzVersion")
implementation("com.google.guava:guava:$guavaVersion")
implementation("com.fasterxml.jackson.dataformat:jackson-dataformat-xml:$jacksonVersion")
implementation("com.microsoft.playwright:playwright:$playwrightVersion")
implementation("org.jsoup:jsoup:$jsoupVersion")
implementation("com.google.code.gson:gson:$gsonVersion")
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,6 @@ enum class ConfigPropertyKey(val key: String) {
TWITTER_CONSUMER_SECRET("twitter_consumer_secret"),
TWITTER_ACCESS_TOKEN("twitter_access_token"),
TWITTER_ACCESS_TOKEN_SECRET("twitter_access_token_secret"),
USE_CRUNCHYROLL_API("use_crunchyroll_api"),
SEO_DESCRIPTION("seo_description"),
SOCIAL_NETWORK_EPISODES_SIZE_LIMIT("social_network_episodes_size_limit"),
FETCH_OLD_EPISODE_DESCRIPTION_SIZE("fetch_old_episode_description_size"),
Expand Down
355 changes: 50 additions & 305 deletions src/main/kotlin/fr/shikkanime/platforms/CrunchyrollPlatform.kt

Large diffs are not rendered by default.

Original file line number Diff line number Diff line change
Expand Up @@ -169,7 +169,6 @@ class EpisodeRepository : AbstractRepository<Episode>() {
"""
FROM Episode e
WHERE e.anime.countryCode = :countryCode AND e.releaseDateTime BETWEEN :start AND :end
ORDER BY e.releaseDateTime ASC, LOWER(e.anime.name) ASC
""".trimIndent(),
getEntityClass()
)
Expand Down
1 change: 1 addition & 0 deletions src/main/kotlin/fr/shikkanime/services/AnimeService.kt
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,7 @@ class AnimeService : AbstractService<Anime, AnimeRepository>() {
)
).capitalizeWords()
val episodes = list.filter { it.releaseDateTime.dayOfWeek == date.dayOfWeek }
.sortedWith(compareBy({ it.releaseDateTime.toLocalTime() }, { it.anime?.name?.lowercase() }))

WeeklyAnimesDto(
dateTitle,
Expand Down
16 changes: 0 additions & 16 deletions src/main/kotlin/fr/shikkanime/utils/ObjectParser.kt
Original file line number Diff line number Diff line change
@@ -1,23 +1,11 @@
package fr.shikkanime.utils

import com.ctc.wstx.stax.WstxInputFactory
import com.fasterxml.jackson.databind.ObjectMapper
import com.fasterxml.jackson.dataformat.xml.XmlMapper
import com.google.gson.Gson
import com.google.gson.JsonObject
import com.google.gson.reflect.TypeToken

object ObjectParser {
private val gson = Gson()
private val objectMapper = ObjectMapper()
private val xmlInputFactory = WstxInputFactory()
private val xmlMapper: XmlMapper

init {
xmlInputFactory.configureForSpeed()
xmlInputFactory.setProperty(WstxInputFactory.IS_NAMESPACE_AWARE, false)
xmlMapper = XmlMapper(xmlInputFactory)
}

fun fromJson(json: String): JsonObject {
return gson.fromJson(json, JsonObject::class.java)
Expand Down Expand Up @@ -62,8 +50,4 @@ object ObjectParser {
fun JsonObject.getAsBoolean(key: String, default: Boolean): Boolean {
return this[key]?.asBoolean ?: default
}

fun <T> fromXml(xml: String, clazz: Class<T>): T {
return objectMapper.writeValueAsString(xmlMapper.readTree(xml)).let { gson.fromJson(it, clazz) }
}
}
22 changes: 22 additions & 0 deletions src/main/resources/db/changelog/2024/03/07-changelog.xml
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
<?xml version="1.0" encoding="UTF-8"?>
<databaseChangeLog
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xmlns="http://www.liquibase.org/xml/ns/dbchangelog"
xsi:schemaLocation="http://www.liquibase.org/xml/ns/dbchangelog
http://www.liquibase.org/xml/ns/dbchangelog/dbchangelog-4.26.xsd"
objectQuotingStrategy="QUOTE_ONLY_RESERVED_WORDS">
<property global="false" name="id" value="1711556853279"/>
<property global="false" name="author" value="Ziedelth"/>

<changeSet id="${id}-1" author="${author}" dbms="postgresql">
<preConditions onFail="MARK_RAN">
<sqlCheck expectedResult="1">SELECT COUNT(*)
FROM config
WHERE property_key = 'use_crunchyroll_api'</sqlCheck>
</preConditions>

<delete tableName="config">
<where>property_key = 'use_crunchyroll_api'</where>
</delete>
</changeSet>
</databaseChangeLog>
1 change: 1 addition & 0 deletions src/main/resources/db/changelog/db.changelog-master.xml
Original file line number Diff line number Diff line change
Expand Up @@ -36,4 +36,5 @@
<include file="/db/changelog/2024/03/04-changelog.xml"/>
<include file="/db/changelog/2024/03/05-changelog.xml"/>
<include file="/db/changelog/2024/03/06-changelog.xml"/>
<include file="/db/changelog/2024/03/07-changelog.xml"/>
</databaseChangeLog>
10 changes: 0 additions & 10 deletions src/test/kotlin/fr/shikkanime/OldEpisodeScraper.kt
Original file line number Diff line number Diff line change
@@ -1,13 +1,10 @@
package fr.shikkanime

import fr.shikkanime.caches.CountryCodeAnimeIdKeyCache
import fr.shikkanime.entities.Config
import fr.shikkanime.entities.Episode
import fr.shikkanime.entities.enums.ConfigPropertyKey
import fr.shikkanime.entities.enums.CountryCode
import fr.shikkanime.platforms.AnimationDigitalNetworkPlatform
import fr.shikkanime.platforms.CrunchyrollPlatform
import fr.shikkanime.services.ConfigService
import fr.shikkanime.services.EpisodeService
import fr.shikkanime.utils.Constant
import fr.shikkanime.utils.HttpRequest
Expand Down Expand Up @@ -62,13 +59,6 @@ fun main() {
println("Simulcasts: $simulcasts")

val episodes = mutableListOf<Episode>()
val configService = Constant.injector.getInstance(ConfigService::class.java)

configService.findByName(ConfigPropertyKey.USE_CRUNCHYROLL_API.key)?.let {
configService.delete(it)
}

configService.save(Config(propertyKey = ConfigPropertyKey.USE_CRUNCHYROLL_API.key, propertyValue = "true"))
val episodeService = Constant.injector.getInstance(EpisodeService::class.java)
val adnPlatform = Constant.injector.getInstance(AnimationDigitalNetworkPlatform::class.java)
val crunchyrollPlatform = Constant.injector.getInstance(CrunchyrollPlatform::class.java)
Expand Down
205 changes: 1 addition & 204 deletions src/test/kotlin/fr/shikkanime/platforms/CrunchyrollPlatformTest.kt
Original file line number Diff line number Diff line change
@@ -1,8 +1,6 @@
package fr.shikkanime.platforms

import fr.shikkanime.caches.CountryCodeAnimeIdKeyCache
import fr.shikkanime.entities.Config
import fr.shikkanime.entities.enums.ConfigPropertyKey
import fr.shikkanime.entities.enums.CountryCode
import fr.shikkanime.entities.enums.LangType
import fr.shikkanime.services.ConfigService
Expand Down Expand Up @@ -38,213 +36,12 @@ class CrunchyrollPlatformTest {
MapCache.invalidate(Config::class.java)
}

@Test
fun fetchEpisodesXML() {
val s = "2023-12-11T18:00:00Z"
val zonedDateTime = ZonedDateTime.parse(s)

platform.simulcasts.set(
CountryCode.FR, setOf(
"16bit sensation: another layer",
"a girl & her guard dog",
"a playthrough of a certain dude's vrmmo life",
"a returner's magic should be special",
"after-school hanako-kun",
"all saints street",
"arknights",
"berserk of gluttony",
"bullbuster",
"butareba -the story of a man turned into a pig-",
"captain tsubasa saison 2, junior youth arc",
"dead mount death play",
"dr. stone",
"firefighter daigo: rescuer in orange",
"frieren",
"girlfriend, girlfriend",
"heaven official's blessing",
"hypnosismic -division rap battle- rhyme anima",
"i'm giving the disgraced noble lady i rescued a crash course in naughtiness",
"i'm in love with the villainess",
"idolish7",
"je survivrai grâce aux potions !",
"jujutsu kaisen",
"kamierabi god.app",
"kawagoe boys sing -now or never-",
"kenshin le vagabond (2023)",
"kizuna no allele",
"l'attaque des titans",
"la valkyrie aux cheveux de jais",
"les 100 petites amies qui t'aiiiment à en mourir",
"les carnets de l'apothicaire",
"les quatre frères yuzuki",
"let me check the walkthrough first",
"mf ghost",
"migi&dali",
"moi, quand je me réincarne en slime",
"my new boss is goofy",
"one piece",
"our dating story: the experienced you and the inexperienced me",
"overtake!",
"paradox live the animation",
"power of hope ~precure full bloom~",
"reign of the seven spellblades",
"ron kamonohashi: deranged detective",
"sasaki and miyano",
"shadowverse",
"shangri-la frontier",
"shy",
"spy x family",
"stardust telepath",
"tearmoon empire",
"tenchi muyo! gxp paradise starting",
"the ancient magus bride",
"the faraway paladin",
"the idolm@ster million live!",
"the kingdoms of ruin",
"the rising of the shield hero",
"the saint's magic power is omnipotent",
"umamusume: pretty derby",
"under ninja",
"witch family!",
"am i actually the strongest?",
"atelier ryza: ever darkness & the secret hideout the animation",
"ayaka",
"ayakashi triangle",
"bang dream! it's mygo!!!!!",
"bungo stray dogs",
"cardfight!! vanguard overdress",
"classroom for heroes",
"fate/strange fake -whispers of dawn-",
"horimiya",
"la princesse & la bête",
"liar, liar",
"link click",
"malevolent spirits: mononogatari",
"masamune-kun's revenge",
"mon chat à tout faire est encore tout déprimé",
"mushoku tensei: jobless reincarnation",
"my tiny senpai",
"my unique skill makes me op even at level 1",
"nier:automata ver1.1a",
"reborn as a vending machine, i now wander the dungeon",
"rent-a-girlfriend",
"sainte cecilia et le pasteur lawrence",
"sugar apple fairy tale",
"sweet reincarnation",
"sword art online -fulldive-",
"tenpuru : on ne vit pas de solitude et d'eau fraîche",
"the devil is a part-timer!",
"the duke of death and his maid",
"the gene of ai",
"the girl i like forgot her glasses",
"the great cleric",
"the misfit of demon king academy",
"theatre of darkness: yamishibai",
"undead murder farce",
"zom 100: bucket list of the dead"
)
)

platform.animeInfoCache.set(
CountryCodeAnimeIdKeyCache(CountryCode.FR, "im-in-love-with-the-villainess"),
CrunchyrollPlatform.CrunchyrollAnimeContent("https://www.crunchyroll.com/imgsrv/display/thumbnail/480x720/catalog/crunchyroll/3f84cdad5e39d6d35ffc2b127c0509c7.jpe")
)
platform.animeInfoCache.set(
CountryCodeAnimeIdKeyCache(CountryCode.FR, "shy"),
CrunchyrollPlatform.CrunchyrollAnimeContent("https://www.crunchyroll.com/imgsrv/display/thumbnail/480x720/catalog/crunchyroll/d117d6da529adb8ee4d972af79024365.jpe")
)
platform.animeInfoCache.set(
CountryCodeAnimeIdKeyCache(CountryCode.FR, "dead-mount-death-play"),
CrunchyrollPlatform.CrunchyrollAnimeContent("https://www.crunchyroll.com/imgsrv/display/thumbnail/480x720/catalog/crunchyroll/3c9bebb79d5abf889928349df37a42b0.jpe")
)
platform.animeInfoCache.set(
CountryCodeAnimeIdKeyCache(CountryCode.FR, "kawagoe-boys-sing-now-or-never-"),
CrunchyrollPlatform.CrunchyrollAnimeContent("https://www.crunchyroll.com/imgsrv/display/thumbnail/480x720/catalog/crunchyroll/cfdf23b73e919c35c4c6168d0939907a.jpe")
)
platform.animeInfoCache.set(
CountryCodeAnimeIdKeyCache(CountryCode.FR, "ron-kamonohashis-forbidden-deductions"),
CrunchyrollPlatform.CrunchyrollAnimeContent("https://www.crunchyroll.com/imgsrv/display/thumbnail/480x720/catalog/crunchyroll/a755fe3f774aedeb7115a6d0419c5fe7.jpe")
)
platform.animeInfoCache.set(
CountryCodeAnimeIdKeyCache(CountryCode.FR, "stardust-telepath"),
CrunchyrollPlatform.CrunchyrollAnimeContent("https://www.crunchyroll.com/imgsrv/display/thumbnail/480x720/catalog/crunchyroll/9b1a36e0ebf66dc87f4fa8f623bf1ec5.jpe")
)
platform.animeInfoCache.set(
CountryCodeAnimeIdKeyCache(CountryCode.FR, "shangri-la-frontier"),
CrunchyrollPlatform.CrunchyrollAnimeContent("https://www.crunchyroll.com/imgsrv/display/thumbnail/480x720/catalog/crunchyroll/a2f948157077e3d65471329d9dd43be1.jpe")
)
platform.animeInfoCache.set(
CountryCodeAnimeIdKeyCache(CountryCode.FR, "mf-ghost"),
CrunchyrollPlatform.CrunchyrollAnimeContent("https://www.crunchyroll.com/imgsrv/display/thumbnail/480x720/catalog/crunchyroll/00c8ed6414e0ef37dbacf36bec86565a.jpe")
)
platform.animeInfoCache.set(
CountryCodeAnimeIdKeyCache(CountryCode.FR, "berserk-of-gluttony"),
CrunchyrollPlatform.CrunchyrollAnimeContent("https://www.crunchyroll.com/imgsrv/display/thumbnail/480x720/catalog/crunchyroll/a5ebfaaf02d69d1c0b254f4a49c43082.jpe")
)
platform.animeInfoCache.set(
CountryCodeAnimeIdKeyCache(CountryCode.FR, "overtake"),
CrunchyrollPlatform.CrunchyrollAnimeContent("https://www.crunchyroll.com/imgsrv/display/thumbnail/480x720/catalog/crunchyroll/150047844fbb4c07a3c57b5def7b42b0.jpe")
)
platform.animeInfoCache.set(
CountryCodeAnimeIdKeyCache(
CountryCode.FR,
"the-family-circumstances-of-the-irregular-witch"
),
CrunchyrollPlatform.CrunchyrollAnimeContent("https://www.crunchyroll.com/imgsrv/display/thumbnail/480x720/catalog/crunchyroll/43fce24dc917ee4e415aeae5231be54e.jpe")
)
platform.animeInfoCache.set(
CountryCodeAnimeIdKeyCache(
CountryCode.FR,
"the-100-girlfriends-who-really-really-really-really-really-love-you"
),
CrunchyrollPlatform.CrunchyrollAnimeContent("https://www.crunchyroll.com/imgsrv/display/thumbnail/480x720/catalog/crunchyroll/0ab1a205c9c541ad87cefdbe7fd23390.jpe")
)
platform.animeInfoCache.set(
CountryCodeAnimeIdKeyCache(CountryCode.FR, "captain-tsubasa-junior-youth-arc"),
CrunchyrollPlatform.CrunchyrollAnimeContent("https://www.crunchyroll.com/imgsrv/display/thumbnail/480x720/catalog/crunchyroll/f0dcc54fec6d201f5a2c7abd01d56e09.jpe")
)
platform.animeInfoCache.set(
CountryCodeAnimeIdKeyCache(CountryCode.FR, "the-idolmster-million-live"),
CrunchyrollPlatform.CrunchyrollAnimeContent("https://www.crunchyroll.com/imgsrv/display/thumbnail/480x720/catalog/crunchyroll/550acd8c749fa3820d4f75868d769e71.jpe")
)
platform.animeInfoCache.set(
CountryCodeAnimeIdKeyCache(CountryCode.FR, "one-piece"),
CrunchyrollPlatform.CrunchyrollAnimeContent("https://www.crunchyroll.com/imgsrv/display/thumbnail/480x720/catalog/crunchyroll/f154230aab3191aba977f337d392f812.jpe")
)
platform.animeInfoCache.set(
CountryCodeAnimeIdKeyCache(CountryCode.FR, "the-apothecary-diaries"),
CrunchyrollPlatform.CrunchyrollAnimeContent("https://www.crunchyroll.com/imgsrv/display/thumbnail/480x720/catalog/crunchyroll/af8f1de4c1b2d5345294490a45fcb22d.jpe")
)

val episodes = platform.fetchEpisodes(
zonedDateTime,
File(
ClassLoader.getSystemClassLoader().getResource("crunchyroll/rss-${s.replace(':', '-')}.xml")?.file
?: throw Exception("File not found")
)
)

assertEquals(true, episodes.isNotEmpty())
assertEquals(18, episodes.size)
assertEquals("I'm in Love with the Villainess", episodes[0].anime?.name)
assertNotNull(episodes[0].description)
assertEquals("I'm in Love with the Villainess", episodes[1].anime?.name)
assertNotNull(episodes[1].description)
assertEquals("SHY", episodes[2].anime?.name)
assertNotNull(episodes[2].description)
assertEquals("Dead Mount Death Play", episodes[3].anime?.name)
assertNotNull(episodes[3].description)
}

@Test
fun fetchEpisodesJSON() {
configService.save(Config(propertyKey = ConfigPropertyKey.USE_CRUNCHYROLL_API.key, propertyValue = "true"))
MapCache.invalidate(Config::class.java)

val s = "2024-01-24T18:45:00Z"
val zonedDateTime = ZonedDateTime.parse(s)

platform.simulcasts.set(CountryCode.FR, setOf("metallic rouge"))
platform.simulcasts[CountryCode.FR] = setOf("metallic rouge")

val episodes = platform.fetchEpisodes(
zonedDateTime,
Expand Down
Loading
Loading