add ip abuse for bot detection

Shikkanime · Feb 18, 2025 · 600c7d2 · 600c7d2
1 parent 5083c50
commit 600c7d2
Show file tree

Hide file tree

Showing 3 changed files with 54 additions and 35 deletions.
diff --git a/src/main/kotlin/fr/shikkanime/modules/Routing.kt b/src/main/kotlin/fr/shikkanime/modules/Routing.kt
@@ -7,6 +7,7 @@ import fr.shikkanime.entities.enums.ConfigPropertyKey
 import fr.shikkanime.entities.enums.CountryCode
 import fr.shikkanime.entities.enums.LangType
 import fr.shikkanime.entities.enums.Platform
+import fr.shikkanime.services.caches.BotDetectorCache
 import fr.shikkanime.services.caches.ConfigCacheService
 import fr.shikkanime.utils.Constant
 import fr.shikkanime.utils.LoggerFactory
@@ -51,6 +52,7 @@ import kotlin.reflect.jvm.jvmErasure
 
 private val logger = LoggerFactory.getLogger("Routing")
 private val callStartTime = AttributeKey<ZonedDateTime>("CallStartTime")
+private val attributeKey = AttributeKey<Boolean>("isBot")
 
 fun Application.configureRouting() {
     val configCacheService = Constant.injector.getInstance(ConfigCacheService::class.java)
@@ -118,8 +120,9 @@ fun logCallDetails(call: ApplicationCall, statusCode: HttpStatusCode? = null) {
     val path = call.request.path()
     val ipAddress = call.request.header("X-Forwarded-For") ?: call.request.origin.remoteHost
     val userAgent = call.request.userAgent() ?: "Unknown"
+    val isBot = call.attributes.getOrNull(attributeKey) == true
 
-    logger.info("[$ipAddress - $userAgent] ($status - $duration ms) $httpMethod ${call.request.origin.uri} -> $path")
+    logger.info("[$ipAddress - $userAgent${if (isBot) " (BOT)" else ""}] ($status - $duration ms) $httpMethod ${call.request.origin.uri} -> $path")
 }
 
 private fun Routing.createRoutes() {
@@ -229,7 +232,17 @@ suspend fun handleTemplateResponse(
     val model = response.data["model"]
     require(model is Map<*, *>) { "Model must be a map" }
     val mutableMap = model.toMutableMap()
-    setGlobalAttributes(ipAddress, userAgent, mutableMap, controller, replacedPath, response.data["title"] as String?)
+
+    var isBot = false
+    val configCacheService = Constant.injector.getInstance(ConfigCacheService::class.java)
+    val botDetectorCache = Constant.injector.getInstance(BotDetectorCache::class.java)
+
+    if (!configCacheService.getValueAsBoolean(ConfigPropertyKey.DISABLE_BOT_DETECTION) && botDetectorCache.isBot(clientIp = ipAddress, userAgent = userAgent)) {
+        isBot = true
+        call.attributes.put(attributeKey, true)
+    }
+
+    setGlobalAttributes(isBot, mutableMap, controller, replacedPath, response.data["title"] as String?)
     call.respond(response.status, FreeMarkerContent(response.data["template"] as String, mutableMap, "", response.contentType))
 }
 

diff --git a/src/main/kotlin/fr/shikkanime/modules/SEOManager.kt b/src/main/kotlin/fr/shikkanime/modules/SEOManager.kt
@@ -6,7 +6,6 @@ import fr.shikkanime.dtos.mappings.EpisodeMappingDto
 import fr.shikkanime.entities.LinkObject
 import fr.shikkanime.entities.enums.ConfigPropertyKey
 import fr.shikkanime.services.caches.AnimeCacheService
-import fr.shikkanime.services.caches.BotDetectorCache
 import fr.shikkanime.services.caches.ConfigCacheService
 import fr.shikkanime.services.caches.SimulcastCacheService
 import fr.shikkanime.utils.Constant
@@ -16,8 +15,7 @@ private fun <T> List<T>.randomIfNotEmpty(): T? = if (isNotEmpty()) random() else
 private fun <T> List<T>.randomIfNotEmpty(predicate: (T) -> Boolean): T? = filter(predicate).randomIfNotEmpty()
 
 fun setGlobalAttributes(
-    ipAddress: String,
-    userAgent: String,
+    isBot: Boolean,
     modelMap: MutableMap<Any?, Any?>,
     controller: Any,
     replacedPath: String,
@@ -30,7 +28,6 @@ fun setGlobalAttributes(
     val configCacheService = Constant.injector.getInstance(ConfigCacheService::class.java)
     val simulcastCacheService = Constant.injector.getInstance(SimulcastCacheService::class.java)
     val animeCacheService = Constant.injector.getInstance(AnimeCacheService::class.java)
-    val botDetectorCache = Constant.injector.getInstance(BotDetectorCache::class.java)
 
     modelMap["su"] = StringUtils
     modelMap["links"] = getLinks(controller, replacedPath, simulcastCacheService)
@@ -42,7 +39,7 @@ fun setGlobalAttributes(
     modelMap["baseUrl"] = Constant.baseUrl
     modelMap["apiUrl"] = Constant.apiUrl
 
-    if (configCacheService.getValueAsBoolean(ConfigPropertyKey.DISABLE_BOT_DETECTION) || !botDetectorCache.isBot(clientIp = ipAddress, userAgent = userAgent)) {
+    if (!isBot) {
         modelMap["additionalHeadTags"] = configCacheService.getValueAsString(ConfigPropertyKey.ADDITIONAL_HEAD_TAGS)
     }
 

diff --git a/src/main/kotlin/fr/shikkanime/services/caches/BotDetectorCache.kt b/src/main/kotlin/fr/shikkanime/services/caches/BotDetectorCache.kt
@@ -20,7 +20,8 @@ class BotDetectorCache : AbstractCacheService {
     }
 
     private val httpRequest = HttpRequest()
-    private val ipv4Regex = Regex("^(?:[0-9]{1,3}\\.){3}[0-9]{1,3}$")
+    private val ipv4Regex = Regex("^(?:[0-9]{1,3}\\.){3}[0-9]{1,3}(?:/[0-9]+)*")
+    private val duration = Duration.ofDays(1)
 
     @Inject
     private lateinit var configCacheService: ConfigCacheService
@@ -44,9 +45,9 @@ class BotDetectorCache : AbstractCacheService {
         return true
     }
 
-    private fun getGoodBotsIP() = MapCache.getOrCompute(
-        "BotDetectorCache.getGoodBotsIP",
-        duration = Duration.ofDays(1),
+    private fun getGoodBotsIPs() = MapCache.getOrCompute(
+        "BotDetectorCache.getGoodBotsIPs",
+        duration = duration,
         key = DEFAULT_ALL_KEY
     ) {
         runBlocking {
@@ -57,13 +58,13 @@ class BotDetectorCache : AbstractCacheService {
                 return@runBlocking emptyList()
             }
 
-            response.bodyAsText().split("\n").filter { it.isNotBlank() || !ipv4Regex.matches(it) }
+            response.bodyAsText().split("\n").filter { it.isNotBlank() && ipv4Regex.matches(it) }
         }
     }
 
     private fun getGoodBotsRegex() = MapCache.getOrCompute(
         "BotDetectorCache.getGoodBotsRegex",
-        duration = Duration.ofDays(1),
+        duration = duration,
         key = DEFAULT_ALL_KEY
     ) {
         runBlocking {
@@ -81,34 +82,42 @@ class BotDetectorCache : AbstractCacheService {
         }
     }
 
-    fun isBot(clientIp: String? = null, userAgent: String? = null): Boolean {
-        if (!clientIp.isNullOrBlank()) {
-            val clientIpBytes = InetAddress.getByName(clientIp).address
-
-            getGoodBotsIP().forEach { botIp ->
-                if (botIp.contains("/")) {
-                    val (ip, cidr) = botIp.split("/")
-                    val mask = createMask(cidr.toInt())
-                    val networkIpBytes = InetAddress.getByName(ip).address
-
-                    if (isInRange(clientIpBytes, networkIpBytes, mask)) {
-                        return true
-                    }
-                } else if (clientIp == botIp) {
-                    return true
-                }
+    private fun getAbuseIPs() = MapCache.getOrCompute(
+        "BotDetectorCache.getAbuseIPs",
+        duration = duration,
+        key = DEFAULT_ALL_KEY
+    ) {
+        runBlocking {
+            val response =
+                httpRequest.get("https://raw.githubusercontent.com/borestad/blocklist-abuseipdb/main/abuseipdb-s100-30d.ipv4")
+
+            if (response.status != HttpStatusCode.OK) {
+                return@runBlocking emptyList()
             }
+
+            response.bodyAsText().split("\n").filter { it.isNotBlank() && ipv4Regex.matches(it) }
+        }
+    }
+
+    private fun isIpInRange(clientIp: String, botIp: String): Boolean {
+        return if (botIp.contains("/")) {
+            val (ip, cidr) = botIp.split("/")
+            val mask = createMask(cidr.toInt())
+            isInRange(InetAddress.getByName(clientIp).address, InetAddress.getByName(ip).address, mask)
+        } else clientIp == botIp
+    }
+
+    fun isBot(clientIp: String? = null, userAgent: String? = null): Boolean {
+        clientIp?.let {
+            if (getGoodBotsIPs().any { botIp -> isIpInRange(it, botIp) }) return true
+            if (getAbuseIPs().contains(it)) return true
         }
 
-        if (!userAgent.isNullOrBlank()) {
+        userAgent?.let {
             val regexes = getGoodBotsRegex().toMutableSet()
             configCacheService.getValueAsString(ConfigPropertyKey.BOT_ADDITIONAL_REGEX)?.toRegex()?.let { regexes.add(it) }
 
-            regexes.forEach { regex ->
-                if (userAgent.contains(regex)) {
-                    return true
-                }
-            }
+            if (regexes.any { regex -> it.contains(regex) }) return true
         }
 
         return false