Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

add ip abuse for bot detection #896

Merged
merged 1 commit into from
Feb 18, 2025
Merged

add ip abuse for bot detection #896

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 15 additions & 2 deletions src/main/kotlin/fr/shikkanime/modules/Routing.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ import fr.shikkanime.entities.enums.ConfigPropertyKey
import fr.shikkanime.entities.enums.CountryCode
import fr.shikkanime.entities.enums.LangType
import fr.shikkanime.entities.enums.Platform
import fr.shikkanime.services.caches.BotDetectorCache
import fr.shikkanime.services.caches.ConfigCacheService
import fr.shikkanime.utils.Constant
import fr.shikkanime.utils.LoggerFactory
Expand Down Expand Up @@ -51,6 +52,7 @@ import kotlin.reflect.jvm.jvmErasure

private val logger = LoggerFactory.getLogger("Routing")
private val callStartTime = AttributeKey<ZonedDateTime>("CallStartTime")
private val attributeKey = AttributeKey<Boolean>("isBot")

fun Application.configureRouting() {
val configCacheService = Constant.injector.getInstance(ConfigCacheService::class.java)
Expand Down Expand Up @@ -118,8 +120,9 @@ fun logCallDetails(call: ApplicationCall, statusCode: HttpStatusCode? = null) {
val path = call.request.path()
val ipAddress = call.request.header("X-Forwarded-For") ?: call.request.origin.remoteHost
val userAgent = call.request.userAgent() ?: "Unknown"
val isBot = call.attributes.getOrNull(attributeKey) == true

logger.info("[$ipAddress - $userAgent] ($status - $duration ms) $httpMethod ${call.request.origin.uri} -> $path")
logger.info("[$ipAddress - $userAgent${if (isBot) " (BOT)" else ""}] ($status - $duration ms) $httpMethod ${call.request.origin.uri} -> $path")
}

private fun Routing.createRoutes() {
Expand Down Expand Up @@ -229,7 +232,17 @@ suspend fun handleTemplateResponse(
val model = response.data["model"]
require(model is Map<*, *>) { "Model must be a map" }
val mutableMap = model.toMutableMap()
setGlobalAttributes(ipAddress, userAgent, mutableMap, controller, replacedPath, response.data["title"] as String?)

var isBot = false
val configCacheService = Constant.injector.getInstance(ConfigCacheService::class.java)
val botDetectorCache = Constant.injector.getInstance(BotDetectorCache::class.java)

if (!configCacheService.getValueAsBoolean(ConfigPropertyKey.DISABLE_BOT_DETECTION) && botDetectorCache.isBot(clientIp = ipAddress, userAgent = userAgent)) {
isBot = true
call.attributes.put(attributeKey, true)
}

setGlobalAttributes(isBot, mutableMap, controller, replacedPath, response.data["title"] as String?)
call.respond(response.status, FreeMarkerContent(response.data["template"] as String, mutableMap, "", response.contentType))
}

Expand Down
7 changes: 2 additions & 5 deletions src/main/kotlin/fr/shikkanime/modules/SEOManager.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,6 @@ import fr.shikkanime.dtos.mappings.EpisodeMappingDto
import fr.shikkanime.entities.LinkObject
import fr.shikkanime.entities.enums.ConfigPropertyKey
import fr.shikkanime.services.caches.AnimeCacheService
import fr.shikkanime.services.caches.BotDetectorCache
import fr.shikkanime.services.caches.ConfigCacheService
import fr.shikkanime.services.caches.SimulcastCacheService
import fr.shikkanime.utils.Constant
Expand All @@ -16,8 +15,7 @@ private fun <T> List<T>.randomIfNotEmpty(): T? = if (isNotEmpty()) random() else
private fun <T> List<T>.randomIfNotEmpty(predicate: (T) -> Boolean): T? = filter(predicate).randomIfNotEmpty()

fun setGlobalAttributes(
ipAddress: String,
userAgent: String,
isBot: Boolean,
modelMap: MutableMap<Any?, Any?>,
controller: Any,
replacedPath: String,
Expand All @@ -30,7 +28,6 @@ fun setGlobalAttributes(
val configCacheService = Constant.injector.getInstance(ConfigCacheService::class.java)
val simulcastCacheService = Constant.injector.getInstance(SimulcastCacheService::class.java)
val animeCacheService = Constant.injector.getInstance(AnimeCacheService::class.java)
val botDetectorCache = Constant.injector.getInstance(BotDetectorCache::class.java)

modelMap["su"] = StringUtils
modelMap["links"] = getLinks(controller, replacedPath, simulcastCacheService)
Expand All @@ -42,7 +39,7 @@ fun setGlobalAttributes(
modelMap["baseUrl"] = Constant.baseUrl
modelMap["apiUrl"] = Constant.apiUrl

if (configCacheService.getValueAsBoolean(ConfigPropertyKey.DISABLE_BOT_DETECTION) || !botDetectorCache.isBot(clientIp = ipAddress, userAgent = userAgent)) {
if (!isBot) {
modelMap["additionalHeadTags"] = configCacheService.getValueAsString(ConfigPropertyKey.ADDITIONAL_HEAD_TAGS)
}

Expand Down
65 changes: 37 additions & 28 deletions src/main/kotlin/fr/shikkanime/services/caches/BotDetectorCache.kt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,8 @@ class BotDetectorCache : AbstractCacheService {
}

private val httpRequest = HttpRequest()
private val ipv4Regex = Regex("^(?:[0-9]{1,3}\\.){3}[0-9]{1,3}$")
private val ipv4Regex = Regex("^(?:[0-9]{1,3}\\.){3}[0-9]{1,3}(?:/[0-9]+)*")
private val duration = Duration.ofDays(1)

@Inject
private lateinit var configCacheService: ConfigCacheService
Expand All @@ -44,9 +45,9 @@ class BotDetectorCache : AbstractCacheService {
return true
}

private fun getGoodBotsIP() = MapCache.getOrCompute(
"BotDetectorCache.getGoodBotsIP",
duration = Duration.ofDays(1),
private fun getGoodBotsIPs() = MapCache.getOrCompute(
"BotDetectorCache.getGoodBotsIPs",
duration = duration,
key = DEFAULT_ALL_KEY
) {
runBlocking {
Expand All @@ -57,13 +58,13 @@ class BotDetectorCache : AbstractCacheService {
return@runBlocking emptyList()
}

response.bodyAsText().split("\n").filter { it.isNotBlank() || !ipv4Regex.matches(it) }
response.bodyAsText().split("\n").filter { it.isNotBlank() && ipv4Regex.matches(it) }
}
}

private fun getGoodBotsRegex() = MapCache.getOrCompute(
"BotDetectorCache.getGoodBotsRegex",
duration = Duration.ofDays(1),
duration = duration,
key = DEFAULT_ALL_KEY
) {
runBlocking {
Expand All @@ -81,34 +82,42 @@ class BotDetectorCache : AbstractCacheService {
}
}

fun isBot(clientIp: String? = null, userAgent: String? = null): Boolean {
if (!clientIp.isNullOrBlank()) {
val clientIpBytes = InetAddress.getByName(clientIp).address

getGoodBotsIP().forEach { botIp ->
if (botIp.contains("/")) {
val (ip, cidr) = botIp.split("/")
val mask = createMask(cidr.toInt())
val networkIpBytes = InetAddress.getByName(ip).address

if (isInRange(clientIpBytes, networkIpBytes, mask)) {
return true
}
} else if (clientIp == botIp) {
return true
}
private fun getAbuseIPs() = MapCache.getOrCompute(
"BotDetectorCache.getAbuseIPs",
duration = duration,
key = DEFAULT_ALL_KEY
) {
runBlocking {
val response =
httpRequest.get("https://raw.githubusercontent.com/borestad/blocklist-abuseipdb/main/abuseipdb-s100-30d.ipv4")

if (response.status != HttpStatusCode.OK) {
return@runBlocking emptyList()
}

response.bodyAsText().split("\n").filter { it.isNotBlank() && ipv4Regex.matches(it) }
}
}

private fun isIpInRange(clientIp: String, botIp: String): Boolean {
return if (botIp.contains("/")) {
val (ip, cidr) = botIp.split("/")
val mask = createMask(cidr.toInt())
isInRange(InetAddress.getByName(clientIp).address, InetAddress.getByName(ip).address, mask)
} else clientIp == botIp
}

fun isBot(clientIp: String? = null, userAgent: String? = null): Boolean {
clientIp?.let {
if (getGoodBotsIPs().any { botIp -> isIpInRange(it, botIp) }) return true
if (getAbuseIPs().contains(it)) return true
}

if (!userAgent.isNullOrBlank()) {
userAgent?.let {
val regexes = getGoodBotsRegex().toMutableSet()
configCacheService.getValueAsString(ConfigPropertyKey.BOT_ADDITIONAL_REGEX)?.toRegex()?.let { regexes.add(it) }

regexes.forEach { regex ->
if (userAgent.contains(regex)) {
return true
}
}
if (regexes.any { regex -> it.contains(regex) }) return true
}

return false
Expand Down
Loading