From 9097bc25326b687385124ce2b41952e8720a8690 Mon Sep 17 00:00:00 2001 From: Cris Barreiro Date: Tue, 10 Dec 2024 09:50:41 +0100 Subject: [PATCH] Add blocking algorithm --- .../app/browser/BrowserWebViewClient.kt | 11 +- .../app/browser/WebViewRequestInterceptor.kt | 3 + .../impl/MaliciousSiteModule.kt | 7 + .../impl/data/MaliciousSiteRepository.kt | 29 ++-- .../MaliciousSiteEmbeddedDataProvider.kt | 4 +- .../impl/data/network/MaliciousSiteService.kt | 68 ++++++++++ .../domain/RealMaliciousSiteProtection.kt | 46 ++++++- .../impl/RealMaliciousSiteProtectionTest.kt | 0 .../domain/RealMaliciousSiteProtectionTest.kt | 128 ++++++++++++++++++ 9 files changed, 275 insertions(+), 21 deletions(-) create mode 100644 malicious-site-protection/malicious-site-protection-impl/src/main/kotlin/com/duckduckgo/malicioussiteprotection/impl/data/network/MaliciousSiteService.kt delete mode 100644 malicious-site-protection/malicious-site-protection-impl/src/test/kotlin/com/duckduckgo/malicioussiteprotection/impl/RealMaliciousSiteProtectionTest.kt create mode 100644 malicious-site-protection/malicious-site-protection-impl/src/test/kotlin/com/duckduckgo/malicioussiteprotection/impl/domain/RealMaliciousSiteProtectionTest.kt diff --git a/app/src/main/java/com/duckduckgo/app/browser/BrowserWebViewClient.kt b/app/src/main/java/com/duckduckgo/app/browser/BrowserWebViewClient.kt index e97175e533b1..3bfd68b4171d 100644 --- a/app/src/main/java/com/duckduckgo/app/browser/BrowserWebViewClient.kt +++ b/app/src/main/java/com/duckduckgo/app/browser/BrowserWebViewClient.kt @@ -80,6 +80,7 @@ import com.duckduckgo.history.api.NavigationHistory import com.duckduckgo.privacy.config.api.AmpLinks import com.duckduckgo.subscriptions.api.Subscriptions import com.duckduckgo.user.agent.api.ClientBrandHintProvider +import com.google.android.material.snackbar.Snackbar import java.net.URI import javax.inject.Inject import kotlinx.coroutines.* @@ -128,10 +129,6 @@ class BrowserWebViewClient @Inject constructor( private var shouldOpenDuckPlayerInNewTab: Boolean = true - private val onSiteBlockedAsync: () -> Unit = { - // TODO (cbarreiro): Handle site blocked asynchronously - } - init { appCoroutineScope.launch { duckPlayer.observeShouldOpenInNewTab().collect { @@ -165,6 +162,11 @@ class BrowserWebViewClient @Inject constructor( Timber.v("shouldOverride webViewUrl: ${webView.url} URL: $url") webViewClientListener?.onShouldOverride() + val onSiteBlockedAsync: () -> Unit = { + Snackbar.make(webView, "Site blocked", Snackbar.LENGTH_SHORT).show() + // TODO (cbarreiro): Handle site blocked asynchronously + } + if (runBlocking { maliciousSiteProtectionWebViewIntegration.shouldOverrideUrlLoading( url, @@ -174,6 +176,7 @@ class BrowserWebViewClient @Inject constructor( ) } ) { + Snackbar.make(webView, "Site blocked", Snackbar.LENGTH_SHORT).show() // TODO (cbarreiro): Handle site blocked synchronously return true } diff --git a/app/src/main/java/com/duckduckgo/app/browser/WebViewRequestInterceptor.kt b/app/src/main/java/com/duckduckgo/app/browser/WebViewRequestInterceptor.kt index 0adaad641815..2b2257ea6636 100644 --- a/app/src/main/java/com/duckduckgo/app/browser/WebViewRequestInterceptor.kt +++ b/app/src/main/java/com/duckduckgo/app/browser/WebViewRequestInterceptor.kt @@ -39,6 +39,7 @@ import com.duckduckgo.httpsupgrade.api.HttpsUpgrader import com.duckduckgo.privacy.config.api.Gpc import com.duckduckgo.request.filterer.api.RequestFilterer import com.duckduckgo.user.agent.api.UserAgentProvider +import com.google.android.material.snackbar.Snackbar import kotlinx.coroutines.withContext import timber.log.Timber @@ -100,11 +101,13 @@ class WebViewRequestInterceptor( val url: Uri? = request.url val onSiteBlockedAsync: () -> Unit = { + Snackbar.make(webView, "Site blocked", Snackbar.LENGTH_SHORT).show() // TODO (cbarreiro): Handle site blocked asynchronously } maliciousSiteProtectionWebViewIntegration.shouldIntercept(request, documentUri, onSiteBlockedAsync)?.let { // TODO (cbarreiro): Handle site blocked synchronously + Snackbar.make(webView, "Site blocked", Snackbar.LENGTH_SHORT).show() return it } diff --git a/malicious-site-protection/malicious-site-protection-impl/src/main/kotlin/com/duckduckgo/malicioussiteprotection/impl/MaliciousSiteModule.kt b/malicious-site-protection/malicious-site-protection-impl/src/main/kotlin/com/duckduckgo/malicioussiteprotection/impl/MaliciousSiteModule.kt index 83c46329dfc0..8682a39687c4 100644 --- a/malicious-site-protection/malicious-site-protection-impl/src/main/kotlin/com/duckduckgo/malicioussiteprotection/impl/MaliciousSiteModule.kt +++ b/malicious-site-protection/malicious-site-protection-impl/src/main/kotlin/com/duckduckgo/malicioussiteprotection/impl/MaliciousSiteModule.kt @@ -26,6 +26,7 @@ import com.squareup.anvil.annotations.ContributesTo import dagger.Module import dagger.Provides import dagger.SingleInstanceIn +import java.security.MessageDigest @Module @ContributesTo(AppScope::class) @@ -45,4 +46,10 @@ class MaliciousSiteModule { fun provideMaliciousSiteDao(database: MaliciousSitesDatabase): MaliciousSiteDao { return database.maliciousSiteDao() } + + @Provides + @SingleInstanceIn(AppScope::class) + fun provideMessageDigest(): MessageDigest { + return MessageDigest.getInstance("SHA-256") + } } diff --git a/malicious-site-protection/malicious-site-protection-impl/src/main/kotlin/com/duckduckgo/malicioussiteprotection/impl/data/MaliciousSiteRepository.kt b/malicious-site-protection/malicious-site-protection-impl/src/main/kotlin/com/duckduckgo/malicioussiteprotection/impl/data/MaliciousSiteRepository.kt index 4c5318667619..f660278fcb61 100644 --- a/malicious-site-protection/malicious-site-protection-impl/src/main/kotlin/com/duckduckgo/malicioussiteprotection/impl/data/MaliciousSiteRepository.kt +++ b/malicious-site-protection/malicious-site-protection-impl/src/main/kotlin/com/duckduckgo/malicioussiteprotection/impl/data/MaliciousSiteRepository.kt @@ -23,15 +23,18 @@ import com.duckduckgo.di.scopes.AppScope import com.duckduckgo.malicioussiteprotection.impl.MaliciousSiteProtectionFeature import com.duckduckgo.malicioussiteprotection.impl.data.db.MaliciousSiteDao import com.duckduckgo.malicioussiteprotection.impl.data.embedded.MaliciousSiteProtectionEmbeddedDataProvider +import com.duckduckgo.malicioussiteprotection.impl.data.network.MaliciousSiteService import com.squareup.anvil.annotations.ContributesBinding import dagger.SingleInstanceIn import javax.inject.Inject import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.launch +import timber.log.Timber interface MaliciousSiteRepository { suspend fun containsHashPrefix(hashPrefix: String): Boolean suspend fun getFilter(hash: String): Filter? + suspend fun matches(hashPrefix: String): List } @ContributesBinding(AppScope::class) @@ -41,6 +44,7 @@ class RealMaliciousSiteRepository @Inject constructor( private val maliciousSiteDao: MaliciousSiteDao, @IsMainProcess private val isMainProcess: Boolean, maliciousSiteProtectionFeature: MaliciousSiteProtectionFeature, + private val maliciousSiteService: MaliciousSiteService, @AppCoroutineScope private val appCoroutineScope: CoroutineScope, dispatcherProvider: DispatcherProvider, ) : MaliciousSiteRepository { @@ -87,6 +91,17 @@ class RealMaliciousSiteRepository @Inject constructor( Filter(it.hash, it.regex) } } + + override suspend fun matches(hashPrefix: String): List { + return try { + maliciousSiteService.getMatches(hashPrefix).matches.also { + Timber.d("\uD83D\uDFE2 Cris: Fetched $it matches for hash prefix $hashPrefix") + } + } catch (e: Exception) { + Timber.d("\uD83D\uDD34 Cris: Failed to fetch matches for hash prefix $hashPrefix") + listOf() + } + } } data class Match( @@ -96,20 +111,6 @@ data class Match( val hash: String, ) -data class HashPrefixResponse( - val insert: Set, - val delete: Set, - val revision: Int, - val replace: Boolean, -) - -data class FilterSetResponse( - val insert: Set, - val delete: Set, - val revision: Int, - val replace: Boolean, -) - data class Filter( val hash: String, val regex: String, diff --git a/malicious-site-protection/malicious-site-protection-impl/src/main/kotlin/com/duckduckgo/malicioussiteprotection/impl/data/embedded/MaliciousSiteEmbeddedDataProvider.kt b/malicious-site-protection/malicious-site-protection-impl/src/main/kotlin/com/duckduckgo/malicioussiteprotection/impl/data/embedded/MaliciousSiteEmbeddedDataProvider.kt index c4d6cd4a2d57..18c4bd7e38d0 100644 --- a/malicious-site-protection/malicious-site-protection-impl/src/main/kotlin/com/duckduckgo/malicioussiteprotection/impl/data/embedded/MaliciousSiteEmbeddedDataProvider.kt +++ b/malicious-site-protection/malicious-site-protection-impl/src/main/kotlin/com/duckduckgo/malicioussiteprotection/impl/data/embedded/MaliciousSiteEmbeddedDataProvider.kt @@ -20,8 +20,8 @@ import android.content.Context import androidx.annotation.RawRes import com.duckduckgo.di.scopes.AppScope import com.duckduckgo.malicioussiteprotection.impl.R -import com.duckduckgo.malicioussiteprotection.impl.data.FilterSetResponse -import com.duckduckgo.malicioussiteprotection.impl.data.HashPrefixResponse +import com.duckduckgo.malicioussiteprotection.impl.data.network.FilterSetResponse +import com.duckduckgo.malicioussiteprotection.impl.data.network.HashPrefixResponse import com.squareup.anvil.annotations.ContributesBinding import com.squareup.moshi.Moshi import javax.inject.Inject diff --git a/malicious-site-protection/malicious-site-protection-impl/src/main/kotlin/com/duckduckgo/malicioussiteprotection/impl/data/network/MaliciousSiteService.kt b/malicious-site-protection/malicious-site-protection-impl/src/main/kotlin/com/duckduckgo/malicioussiteprotection/impl/data/network/MaliciousSiteService.kt new file mode 100644 index 000000000000..6f3fd6787c5d --- /dev/null +++ b/malicious-site-protection/malicious-site-protection-impl/src/main/kotlin/com/duckduckgo/malicioussiteprotection/impl/data/network/MaliciousSiteService.kt @@ -0,0 +1,68 @@ +/* + * Copyright (c) 2024 DuckDuckGo + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.duckduckgo.malicioussiteprotection.impl.data.network + +import com.duckduckgo.anvil.annotations.ContributesServiceApi +import com.duckduckgo.common.utils.AppUrl.Url.API +import com.duckduckgo.di.scopes.AppScope +import com.duckduckgo.malicioussiteprotection.impl.data.Filter +import com.duckduckgo.malicioussiteprotection.impl.data.Match +import retrofit2.http.GET +import retrofit2.http.Query + +private const val BASE_URL = "$API/api/protection" +private const val HASH_PREFIX_PATH = "/hashPrefix" +private const val FILTER_SET_PATH = "/filterSet" +private const val CATEGORY = "category" +private const val PHISHING = "phishing" +private const val MALWARE = "malware" + +@ContributesServiceApi(AppScope::class) +interface MaliciousSiteService { + @GET("$BASE_URL$HASH_PREFIX_PATH?$CATEGORY=$PHISHING") + suspend fun getPhishingHashPrefixes(@Query("revision") revision: Int): HashPrefixResponse + + @GET("$BASE_URL$HASH_PREFIX_PATH?$CATEGORY=$MALWARE") + suspend fun getMalwareHashPrefixes(@Query("revision") revision: Int): HashPrefixResponse + + @GET("$BASE_URL$FILTER_SET_PATH?$CATEGORY=$PHISHING") + suspend fun getPhishingFilterSet(@Query("revision") revision: Int): FilterSetResponse + + @GET("$BASE_URL$FILTER_SET_PATH?$CATEGORY=$MALWARE") + suspend fun getMalwareFilterSet(@Query("revision") revision: Int): FilterSetResponse + + @GET("$BASE_URL/matches") + suspend fun getMatches(@Query("hashPrefix") hashPrefix: String): MatchesResponse +} + +data class MatchesResponse( + val matches: List, +) + +data class HashPrefixResponse( + val insert: Set, + val delete: Set, + val revision: Int, + val replace: Boolean, +) + +data class FilterSetResponse( + val insert: Set, + val delete: Set, + val revision: Int, + val replace: Boolean, +) diff --git a/malicious-site-protection/malicious-site-protection-impl/src/main/kotlin/com/duckduckgo/malicioussiteprotection/impl/domain/RealMaliciousSiteProtection.kt b/malicious-site-protection/malicious-site-protection-impl/src/main/kotlin/com/duckduckgo/malicioussiteprotection/impl/domain/RealMaliciousSiteProtection.kt index 7ee4496692bc..9e36c77e64d2 100644 --- a/malicious-site-protection/malicious-site-protection-impl/src/main/kotlin/com/duckduckgo/malicioussiteprotection/impl/domain/RealMaliciousSiteProtection.kt +++ b/malicious-site-protection/malicious-site-protection-impl/src/main/kotlin/com/duckduckgo/malicioussiteprotection/impl/domain/RealMaliciousSiteProtection.kt @@ -23,9 +23,12 @@ import com.duckduckgo.common.utils.DispatcherProvider import com.duckduckgo.di.scopes.AppScope import com.duckduckgo.malicioussiteprotection.api.MaliciousSiteProtection import com.duckduckgo.malicioussiteprotection.impl.MaliciousSiteProtectionFeature +import com.duckduckgo.malicioussiteprotection.impl.data.MaliciousSiteRepository import com.duckduckgo.privacy.config.api.PrivacyConfigCallbackPlugin import com.squareup.anvil.annotations.ContributesBinding import com.squareup.anvil.annotations.ContributesMultibinding +import java.security.MessageDigest +import java.util.regex.Pattern import javax.inject.Inject import kotlinx.coroutines.CoroutineScope import kotlinx.coroutines.launch @@ -39,6 +42,8 @@ class RealMaliciousSiteProtection @Inject constructor( private val maliciousSiteProtectionFeature: MaliciousSiteProtectionFeature, @IsMainProcess private val isMainProcess: Boolean, @AppCoroutineScope private val appCoroutineScope: CoroutineScope, + private val maliciousSiteRepository: MaliciousSiteRepository, + private val messageDigest: MessageDigest, ) : MaliciousSiteProtection, PrivacyConfigCallbackPlugin { private var _isFeatureEnabled = false @@ -72,7 +77,46 @@ class RealMaliciousSiteProtection @Inject constructor( override suspend fun isMalicious(url: Uri, onSiteBlockedAsync: () -> Unit): Boolean { Timber.tag("MaliciousSiteProtection").d("isMalicious $url") - // TODO (cbarreiro): Implement the logic to check if the URL is malicious + + val hostname = url.host ?: return false + val hash = messageDigest + .digest(hostname.toByteArray()) + .joinToString("") { "%02x".format(it) } + val hashPrefix = hash.substring(0, 8) + + if (!maliciousSiteRepository.containsHashPrefix(hashPrefix)) { + Timber.d("\uD83D\uDFE2 Cris: should not block (no hash) $hashPrefix, $url") + return false + } + maliciousSiteRepository.getFilter(hash)?.let { + if (Pattern.matches(it.regex, url.toString())) { + Timber.d("\uD83D\uDFE2 Cris: shouldBlock $url") + return true + } + } + appCoroutineScope.launch(dispatchers.io()) { + matches(hashPrefix, url, hostname, hash).let { matches -> + if (matches) { + onSiteBlockedAsync() + } + } + } return false } + + private suspend fun matches( + hashPrefix: String, + url: Uri, + hostname: String, + hash: String, + ): Boolean { + val matches = maliciousSiteRepository.matches(hashPrefix) + return matches.any { match -> + Pattern.matches(match.regex, url.toString()) && + (hostname == match.hostname) && + (hash == match.hash) + }.also { matched -> + Timber.d("\uD83D\uDFE2 Cris: should block $matched") + } + } } diff --git a/malicious-site-protection/malicious-site-protection-impl/src/test/kotlin/com/duckduckgo/malicioussiteprotection/impl/RealMaliciousSiteProtectionTest.kt b/malicious-site-protection/malicious-site-protection-impl/src/test/kotlin/com/duckduckgo/malicioussiteprotection/impl/RealMaliciousSiteProtectionTest.kt deleted file mode 100644 index e69de29bb2d1..000000000000 diff --git a/malicious-site-protection/malicious-site-protection-impl/src/test/kotlin/com/duckduckgo/malicioussiteprotection/impl/domain/RealMaliciousSiteProtectionTest.kt b/malicious-site-protection/malicious-site-protection-impl/src/test/kotlin/com/duckduckgo/malicioussiteprotection/impl/domain/RealMaliciousSiteProtectionTest.kt new file mode 100644 index 000000000000..82c80b129541 --- /dev/null +++ b/malicious-site-protection/malicious-site-protection-impl/src/test/kotlin/com/duckduckgo/malicioussiteprotection/impl/domain/RealMaliciousSiteProtectionTest.kt @@ -0,0 +1,128 @@ +/* + * Copyright (c) 2024 DuckDuckGo + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package com.duckduckgo.malicioussiteprotection.impl.domain + +import android.net.Uri +import androidx.test.ext.junit.runners.AndroidJUnit4 +import com.duckduckgo.common.test.CoroutineTestRule +import com.duckduckgo.feature.toggles.api.FakeFeatureToggleFactory +import com.duckduckgo.malicioussiteprotection.impl.MaliciousSiteProtectionFeature +import com.duckduckgo.malicioussiteprotection.impl.data.Filter +import com.duckduckgo.malicioussiteprotection.impl.data.MaliciousSiteRepository +import com.duckduckgo.malicioussiteprotection.impl.data.Match +import java.security.MessageDigest +import kotlinx.coroutines.test.runTest +import org.junit.Assert.assertFalse +import org.junit.Assert.assertTrue +import org.junit.Before +import org.junit.Rule +import org.junit.Test +import org.junit.runner.RunWith +import org.mockito.Mockito.mock +import org.mockito.kotlin.whenever + +@RunWith(AndroidJUnit4::class) +class RealMaliciousSiteProtectionTest { + + @get:Rule + var coroutinesTestRule = CoroutineTestRule() + + private lateinit var realMaliciousSiteProtection: RealMaliciousSiteProtection + private val maliciousSiteProtectionFeature = FakeFeatureToggleFactory.create(MaliciousSiteProtectionFeature::class.java) + private val isMainProcess: Boolean = true + private val maliciousSiteRepository: MaliciousSiteRepository = mock() + private val messageDigest: MessageDigest = MessageDigest.getInstance("SHA-256") + + @Before + fun setup() { + realMaliciousSiteProtection = RealMaliciousSiteProtection( + coroutinesTestRule.testDispatcherProvider, + maliciousSiteProtectionFeature, + isMainProcess, + coroutinesTestRule.testScope, + maliciousSiteRepository, + messageDigest, + ) + } + + @Test + fun isMalicious_returnsFalse_whenUrlIsNotMalicious() = runTest { + val url = Uri.parse("https://example.com") + val hostname = url.host!! + val hash = messageDigest.digest(hostname.toByteArray()).joinToString("") { "%02x".format(it) } + val hashPrefix = hash.substring(0, 8) + + whenever(maliciousSiteRepository.containsHashPrefix(hashPrefix)).thenReturn(false) + + val result = realMaliciousSiteProtection.isMalicious(url) {} + + assertFalse(result) + } + + @Test + fun isMalicious_returnsTrue_whenUrlIsMalicious() = runTest { + val url = Uri.parse("https://malicious.com") + val hostname = url.host!! + val hash = messageDigest.digest(hostname.toByteArray()).joinToString("") { "%02x".format(it) } + val hashPrefix = hash.substring(0, 8) + val filter = Filter(hash, ".*malicious.*") + + whenever(maliciousSiteRepository.containsHashPrefix(hashPrefix)).thenReturn(true) + whenever(maliciousSiteRepository.getFilter(hash)).thenReturn(filter) + + val result = realMaliciousSiteProtection.isMalicious(url) {} + + assertTrue(result) + } + + @Test + fun isMalicious_returnsFalse_whenUrlDoesNotMatchFilter() = runTest { + val url = Uri.parse("https://safe.com") + val hostname = url.host!! + val hash = messageDigest.digest(hostname.toByteArray()).joinToString("") { "%02x".format(it) } + val hashPrefix = hash.substring(0, 8) + val filter = Filter(hash, ".*unsafe.*") + + whenever(maliciousSiteRepository.containsHashPrefix(hashPrefix)).thenReturn(true) + whenever(maliciousSiteRepository.getFilter(hash)).thenReturn(filter) + + val result = realMaliciousSiteProtection.isMalicious(url) {} + + assertFalse(result) + } + + @Test + fun isMalicious_invokesOnSiteBlockedAsync_whenUrlIsMaliciousAndNeedsToGoToNetwork() = runTest { + val url = Uri.parse("https://malicious.com") + val hostname = url.host!! + val hash = messageDigest.digest(hostname.toByteArray()).joinToString("") { "%02x".format(it) } + val hashPrefix = hash.substring(0, 8) + val filter = Filter(hash, ".*whatever.*") + var onSiteBlockedAsyncCalled = false + + whenever(maliciousSiteRepository.containsHashPrefix(hashPrefix)).thenReturn(true) + whenever(maliciousSiteRepository.getFilter(hash)).thenReturn(filter) + whenever(maliciousSiteRepository.matches(hashPrefix)) + .thenReturn(listOf(Match(hostname, url.toString(), ".*malicious.*", hash))) + + realMaliciousSiteProtection.isMalicious(url) { + onSiteBlockedAsyncCalled = true + } + + assertTrue(onSiteBlockedAsyncCalled) + } +}