Load and store embedded dataset

malicious-site-protection/malicious-site-protection-impl/build.gradle

@@ -30,6 +30,8 @@ dependencies {
     implementation project(path: ':anvil-annotations')
     implementation project(path: ':di')
     ksp AndroidX.room.compiler
+    implementation AndroidX.room.runtime
+    implementation AndroidX.room.ktx
 
     implementation KotlinX.coroutines.android
     implementation AndroidX.core.ktx
@@ -43,6 +45,7 @@ dependencies {
     implementation Google.android.material
 
     testImplementation AndroidX.test.ext.junit
+    testImplementation 'org.jetbrains.kotlinx:kotlinx-coroutines-test:1.5.2'
     testImplementation Testing.junit4
     testImplementation "org.mockito.kotlin:mockito-kotlin:_"
     testImplementation project(path: ':common-test')

malicious-site-protection/malicious-site-protection-impl/src/main/kotlin/com/duckduckgo/malicioussiteprotection/impl/MaliciousSiteDao.kt

@@ -0,0 +1,99 @@
+/*
+ * Copyright (c) 2024 DuckDuckGo
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.duckduckgo.malicioussiteprotection.impl
+
+import androidx.room.Dao
+import androidx.room.Insert
+import androidx.room.OnConflictStrategy
+import androidx.room.Query
+import androidx.room.Transaction
+import kotlin.math.max
+
+@Dao
+interface MaliciousSiteDao {
+    @Insert(onConflict = OnConflictStrategy.REPLACE)
+    suspend fun insertRevision(revision: RevisionEntity)
+
+    @Insert(onConflict = OnConflictStrategy.REPLACE)
+    suspend fun insertHashPrefixes(items: List<HashPrefixEntity>)
+
+    @Insert(onConflict = OnConflictStrategy.REPLACE)
+    suspend fun insertFilters(items: Set<FilterEntity>)
+
+    @Query("SELECT * FROM revisions LIMIT 1")
+    suspend fun getLatestRevision(): RevisionEntity?
+
+    @Transaction
+    @Query("SELECT * FROM revisions")
+    suspend fun getMaliciousSiteData(): DataWithFilters? {
+        val revision = getLatestRevision() ?: return null
+        val phishingHashPrefixes = getPhishingHashPrefixes()
+        val phishingFilters = getPhishingFilters()
+        val malwareHashPrefixes = getMalwareHashPrefixes()
+        val malwareFilters = getMalwareFilters()
+        return DataWithFilters(revision, phishingFilters, malwareFilters, phishingHashPrefixes, malwareHashPrefixes)
+    }
+
+    @Query("SELECT * FROM hash_prefixes WHERE type = 'phishing'")
+    suspend fun getPhishingHashPrefixes(): List<HashPrefixEntity>
+
+    @Query("SELECT * FROM hash_prefixes WHERE type = 'malware'")
+    suspend fun getMalwareHashPrefixes(): List<HashPrefixEntity>
+
+    @Query("SELECT * FROM filters WHERE type = 'phishing'")
+    suspend fun getPhishingFilters(): List<FilterEntity>
+
+    @Query("SELECT * FROM filters WHERE type = 'malware'")
+    suspend fun getMalwareFilters(): List<FilterEntity>
+
+    @Query("DELETE FROM revisions")
+    suspend fun deleteRevisions()
+
+    @Query("SELECT * FROM hash_prefixes WHERE hashPrefix = :hashPrefix")
+    suspend fun getHashPrefix(hashPrefix: String): HashPrefixEntity?
+
+    @Query("SELECT * FROM filters WHERE hash = :hash")
+    suspend fun getFilter(hash: String): FilterEntity?
+
+    @Transaction
+    suspend fun insertData(
+        phishingFilterSetRevision: Int?,
+        malwareFilterSetRevision: Int?,
+        phishingHashPrefixesRevision: Int?,
+        malwareHashPrefixesRevision: Int?,
+        phishingHashPrefixes: Set<String>,
+        phishingFilterSet: Set<FilterEntity>,
+        malwareHashPrefixes: Set<String>,
+        malwareFilterSet: Set<FilterEntity>,
+    ) {
+        val lastRevision = getLatestRevision()
+        deleteRevisions()
+
+        insertRevision(
+            RevisionEntity(
+                phishingHashPrefixesRevision = max(lastRevision?.phishingHashPrefixesRevision ?: 0, phishingHashPrefixesRevision ?: 0),
+                malwareHashPrefixesRevision = max(lastRevision?.malwareHashPrefixesRevision ?: 0, malwareHashPrefixesRevision ?: 0),
+                phishingFiltersRevision = max(lastRevision?.phishingFiltersRevision ?: 0, phishingFilterSetRevision ?: 0),
+                malwareFiltersRevision = max(lastRevision?.malwareFiltersRevision ?: 0, malwareFilterSetRevision ?: 0),
+            ),
+        )
+        insertHashPrefixes(phishingHashPrefixes.map { HashPrefixEntity(hashPrefix = it, type = "phishing") })
+        insertFilters(phishingFilterSet)
+        insertHashPrefixes(malwareHashPrefixes.map { HashPrefixEntity(hashPrefix = it, type = "malware") })
+        insertFilters(malwareFilterSet)
+    }
+}

malicious-site-protection/malicious-site-protection-impl/src/main/kotlin/com/duckduckgo/malicioussiteprotection/impl/MaliciousSiteEmbeddedDataProvider.kt

@@ -0,0 +1,112 @@
+/*
+ * Copyright (c) 2024 DuckDuckGo
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.duckduckgo.malicioussiteprotection.impl
+
+import android.content.Context
+import androidx.annotation.RawRes
+import com.duckduckgo.di.scopes.AppScope
+import com.squareup.anvil.annotations.ContributesBinding
+import com.squareup.moshi.Moshi
+import javax.inject.Inject
+import timber.log.Timber
+
+interface MaliciousSiteProtectionEmbeddedDataProvider {
+    fun loadEmbeddedPhishingFilterSet(): FilterSetResponse?
+    fun loadEmbeddedPhishingHashPrefixes(): HashPrefixResponse?
+    fun loadEmbeddedMalwareFilterSet(): FilterSetResponse?
+    fun loadEmbeddedMalwareHashPrefixes(): HashPrefixResponse?
+}
+
+@ContributesBinding(AppScope::class)
+class RealMaliciousSiteProtectionEmbeddedDataProvider @Inject constructor(
+    private val context: Context,
+    private val moshi: Moshi,
+) : MaliciousSiteProtectionEmbeddedDataProvider {
+
+    private fun loadEmbeddedData(@RawRes file: Int): ByteArray {
+        return context.resources.openRawResource(file).use { it.readBytes() }
+    }
+
+    override fun loadEmbeddedPhishingFilterSet(): FilterSetResponse? {
+        return try {
+            val filterSetData = loadEmbeddedData(R.raw.phishing_filter_set)
+            val adapter = moshi.adapter(FilterSetResponse::class.java)
+            adapter.fromJson(String(filterSetData))
+        } catch (e: Exception) {
+            null
+        }
+    }
+
+    override fun loadEmbeddedPhishingHashPrefixes(): HashPrefixResponse? {
+        return try {
+            val hashPrefixData = loadEmbeddedData(R.raw.phishing_hash_prefix)
+            val adapter = moshi.adapter(HashPrefixResponse::class.java)
+            adapter.fromJson(String(hashPrefixData))
+        } catch (e: Exception) {
+            Timber.d("\uD83D\uDD34 Failed to fetch embedded phishing hash prefixes")
+            null
+        }
+    }
+
+    override fun loadEmbeddedMalwareFilterSet(): FilterSetResponse? {
+        return try {
+            val filterSetData = loadEmbeddedData(R.raw.malware_filter_set)
+            val adapter = moshi.adapter(FilterSetResponse::class.java)
+            adapter.fromJson(String(filterSetData))
+        } catch (e: Exception) {
+            Timber.d("\uD83D\uDD34 Failed to fetch embedded malware filter set")
+            null
+        }
+    }
+
+    override fun loadEmbeddedMalwareHashPrefixes(): HashPrefixResponse? {
+        return try {
+            val hashPrefixData = loadEmbeddedData(R.raw.malware_hash_prefix)
+            val adapter = moshi.adapter(HashPrefixResponse::class.java)
+            adapter.fromJson(String(hashPrefixData))
+        } catch (e: Exception) {
+            Timber.d("\uD83D\uDD34 Cris: Failed to fetch embedded malware hash prefixes")
+            null
+        }
+    }
+}
+
+data class Match(
+    val hostname: String,
+    val url: String,
+    val regex: String,
+    val hash: String,
+)
+
+data class HashPrefixResponse(
+    val insert: Set<String>,
+    val delete: Set<String>,
+    val revision: Int,
+    val replace: Boolean,
+)
+
+data class FilterSetResponse(
+    val insert: Set<Filter>,
+    val delete: Set<Filter>,
+    val revision: Int,
+    val replace: Boolean,
+)
+
+data class Filter(
+    val hash: String,
+    val regex: String,
+)

malicious-site-protection/malicious-site-protection-impl/src/main/kotlin/com/duckduckgo/malicioussiteprotection/impl/MaliciousSiteModule.kt

@@ -0,0 +1,46 @@
+/*
+ * Copyright (c) 2024 DuckDuckGo
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.duckduckgo.malicioussiteprotection.impl
+
+import android.content.Context
+import androidx.room.Room
+import com.duckduckgo.di.scopes.AppScope
+import com.duckduckgo.malicioussiteprotection.impl.MaliciousSitesDatabase.Companion.ALL_MIGRATIONS
+import com.squareup.anvil.annotations.ContributesTo
+import dagger.Module
+import dagger.Provides
+import dagger.SingleInstanceIn
+
+@Module
+@ContributesTo(AppScope::class)
+class MaliciousSiteModule {
+
+    @Provides
+    @SingleInstanceIn(AppScope::class)
+    fun provideMaliciousSiteProtectionDatabase(context: Context): MaliciousSitesDatabase {
+        return Room.databaseBuilder(context, MaliciousSitesDatabase::class.java, "malicious_sites.db")
+            .addMigrations(*ALL_MIGRATIONS)
+            .fallbackToDestructiveMigration()
+            .build()
+    }
+
+    @Provides
+    @SingleInstanceIn(AppScope::class)
+    fun provideMaliciousSiteDao(database: MaliciousSitesDatabase): MaliciousSiteDao {
+        return database.maliciousSiteDao()
+    }
+}

malicious-site-protection/malicious-site-protection-impl/src/main/kotlin/com/duckduckgo/malicioussiteprotection/impl/MaliciousSiteRepository.kt

@@ -0,0 +1,87 @@
+/*
+ * Copyright (c) 2024 DuckDuckGo
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.duckduckgo.malicioussiteprotection.impl
+
+import com.duckduckgo.app.di.AppCoroutineScope
+import com.duckduckgo.app.di.IsMainProcess
+import com.duckduckgo.common.utils.DispatcherProvider
+import com.duckduckgo.di.scopes.AppScope
+import com.squareup.anvil.annotations.ContributesBinding
+import dagger.SingleInstanceIn
+import javax.inject.Inject
+import kotlinx.coroutines.CoroutineScope
+import kotlinx.coroutines.launch
+
+interface MaliciousSiteRepository {
+    suspend fun containsHashPrefix(hashPrefix: String): Boolean
+    suspend fun getFilter(hash: String): Filter?
+}
+
+@ContributesBinding(AppScope::class)
+@SingleInstanceIn(AppScope::class)
+class RealMaliciousSiteRepository @Inject constructor(
+    private val dataProvider: MaliciousSiteProtectionEmbeddedDataProvider,
+    private val maliciousSiteDao: MaliciousSiteDao,
+    @IsMainProcess private val isMainProcess: Boolean,
+    maliciousSiteProtectionFeature: MaliciousSiteProtectionFeature,
+    @AppCoroutineScope private val appCoroutineScope: CoroutineScope,
+    dispatcherProvider: DispatcherProvider,
+) : MaliciousSiteRepository {
+
+    init {
+        if (isMainProcess) {
+            appCoroutineScope.launch(dispatcherProvider.io()) {
+                if (maliciousSiteProtectionFeature.self().isEnabled()) {
+                    loadEmbeddedData()
+                }
+            }
+        }
+    }
+
+    private suspend fun loadEmbeddedData() {
+        val embeddedPhishingHashPrefixes = dataProvider.loadEmbeddedPhishingHashPrefixes()
+
+        val embeddedPhishingFilterSet = dataProvider.loadEmbeddedPhishingFilterSet()
+
+        val embeddedMalwareHashPrefixes = dataProvider.loadEmbeddedMalwareHashPrefixes()
+
+        val embeddedMalwareFilterSet = dataProvider.loadEmbeddedMalwareFilterSet()
+
+        // TODO (cbarreiro): Once we have the download scheduler, we should check the revision and update the data accordingly
+
+        maliciousSiteDao.insertData(
+            phishingFilterSetRevision = embeddedPhishingFilterSet?.revision,
+            malwareFilterSetRevision = embeddedMalwareFilterSet?.revision,
+            phishingHashPrefixesRevision = embeddedPhishingHashPrefixes?.revision,
+            malwareHashPrefixesRevision = embeddedMalwareHashPrefixes?.revision,
+            phishingHashPrefixes = embeddedPhishingHashPrefixes?.insert?.toSet() ?: setOf(),
+            phishingFilterSet = embeddedPhishingFilterSet?.insert?.map { FilterEntity(it.hash, it.regex, type = "phishing") }?.toSet() ?: setOf(),
+            malwareHashPrefixes = embeddedMalwareHashPrefixes?.insert?.toSet() ?: setOf(),
+            malwareFilterSet = embeddedMalwareFilterSet?.insert?.map { FilterEntity(it.hash, it.regex, type = "malware") }?.toSet() ?: setOf(),
+        )
+    }
+
+    override suspend fun containsHashPrefix(hashPrefix: String): Boolean {
+        return maliciousSiteDao.getHashPrefix(hashPrefix) != null
+    }
+
+    override suspend fun getFilter(hash: String): Filter? {
+        return maliciousSiteDao.getPhishingFilters().firstOrNull { it.hash == hash }?.let {
+            Filter(it.hash, it.regex)
+        }
+    }
+}