Incremental clone detecting

kotoed-server/db.properties

@@ -1,4 +1,4 @@
-db.url=jdbc:postgresql://localhost/kotoed
+db.url=jdbc:postgresql://localhost:5432/kotoedProd
 db.user=kotoed
 db.password=kotoed
 testdb.url=jdbc:postgresql://localhost/kotoed-test

kotoed-server/src/main/kotlin/org/jetbrains/research/kotoed/api/HashComputingVerticle.kt

@@ -0,0 +1,185 @@
+package org.jetbrains.research.kotoed.api
+
+import com.intellij.psi.PsiElement
+import kotlinx.coroutines.withContext
+import org.jetbrains.kotlin.psi.KtClassBody
+import org.jetbrains.kotlin.psi.KtNamedFunction
+import org.jetbrains.kotlin.psi.psiUtil.collectDescendantsOfType
+import org.jetbrains.kotlin.psi.psiUtil.endOffset
+import org.jetbrains.kotlin.psi.psiUtil.startOffsetSkippingComments
+import org.jetbrains.research.kotoed.code.diff.HunkJsonable
+import org.jetbrains.research.kotoed.code.diff.RangeJsonable
+import org.jetbrains.research.kotoed.data.api.Code
+import org.jetbrains.research.kotoed.data.api.VerificationData
+import org.jetbrains.research.kotoed.data.api.VerificationStatus
+import org.jetbrains.research.kotoed.data.vcs.CloneStatus
+import org.jetbrains.research.kotoed.database.tables.records.*
+import org.jetbrains.research.kotoed.db.processors.getFullName
+import org.jetbrains.research.kotoed.eventbus.Address
+import org.jetbrains.research.kotoed.util.*
+import org.jetbrains.research.kotoed.util.code.getPsi
+import org.jetbrains.research.kotoed.util.code.temporaryKotlinEnv
+import java.util.function.Consumer
+
+@AutoDeployable
+class HashComputingVerticle : AbstractKotoedVerticle(), Loggable {
+    private val ee by lazy { betterSingleThreadContext("hashComputingVerticle.executor") }
+    private val treeHashVisitor = TreeHashVisitor()
+
+    @JsonableEventBusConsumerFor(Address.Code.Hashes)
+    suspend fun computeHashesFromSub(res: SubmissionRecord): VerificationData {
+        log.info("Start computing hashing for submission=[${res.id}]")
+        try {
+            val diffResponse: Code.Submission.DiffResponse = sendJsonableAsync(
+                Address.Api.Submission.Code.DiffWithPrevious,
+                Code.Submission.DiffRequest(submissionId = res.id)
+            )
+
+            val files: Code.ListResponse = sendJsonableAsync(
+                Address.Api.Submission.Code.List,
+                Code.Submission.ListRequest(res.id)
+            )
+
+            temporaryKotlinEnv {
+                withContext(ee) {
+                    val ktFiles =
+                        files.root?.toFileSeq()
+                            .orEmpty()
+                            .filter { it.endsWith(".kt") }
+                            .toList()                           //FIXME
+                            .map { filename ->
+                                val resp: Code.Submission.ReadResponse = sendJsonableAsync(
+                                    Address.Api.Submission.Code.Read,
+                                    Code.Submission.ReadRequest(
+                                        submissionId = res.id, path = filename
+                                    )
+                                )
+                                getPsi(resp.contents, filename)
+                            }
+                    val functionsList = ktFiles.asSequence()
+                        .flatMap { file ->
+                            file.collectDescendantsOfType<KtNamedFunction>().asSequence()
+                        }
+                        .filter { method ->
+                            method.annotationEntries.all { anno -> "@Test" != anno.text } &&
+                                    !method.containingFile.name.startsWith("test")
+                        }
+                        .toList()
+
+                    val changesInFiles = diffResponse.diff.associate {
+                        if (it.toFile != it.fromFile) {
+                            log.warn("File [${it.fromFile}] is renamed to [${it.toFile}]")
+                        }
+                        it.toFile to it.changes
+                    }
+                    val project = dbFindAsync(ProjectRecord().apply { id = res.projectId }).first()
+                    for (function in functionsList) {
+                        val needProcess = function.isTopLevel || function.parent is KtClassBody
+                        if (!needProcess) {
+                            continue
+                        }
+                        processFunction(function, res, changesInFiles, project)
+                    }
+
+                }
+            }
+        } catch (ex: Throwable) {
+            log.error(ex)
+            return VerificationData(VerificationStatus.Invalid, emptyList())
+        }
+        return VerificationData(VerificationStatus.Processed, emptyList())
+    }
+
+    private suspend fun processFunction(
+        psiFunction: KtNamedFunction,
+        res: SubmissionRecord,
+        changesInFiles: Map<String, List<HunkJsonable>>,
+        project: ProjectRecord
+    ) {
+        val functionFullName = psiFunction.getFullName()
+        if (psiFunction.bodyExpression == null) {
+            log.info("BodyExpression is null in function=${functionFullName}, submission=${res.id}")
+            return
+        }
+        val functionRecord = dbFindAsync(FunctionRecord().apply { name = functionFullName })
+        val functionFromDb: FunctionRecord
+        when (functionRecord.size) {
+            0 -> {
+                log.info("Add new function=[${functionFullName}] in submission=[${res.id}]")
+                try {
+                    functionFromDb = dbCreateAsync(FunctionRecord().apply { name = functionFullName })
+                } catch (e: Exception) {
+                    log.error("Cant add function $functionFullName to functions table", e)
+                    return
+                }
+            }
+
+            1 -> {
+                functionFromDb = functionRecord.first()!!
+            }
+
+            else -> {
+                throw IllegalStateException(
+                    "Amount of function [${functionFullName}] in table is ${functionRecord.size}"
+                )
+            }
+        }
+        val document = psiFunction.containingFile.viewProvider.document
+            ?: throw IllegalStateException("Function's=[${psiFunction.containingFile.name}] document is null")
+        val fileChanges = changesInFiles[psiFunction.containingFile.name] ?: return //no changes in file at all
+        val funStartLine = document.getLineNumber(psiFunction.startOffsetSkippingComments) + 1
+        val funFinishLine = document.getLineNumber(psiFunction.endOffset) + 1
+        for (change in fileChanges) {
+            val fileRange = change.to
+            if (isNeedToRecomputeHash(funStartLine, funFinishLine, fileRange)) {
+                val hashesForLevels: MutableList<VisitResult> = computeHashesForElement(psiFunction.bodyExpression!!)
+                putHashesInTable(hashesForLevels, functionFromDb, res, project)
+                return
+            }
+        }
+    }
+
+    private suspend fun putHashesInTable(
+        hashes: MutableList<VisitResult>,
+        functionFromDb: FunctionRecord,
+        res: SubmissionRecord,
+        project: ProjectRecord
+    ) {
+        if (hashes.isEmpty()) {
+            log.info("Hashes for funId=${functionFromDb.id}, subId=${res.id} is empty")
+            return
+        }
+        dbBatchCreateAsync(hashes.map {
+            FunctionPartHashRecord().apply {
+                functionid = functionFromDb.id
+                submissionid = res.id
+                projectid = project.id
+                leftbound = it.leftBound
+                rightbound = it.rightBound
+                hash = it.levelHash
+            }
+        })
+        log.info("functionid = ${functionFromDb.id}, submissionid = ${res.id}, leavesСount = ${hashes.last().leafNum}")
+        dbCreateAsync(FunctionLeavesRecord().apply {
+            functionid = functionFromDb.id
+            submissionid = res.id
+            leavescount = hashes.last().leafNum
+        })
+    }
+
+    fun computeHashesForElement(root: PsiElement): MutableList<VisitResult> {
+        val visitResults = mutableListOf<VisitResult>()
+        val consumers = listOf(Consumer<VisitResult>{
+            visitResults.add(it)
+        })
+        treeHashVisitor.visitTree(root, consumers)
+        return visitResults
+    }
+
+    private fun isNeedToRecomputeHash(funStartLine: Int, funFinishLine: Int, changesRange: RangeJsonable): Boolean {
+        val start = changesRange.start
+        val finish = start + changesRange.count
+        val out = start > funFinishLine || finish < funStartLine
+        return !out
+    }
+}

kotoed-server/src/main/kotlin/org/jetbrains/research/kotoed/api/SubmissionCodeVerticle.kt

@@ -1,19 +1,28 @@
 package org.jetbrains.research.kotoed.api
 
+import org.jetbrains.research.kotoed.data.api.Code
 import org.jetbrains.research.kotoed.data.api.Code.FileRecord
 import org.jetbrains.research.kotoed.data.api.Code.FileType.directory
 import org.jetbrains.research.kotoed.data.api.Code.FileType.file
 import org.jetbrains.research.kotoed.data.api.Code.ListResponse
 import org.jetbrains.research.kotoed.data.api.Code.Submission.RemoteRequest
+import org.jetbrains.research.kotoed.data.api.DiffType
 import org.jetbrains.research.kotoed.data.db.ComplexDatabaseQuery
+import org.jetbrains.research.kotoed.data.db.setPageForQuery
 import org.jetbrains.research.kotoed.data.vcs.*
+import org.jetbrains.research.kotoed.database.Tables
 import org.jetbrains.research.kotoed.database.enums.SubmissionState
+import org.jetbrains.research.kotoed.database.tables.records.CommentTemplateRecord
 import org.jetbrains.research.kotoed.database.tables.records.CourseRecord
 import org.jetbrains.research.kotoed.database.tables.records.ProjectRecord
 import org.jetbrains.research.kotoed.database.tables.records.SubmissionRecord
+import org.jetbrains.research.kotoed.db.condition.lang.formatToQuery
 import org.jetbrains.research.kotoed.eventbus.Address
 import org.jetbrains.research.kotoed.util.*
+import org.jetbrains.research.kotoed.util.AnyAsJson.get
 import org.jetbrains.research.kotoed.util.database.toRecord
+import java.sql.Timestamp
+import java.time.OffsetDateTime
 import org.jetbrains.research.kotoed.data.api.Code.Course.ListRequest as CrsListRequest
 import org.jetbrains.research.kotoed.data.api.Code.Course.ReadRequest as CrsReadRequest
 import org.jetbrains.research.kotoed.data.api.Code.Course.ReadResponse as CrsReadResponse
@@ -132,6 +141,18 @@ class SubmissionCodeVerticle : AbstractKotoedVerticle() {
 
     @JsonableEventBusConsumerFor(Address.Api.Submission.Code.Diff)
     suspend fun handleSubmissionCodeDiff(message: SubDiffRequest): SubDiffResponse {
+        return diffResponse(message, DiffType.DIFF_WITH_CLOSED)
+    }
+
+    @JsonableEventBusConsumerFor(Address.Api.Submission.Code.DiffWithPrevious)
+    suspend fun handleSubmissionCodeDiffWithPrevious(message: SubDiffRequest): SubDiffResponse {
+        return diffResponse(message, DiffType.DIFF_WITH_PREVIOUS)
+    }
+
+    private suspend fun diffResponse(
+        message: Code.Submission.DiffRequest,
+        diffType: DiffType
+    ): Code.Submission.DiffResponse {
         val submission: SubmissionRecord = dbFetchAsync(SubmissionRecord().apply { id = message.submissionId })
         val repoInfo = getCommitInfo(submission)
         when (repoInfo.cloneStatus) {
@@ -140,7 +161,7 @@ class SubmissionCodeVerticle : AbstractKotoedVerticle() {
             else -> {
             }
         }
-        val diff = submissionCodeDiff(submission, repoInfo)
+        val diff = getDiff(submission, repoInfo, diffType)
         return SubDiffResponse(diff = diff.contents, status = repoInfo.cloneStatus)
     }
 
@@ -262,7 +283,21 @@ class SubmissionCodeVerticle : AbstractKotoedVerticle() {
         )
     }
 
-    private suspend fun submissionCodeDiff(submission: SubmissionRecord, repoInfo: CommitInfo): DiffResponse {
+    private suspend fun getDiff(
+        submission: SubmissionRecord,
+        repoInfo: CommitInfo,
+        diffType: DiffType
+    ): DiffResponse {
+        return when (diffType) {
+            DiffType.DIFF_WITH_CLOSED -> submissionCodeDiff(submission, repoInfo)
+            DiffType.DIFF_WITH_PREVIOUS -> submissionCodeDiffWithPrevious(submission, repoInfo)
+        }
+    }
+
+    private suspend fun submissionCodeDiff(
+        submission: SubmissionRecord,
+        repoInfo: CommitInfo
+    ): DiffResponse {
         val closedSubs = dbFindAsync(SubmissionRecord().apply {
             projectId = submission.projectId
             state = SubmissionState.closed
@@ -272,6 +307,38 @@ class SubmissionCodeVerticle : AbstractKotoedVerticle() {
             it.datetime < submission.datetime
         }.sortedByDescending { it.datetime }.firstOrNull()
 
+        return getDiffBetweenSubmission(foundationSub, submission, repoInfo)
+    }
+
+    private suspend fun submissionCodeDiffWithPrevious(
+        submission: SubmissionRecord,
+        repoInfo: CommitInfo
+    ): DiffResponse {
+
+        val newestPrevSub: SubmissionRecord? = dbQueryAsync(
+            ComplexDatabaseQuery(Tables.SUBMISSION)
+                .filter(
+                    ("${Tables.SUBMISSION.PROJECT_ID.name} == %s and " +
+                            "${Tables.SUBMISSION.STATE.name} != %s and " +
+                            "${Tables.SUBMISSION.STATE.name} != %s and " +
+                            "${Tables.SUBMISSION.DATETIME.name} < %s").formatToQuery(
+                        submission.projectId,
+                        SubmissionState.invalid,
+                        SubmissionState.pending,
+                        submission.datetime
+                    )
+                )
+                .sortBy(Tables.SUBMISSION.DATETIME.name)
+        ).lastOrNull()?.toRecord()
+
+        return getDiffBetweenSubmission(newestPrevSub, submission, repoInfo)
+    }
+
+    private suspend fun getDiffBetweenSubmission(
+        foundationSub: SubmissionRecord?,
+        submission: SubmissionRecord,
+        repoInfo: CommitInfo
+    ): DiffResponse {
         var baseRev = foundationSub?.revision
 
         if (baseRev == null) {

kotoed-server/src/main/kotlin/org/jetbrains/research/kotoed/code/klones/KloneVerticle.kt

@@ -13,21 +13,26 @@ import org.jetbrains.kotlin.psi.KtNamedFunction
 import org.jetbrains.kotlin.psi.psiUtil.collectDescendantsOfType
 import org.jetbrains.research.kotoed.code.Filename
 import org.jetbrains.research.kotoed.data.api.Code
+import org.jetbrains.research.kotoed.data.api.VerificationData
+import org.jetbrains.research.kotoed.data.api.VerificationStatus
 import org.jetbrains.research.kotoed.data.db.ComplexDatabaseQuery
+import org.jetbrains.research.kotoed.data.db.setPageForQuery
 import org.jetbrains.research.kotoed.data.vcs.CloneStatus
+import org.jetbrains.research.kotoed.database.Tables
 import org.jetbrains.research.kotoed.database.enums.SubmissionState
-import org.jetbrains.research.kotoed.database.tables.records.CourseRecord
-import org.jetbrains.research.kotoed.database.tables.records.ProjectRecord
-import org.jetbrains.research.kotoed.database.tables.records.SubmissionResultRecord
+import org.jetbrains.research.kotoed.database.tables.records.*
 import org.jetbrains.research.kotoed.db.condition.lang.formatToQuery
 import org.jetbrains.research.kotoed.eventbus.Address
 import org.jetbrains.research.kotoed.parsers.HaskellLexer
 import org.jetbrains.research.kotoed.util.*
 import org.jetbrains.research.kotoed.util.code.getPsi
 import org.jetbrains.research.kotoed.util.code.temporaryKotlinEnv
+import org.jetbrains.research.kotoed.util.database.toRecord
+import org.jooq.impl.DSL
 import org.kohsuke.randname.RandomNameGenerator
 import ru.spbstu.ktuples.placeholders._0
 import ru.spbstu.ktuples.placeholders.bind
+import java.util.concurrent.atomic.AtomicInteger
 
 sealed class KloneRequest(val priority: Int) : Jsonable, Comparable<KloneRequest> {
     override fun compareTo(other: KloneRequest): Int = priority - other.priority
@@ -75,6 +80,38 @@ class KloneVerticle : AbstractKotoedVerticle(), Loggable {
         return sendJsonableCollectAsync(Address.DB.query("submission"), q)
     }
 
+    @JsonableEventBusConsumerFor(Address.Code.ProjectKloneCheck)
+    suspend fun handleSimilarHashesForProject(projectRecord: ProjectRecord) {
+        dbQueryAsync(ComplexDatabaseQuery(Tables.FUNCTION_PART_HASH).filter("${projectRecord.id}"))
+    }
+
+    @JsonableEventBusConsumerFor(Address.Code.DifferenceBetweenKlones)
+    suspend fun handleDifference(projectRecord: ProjectRecord) {
+        dbQueryAsync(ComplexDatabaseQuery(Tables.FUNCTION_PART_HASH))
+    }
+
+    @JsonableEventBusConsumerFor(Address.Code.AllHashes)
+    suspend fun computeHashesForAllSubs(projectRecord: ProjectRecord) {
+        val startTime = System.currentTimeMillis()
+        val count = AtomicInteger()
+        val allSubs: List<JsonObject> = dbQueryAsync(
+            ComplexDatabaseQuery(Tables.SUBMISSION)
+                .join(ComplexDatabaseQuery(Tables.PROJECT).join(Tables.COURSE))
+                .filter("state != %s and state != %s and project.course.name == %s"
+                    .formatToQuery(SubmissionState.invalid, SubmissionState.pending, "KotlinAsFirst-2022"))
+                .limit(1000)
+        )
+        for (sub in allSubs) {
+            val submissionRecord = sub.toRecord<SubmissionRecord>()
+            val data: VerificationData = sendJsonableAsync(Address.Code.Hashes, submissionRecord)
+            if (data.status != VerificationStatus.Invalid) {
+                log.info("Count hashes for ${count.incrementAndGet()} submission")
+            }
+        }
+        log.info("All time in millis: ${System.currentTimeMillis() - startTime}")
+        log.info("Count hashes for ${count.get()} submission")
+    }
+
     @JsonableEventBusConsumerFor(Address.Code.KloneCheck)
     suspend fun handleCheck(course: CourseRecord) {
 

kotoed-server/src/main/kotlin/org/jetbrains/research/kotoed/data/api/Data.kt

@@ -23,6 +23,9 @@ enum class VerificationStatus {
     Processed,
     Invalid
 }
+enum class DiffType {
+    DIFF_WITH_PREVIOUS, DIFF_WITH_CLOSED
+}
 
 fun VerificationData?.bang() = this ?: VerificationData.Unknown