Incremental clone detecting

kotoed-server/db.properties

@@ -1,4 +1,4 @@
-db.url=jdbc:postgresql://localhost/kotoed
+db.url=jdbc:postgresql://localhost:5432/kotoedFinal
 db.user=kotoed
 db.password=kotoed
 testdb.url=jdbc:postgresql://localhost/kotoed-test

kotoed-server/src/main/kotlin/org/jetbrains/research/kotoed/api/SubmissionCodeVerticle.kt

@@ -1,19 +1,28 @@
 package org.jetbrains.research.kotoed.api
 
+import org.jetbrains.research.kotoed.data.api.Code
 import org.jetbrains.research.kotoed.data.api.Code.FileRecord
 import org.jetbrains.research.kotoed.data.api.Code.FileType.directory
 import org.jetbrains.research.kotoed.data.api.Code.FileType.file
 import org.jetbrains.research.kotoed.data.api.Code.ListResponse
 import org.jetbrains.research.kotoed.data.api.Code.Submission.RemoteRequest
+import org.jetbrains.research.kotoed.data.api.DiffType
 import org.jetbrains.research.kotoed.data.db.ComplexDatabaseQuery
+import org.jetbrains.research.kotoed.data.db.setPageForQuery
 import org.jetbrains.research.kotoed.data.vcs.*
+import org.jetbrains.research.kotoed.database.Tables
 import org.jetbrains.research.kotoed.database.enums.SubmissionState
+import org.jetbrains.research.kotoed.database.tables.records.CommentTemplateRecord
 import org.jetbrains.research.kotoed.database.tables.records.CourseRecord
 import org.jetbrains.research.kotoed.database.tables.records.ProjectRecord
 import org.jetbrains.research.kotoed.database.tables.records.SubmissionRecord
+import org.jetbrains.research.kotoed.db.condition.lang.formatToQuery
 import org.jetbrains.research.kotoed.eventbus.Address
 import org.jetbrains.research.kotoed.util.*
+import org.jetbrains.research.kotoed.util.AnyAsJson.get
 import org.jetbrains.research.kotoed.util.database.toRecord
+import java.sql.Timestamp
+import java.time.OffsetDateTime
 import org.jetbrains.research.kotoed.data.api.Code.Course.ListRequest as CrsListRequest
 import org.jetbrains.research.kotoed.data.api.Code.Course.ReadRequest as CrsReadRequest
 import org.jetbrains.research.kotoed.data.api.Code.Course.ReadResponse as CrsReadResponse
@@ -132,6 +141,18 @@ class SubmissionCodeVerticle : AbstractKotoedVerticle() {
 
     @JsonableEventBusConsumerFor(Address.Api.Submission.Code.Diff)
     suspend fun handleSubmissionCodeDiff(message: SubDiffRequest): SubDiffResponse {
+        return diffResponse(message, DiffType.DIFF_WITH_CLOSED)
+    }
+
+    @JsonableEventBusConsumerFor(Address.Api.Submission.Code.DiffWithPrevious)
+    suspend fun handleSubmissionCodeDiffWithPrevious(message: SubDiffRequest): SubDiffResponse {
+        return diffResponse(message, DiffType.DIFF_WITH_PREVIOUS)
+    }
+
+    private suspend fun diffResponse(
+        message: Code.Submission.DiffRequest,
+        diffType: DiffType
+    ): Code.Submission.DiffResponse {
         val submission: SubmissionRecord = dbFetchAsync(SubmissionRecord().apply { id = message.submissionId })
         val repoInfo = getCommitInfo(submission)
         when (repoInfo.cloneStatus) {
@@ -140,7 +161,7 @@ class SubmissionCodeVerticle : AbstractKotoedVerticle() {
             else -> {
             }
         }
-        val diff = submissionCodeDiff(submission, repoInfo)
+        val diff = getDiff(submission, repoInfo, diffType)
         return SubDiffResponse(diff = diff.contents, status = repoInfo.cloneStatus)
     }
 
@@ -262,7 +283,21 @@ class SubmissionCodeVerticle : AbstractKotoedVerticle() {
         )
     }
 
-    private suspend fun submissionCodeDiff(submission: SubmissionRecord, repoInfo: CommitInfo): DiffResponse {
+    private suspend fun getDiff(
+        submission: SubmissionRecord,
+        repoInfo: CommitInfo,
+        diffType: DiffType
+    ): DiffResponse {
+        return when (diffType) {
+            DiffType.DIFF_WITH_CLOSED -> submissionCodeDiff(submission, repoInfo)
+            DiffType.DIFF_WITH_PREVIOUS -> submissionCodeDiffWithPrevious(submission, repoInfo)
+        }
+    }
+
+    private suspend fun submissionCodeDiff(
+        submission: SubmissionRecord,
+        repoInfo: CommitInfo
+    ): DiffResponse {
         val closedSubs = dbFindAsync(SubmissionRecord().apply {
             projectId = submission.projectId
             state = SubmissionState.closed
@@ -272,6 +307,38 @@ class SubmissionCodeVerticle : AbstractKotoedVerticle() {
             it.datetime < submission.datetime
         }.sortedByDescending { it.datetime }.firstOrNull()
 
+        return getDiffBetweenSubmission(foundationSub, submission, repoInfo)
+    }
+
+    private suspend fun submissionCodeDiffWithPrevious(
+        submission: SubmissionRecord,
+        repoInfo: CommitInfo
+    ): DiffResponse {
+
+        val newestPrevSub: SubmissionRecord? = dbQueryAsync(
+            ComplexDatabaseQuery(Tables.SUBMISSION)
+                .filter(
+                    ("${Tables.SUBMISSION.PROJECT_ID.name} == %s and " +
+                            "${Tables.SUBMISSION.STATE.name} != %s and " +
+                            "${Tables.SUBMISSION.STATE.name} != %s and " +
+                            "${Tables.SUBMISSION.DATETIME.name} < %s").formatToQuery(
+                        submission.projectId,
+                        SubmissionState.invalid,
+                        SubmissionState.pending,
+                        submission.datetime
+                    )
+                )
+                .sortBy(Tables.SUBMISSION.DATETIME.name)
+        ).lastOrNull()?.toRecord()
+
+        return getDiffBetweenSubmission(newestPrevSub, submission, repoInfo)
+    }
+
+    private suspend fun getDiffBetweenSubmission(
+        foundationSub: SubmissionRecord?,
+        submission: SubmissionRecord,
+        repoInfo: CommitInfo
+    ): DiffResponse {
         var baseRev = foundationSub?.revision
 
         if (baseRev == null) {

kotoed-server/src/main/kotlin/org/jetbrains/research/kotoed/code/klones/KloneVerticle.kt

@@ -14,9 +14,13 @@ import org.jetbrains.kotlin.psi.psiUtil.collectDescendantsOfType
 import org.jetbrains.research.kotoed.code.Filename
 import org.jetbrains.research.kotoed.data.api.Code
 import org.jetbrains.research.kotoed.data.db.ComplexDatabaseQuery
+import org.jetbrains.research.kotoed.data.db.setPageForQuery
 import org.jetbrains.research.kotoed.data.vcs.CloneStatus
+import org.jetbrains.research.kotoed.database.Tables
 import org.jetbrains.research.kotoed.database.enums.SubmissionState
 import org.jetbrains.research.kotoed.database.tables.records.CourseRecord
+import org.jetbrains.research.kotoed.database.tables.records.FunctionPartHashRecord
+import org.jetbrains.research.kotoed.database.tables.records.ProcessedProjectSubRecord
 import org.jetbrains.research.kotoed.database.tables.records.ProjectRecord
 import org.jetbrains.research.kotoed.database.tables.records.SubmissionResultRecord
 import org.jetbrains.research.kotoed.db.condition.lang.formatToQuery
@@ -25,6 +29,7 @@ import org.jetbrains.research.kotoed.parsers.HaskellLexer
 import org.jetbrains.research.kotoed.util.*
 import org.jetbrains.research.kotoed.util.code.getPsi
 import org.jetbrains.research.kotoed.util.code.temporaryKotlinEnv
+import org.jooq.impl.DSL
 import org.kohsuke.randname.RandomNameGenerator
 import ru.spbstu.ktuples.placeholders._0
 import ru.spbstu.ktuples.placeholders.bind
@@ -75,6 +80,16 @@ class KloneVerticle : AbstractKotoedVerticle(), Loggable {
         return sendJsonableCollectAsync(Address.DB.query("submission"), q)
     }
 
+    @JsonableEventBusConsumerFor(Address.Code.ProjectKloneCheck)
+    suspend fun handleSimilarHashesForProject(projectRecord: ProjectRecord) {
+       dbQueryAsync(ComplexDatabaseQuery(Tables.FUNCTION_PART_HASH).filter("${projectRecord.id}"))
+    }
+
+    @JsonableEventBusConsumerFor(Address.Code.DifferenceBetweenKlones)
+    suspend fun handleDifference(projectRecord: ProjectRecord) {
+        dbQueryAsync(ComplexDatabaseQuery(Tables.FUNCTION_PART_HASH))
+    }
+
     @JsonableEventBusConsumerFor(Address.Code.KloneCheck)
     suspend fun handleCheck(course: CourseRecord) {
 

kotoed-server/src/main/kotlin/org/jetbrains/research/kotoed/data/api/Data.kt

@@ -23,6 +23,9 @@ enum class VerificationStatus {
     Processed,
     Invalid
 }
+enum class DiffType {
+    DIFF_WITH_PREVIOUS, DIFF_WITH_CLOSED
+}
 
 fun VerificationData?.bang() = this ?: VerificationData.Unknown
 

kotoed-server/src/main/kotlin/org/jetbrains/research/kotoed/db/FunctionLeavesVerticle.kt

@@ -0,0 +1,8 @@
+package org.jetbrains.research.kotoed.db
+
+import org.jetbrains.research.kotoed.database.Tables
+import org.jetbrains.research.kotoed.database.tables.records.FunctionLeavesRecord
+import org.jetbrains.research.kotoed.util.AutoDeployable
+
+@AutoDeployable
+class FunctionLeavesVerticle : CrudDatabaseVerticleWithReferences<FunctionLeavesRecord>(Tables.FUNCTION_LEAVES)