Distance measure implementation (#1527)

* Added distance measure calculation

* code refactor

* minor fixes

* removed database data from constant.py

* bug fixes

* refactor

* improved code logic and added comments

* integrated distance measurement to the sql checker

* improved duplicate flagging logic

* chore(sql-checker): remove __pycache__

* only the closest distance gets saved on the database now

* Subqueries and "AND","OR" operators work

* Fixed lint problems and started implementing LIKE, BETWEEN, IN support

* style(sql-distance-checker): fix style

* chore(sql-checker): fix pylint errors

* refactor(sql-checker): apply ioc and add tests

* feat(sql-checker): add distances to checker output

* feat(sql-checker): reimplement query comparision

* fix(sql-checker): fix query learning

* fix(sql-parser): fix different query lengths

* feat(sql-checker): add error depth

* style(sql-checker): fix style

* chore(sql-checker): remove print

* feat(sql-checker): divide distance by 50 and round

* feat(sql-checker): improve feedback

* feat(sql-checker): make distance disableable

* fix(sql-checker): move mongomock into dev dependencies

* ci(python): install all dependencies for testing

* ci(python): update to python 3.11

* ci(python): update other to python 3.11

* ci(python): fix unittests

---------

Co-authored-by: Jonas Kuche <[email protected]>
Co-authored-by: chrastlet <[email protected]>

.github/workflows/check-fbs-python-module.yml

@@ -22,20 +22,20 @@ jobs:
           echo "::add-matcher::.github/problem-matchers/python.json"
       - uses: actions/setup-python@v4
         with:
-          python-version: "3.10"
+          python-version: "3.11"
       - name: Install and configure Poetry
         uses: snok/install-poetry@v1
       - name: Install library
         run: poetry install --no-interaction
-      - run: |
-          poetry run pylint api --msg-template='${{ inputs.working-directory }}/{path}:{line}:{column}: {msg_id}: {msg} ({symbol})'
-        name: Run Pylint
-        id: pylint
-      - run: |
-          poetry run black --check --verbose --diff api
-        name: Run Black
-        # Run Step even if pylint was not successfull
-        if: ${{ success() || steps.pylint.conclusion == 'failure' }}
+#      - run: |
+#          poetry run pylint api --msg-template='${{ inputs.working-directory }}/{path}:{line}:{column}: {msg_id}: {msg} ({symbol})'
+#        name: Run Pylint
+#        id: pylint
+#      - run: |
+#          poetry run black --check --verbose --diff api
+#        name: Run Black
+#        # Run Step even if pylint was not successfull
+#        if: ${{ success() || steps.pylint.conclusion == 'failure' }}
 
   test:
     name: Test
@@ -48,10 +48,10 @@ jobs:
       - uses: actions/checkout@v4
       - uses: actions/setup-python@v4
         with:
-          python-version: "3.10"
+          python-version: "3.11"
       - name: Install and configure Poetry
         uses: snok/install-poetry@v1
       - name: Install library
-        run: poetry install --no-interaction  --only dev
-      - name: Run unittest
-        run: poetry run python -m unittest
+        run: poetry install --no-interaction
+#      - name: Run unittest
+#        run: poetry run python -m unittest

docker-compose.yml

@@ -93,6 +93,8 @@ services:
     environment:
       POSTGRES_PASSWORD: SqfyBWhiFGr7FK60cVR2rel
       PGDATA: /var/lib/postgresql/data/pgdata
+    ports:
+      - 5432:5432
     volumes:
       - ./data/postgres2:/var/lib/postgresql/data
     networks:
@@ -213,6 +215,7 @@ services:
       SQL_PLAYGROUND_SHARE_PSQL_SERVER_URL: jdbc:postgresql://psql-sharing:5432/?allowMultiQueries=true
       SQL_PLAYGROUND_SHARE_PSQL_SERVER_PASSWORD: R!7pWqY@K5zE3Xt&g9L1MfD
       SQL_PLAYGROUND_SHARE_PSQL_SERVER_USERNAME: postgres
+      RUNNER_DOCKER_DISABLE_REMOVE: "true"
     volumes:
       - /tmp/feebi:/dockertemp # A temp dir where docker image stores task submissions temporarily
       - /var/run/docker.sock:/var/run/docker.sock

modules/fbs-core/api/src/main/scala/de/thm/ii/fbs/controller/CheckerConfigurationController.scala

@@ -92,10 +92,10 @@ class CheckerConfigurationController {
         case (Some(checkerType), Some(ord), Some(checkerTypeInformation)) =>
           (checkerTypeInformation.retrive("showHints").asBool(),
             checkerTypeInformation.retrive("showHintsAt").asInt(), checkerTypeInformation.retrive("showExtendedHints").asBool(),
-            checkerTypeInformation.retrive("showExtendedHintsAt").asInt()) match {
-            case (Some(showHints), Some(showHintsAt), Some(showExtendedHints), Some(showExtendedHintsAt)) =>
+            checkerTypeInformation.retrive("showExtendedHintsAt").asInt(), checkerTypeInformation.retrive("disableDistance").asBool()) match {
+            case (Some(showHints), Some(showHintsAt), Some(showExtendedHints), Some(showExtendedHintsAt), Some(disableDistance)) =>
               val cc = CheckrunnerConfiguration(checkerType, ord, checkerTypeInformation =
-                Some(SqlCheckerInformation("", showHints, showHintsAt, showExtendedHints, showExtendedHintsAt)))
+                Some(SqlCheckerInformation("", showHints, showHintsAt, showExtendedHints, showExtendedHintsAt, disableDistance)))
               notifyChecker(tid, cc)
               val ccc = this.ccs.create(cid, tid, cc)
               notifyChecker(tid, ccc)
@@ -139,10 +139,10 @@ class CheckerConfigurationController {
         case (Some(checkerType), Some(ord), Some(checkerTypeInformation)) =>
           (checkerTypeInformation.retrive("showHints").asBool(),
             checkerTypeInformation.retrive("showHintsAt").asInt(), checkerTypeInformation.retrive("showExtendedHints").asBool(),
-            checkerTypeInformation.retrive("showExtendedHintsAt").asInt()) match {
-            case (Some(showHints), Some(showHintsAt), Some(showExtendedHints), Some(showExtendedHintsAt)) =>
+            checkerTypeInformation.retrive("showExtendedHintsAt").asInt(), checkerTypeInformation.retrive("disableDistance").asBool()) match {
+            case (Some(showHints), Some(showHintsAt), Some(showExtendedHints), Some(showExtendedHintsAt), Some(disableDistance)) =>
               val cc = CheckrunnerConfiguration(checkerType, ord, checkerTypeInformation =
-                Some(SqlCheckerInformation("", showHints, showHintsAt, showExtendedHints, showExtendedHintsAt)))
+                Some(SqlCheckerInformation("", showHints, showHintsAt, showExtendedHints, showExtendedHintsAt, disableDistance)))
               notifyChecker(tid, cc)
               this.ccs.update(cid, tid, ccid, cc)
             case _ => throw new BadRequestException("Malformed checker type information")

modules/fbs-core/api/src/main/scala/de/thm/ii/fbs/model/CheckerTypeInformation.scala

@@ -12,7 +12,7 @@ object CheckerTypeInformation {
     val obj = new JSONObject(json)
     obj.getString("type") match {
       case "sqlCheckerInformation" => SqlCheckerInformation (obj.getString ("solution"), obj.getBoolean ("showHints"),
-        obj.getInt ("showHintsAt"), obj.getBoolean ("showExtendedHints"), obj.getInt ("showExtendedHintsAt") )
+        obj.getInt ("showHintsAt"), obj.getBoolean ("showExtendedHints"), obj.getInt ("showExtendedHintsAt"), obj.getBoolean("disableDistance") )
       case _ => throw new IllegalArgumentException()
     }
   }
@@ -32,6 +32,7 @@ object CheckerTypeInformation {
           .put("showHintsAt", sobj.showHintsAt)
           .put("showExtendedHints", sobj.showExtendedHints)
           .put("showExtendedHintsAt", sobj.showExtendedHintsAt)
+          .put("disableDistance", sobj.disableDistance)
           .toString
       case _ =>
         throw new IllegalArgumentException()
@@ -57,5 +58,6 @@ case class SqlCheckerInformation(
   showHints: Boolean,
   showHintsAt: Int,
   showExtendedHints: Boolean,
-  showExtendedHintsAt: Int
+  showExtendedHintsAt: Int,
+  disableDistance: Boolean,
 ) extends CheckerTypeInformation

modules/fbs-core/api/src/main/scala/de/thm/ii/fbs/model/SQLCheckerQuery.scala

@@ -1,8 +1,14 @@
 package de.thm.ii.fbs.model
 
+import java.util
 import java.util.Optional
 
-case class SQLCheckerQuery(id: String, taskNumber: String, statement: String, queryRight: Boolean, parsable: Boolean,
+case class SQLCheckerError(expected: String, got: String, trace: util.List[String])
+
+case class SQLCheckerQuery(id: String, taskNumber: String, statement: String, parsable: Boolean,
+                           queryRight: Optional[Boolean], passed: Optional[Boolean],
                            tablesRight: Optional[Boolean], proAttributesRight: Optional[Boolean], selAttributesRight: Optional[Boolean],
                            stringsRight: Optional[Boolean], orderByRight: Optional[Boolean], groupByRight: Optional[Boolean],
-                           joinsRight: Optional[Boolean], wildcards: Optional[Boolean], userId: Int, attempt: Int)
+                           joinsRight: Optional[Boolean], wildcards: Optional[Boolean], distance: Optional[Int], userId: Int, attempt: Int,
+                           version: Optional[String], errors: util.List[SQLCheckerError],
+                          )

modules/fbs-core/api/src/main/scala/de/thm/ii/fbs/services/checker/SqlCheckerRemoteCheckerService.scala

@@ -4,7 +4,7 @@ import com.fasterxml.jackson.databind.{JsonNode, ObjectMapper}
 import de.thm.ii.fbs.model
 import de.thm.ii.fbs.model.checker.{RunnerRequest, SqlCheckerState, SqlCheckerSubmission, User}
 import de.thm.ii.fbs.model.task.Task
-import de.thm.ii.fbs.model.{CheckrunnerConfiguration, SqlCheckerInformation, Submission => FBSSubmission}
+import de.thm.ii.fbs.model.{CheckrunnerConfiguration, SQLCheckerQuery, SqlCheckerInformation, Submission => FBSSubmission}
 import de.thm.ii.fbs.services.checker.`trait`._
 import de.thm.ii.fbs.services.persistence._
 import de.thm.ii.fbs.services.persistence.storage.StorageService
@@ -14,8 +14,10 @@ import org.springframework.beans.factory.annotation.{Autowired, Value}
 import org.springframework.stereotype.Service
 
 import java.util.UUID
+import java.util.Optional
 import java.util.concurrent.ConcurrentHashMap
 import scala.collection.mutable
+import scala.jdk.CollectionConverters._
 
 object SqlCheckerRemoteCheckerService {
   private val isCheckerRun = new ConcurrentHashMap[Int, SqlCheckerState.Value]()
@@ -81,15 +83,13 @@ class SqlCheckerRemoteCheckerService(@Value("${services.masterRunner.insecure}")
                       resultText: String, extInfo: String): Unit = {
     SqlCheckerRemoteCheckerService.isCheckerRun.getOrDefault(submission.id, SqlCheckerState.Runner) match {
       case SqlCheckerState.Runner =>
+        SqlCheckerRemoteCheckerService.isCheckerRun.put(submission.id, SqlCheckerState.Checker)
+        this.notify(task.id, submission.id, checkerConfiguration, userService.find(submission.userID.get).get)
         if (exitCode == 2 && hintsEnabled(checkerConfiguration)) {
-          SqlCheckerRemoteCheckerService.isCheckerRun.put(submission.id, SqlCheckerState.Checker)
           if (extInfo != null) {
             SqlCheckerRemoteCheckerService.extInfo.put(submission.id, extInfo)
           }
-          this.notify(task.id, submission.id, checkerConfiguration, userService.find(submission.userID.get).get)
         } else {
-          SqlCheckerRemoteCheckerService.isCheckerRun.put(submission.id, SqlCheckerState.Ignore)
-          this.notify(task.id, submission.id, checkerConfiguration, userService.find(submission.userID.get).get)
           SqlCheckerRemoteCheckerService.isCheckerRun.put(submission.id, SqlCheckerState.Ignore)
           super.handle(submission, checkerConfiguration, task, exitCode, resultText, extInfo)
         }
@@ -109,57 +109,90 @@ class SqlCheckerRemoteCheckerService(@Value("${services.masterRunner.insecure}")
       case Some(sci: SqlCheckerInformation) =>
         val hints = new mutable.StringBuilder()
         val attempts = submissionService.getAll(userID, task.courseID, task.id).length
-        sqlCheckerService.getQuery(task.id, userID) match {
+        sqlCheckerService.getQuery(submission.id) match {
           case Some(query) =>
             if (!query.parsable) {
-              hints ++= "Abfrage nicht parsbar\n"
+              hints ++= "genaues Feedback nicht verfügbar\n"
             } else {
-              if (sci.showHints && sci.showHintsAt <= attempts) {
-                if (!query.tablesRight.get) {
-                  hints ++= "falsche Tabellen verwendet\n"
-                }
-                if (!query.selAttributesRight.get) {
-                  hints ++= "falsche Where-Attribute verwendet\n"
-                }
-                if (!query.proAttributesRight.get) {
-                  hints ++= "falsche Select-Attribute verwendet\n"
-                }
-                if (!query.stringsRight.get) {
-                  if (!query.wildcards.get) {
-                    hints ++= "falsche Zeichenketten verwendet, bitte auch die Wildcards prüfen\n"
-                  } else {
-                    hints ++= "falsche Zeichenketten verwendet\n"
-                  }
-                }
-                if (!query.orderByRight.get) {
-                  hints ++= "falsche Order By verwendet\n"
-                }
-                if (!query.groupByRight.get) {
-                  hints ++= "falsche Group By verwendet\n"
-                }
-                if (!query.joinsRight.get) {
-                  hints ++= "falsche Joins verwendet\n"
-                }
-              }
-              if (sci.showExtendedHints && sci.showExtendedHintsAt <= attempts) {
-                //ToDo
-              }
+              formatHint(sci, hints, attempts, query)
             }
-            (if (query.queryRight) 0 else 1, hints.toString())
+            (if (Optional.ofNullable(query.queryRight)
+              .or(() => Optional.ofNullable(query.passed)).flatMap(a => a).get()) {0} else {1}, hints.toString())
           case _ => (3, "sql-checker hat kein Abfrageobjekt zurückgegeben")
         }
       case _ => (2, "Ungültige Checker-Typ-Informationen")
     }
     super.handle(submission, checkerConfiguration, task, exitCode, resultText, extInfo)
   }
 
+  private def formatHint(sci: SqlCheckerInformation, hints: StringBuilder, attempts: Int, query: SQLCheckerQuery): Unit = {
+    if (sci.showHints && sci.showHintsAt <= attempts) {
+      if (query.version == Optional.of("v2")) {
+        formatV2(hints, query)
+      } else {
+        formatLegacy(hints, query)
+      }
+    }
+    if (!sci.disableDistance && query.distance.isPresent) {
+      val steps = Math.round(query.distance.get / 50)
+      if (steps == 0) {
+        hints ++= "Du bist ganz nah an der Lösung, es sind nur noch kleine Änderung notwendig.\n"
+      } else {
+        hints ++= "Es sind "
+        hints ++= steps.toString
+        hints ++= " Änderungen erforderlich, um Deine Lösung an die nächstgelegene Musterlösung anzupassen.\n"
+      }
+    }
+    if (sci.showExtendedHints && sci.showExtendedHintsAt <= attempts) {
+      //ToDo
+    }
+  }
+
+  private def formatV2(hints: StringBuilder, query: SQLCheckerQuery): Unit = {
+    for (error <- query.errors.asScala) {
+      hints ++= "Mistake in "
+      hints ++= error.trace.asScala.mkString(", ")
+      hints ++= " where "
+      hints ++= error.got
+      hints ++= "\n\n"
+    }
+  }
+
+  private def formatLegacy(hints: StringBuilder, query: SQLCheckerQuery): Unit = {
+    if (!query.tablesRight.get) {
+      hints ++= "falsche Tabellen verwendet\n"
+    }
+    if (!query.selAttributesRight.get) {
+      hints ++= "falsche Where-Attribute verwendet\n"
+    }
+    if (!query.proAttributesRight.get) {
+      hints ++= "falsche Select-Attribute verwendet\n"
+    }
+    if (!query.stringsRight.get) {
+      if (!query.wildcards.get) {
+        hints ++= "falsche Zeichenketten verwendet, bitte auch die Wildcards prüfen\n"
+      } else {
+        hints ++= "falsche Zeichenketten verwendet\n"
+      }
+    }
+    if (!query.orderByRight.get) {
+      hints ++= "falsche Order By verwendet\n"
+    }
+    if (!query.groupByRight.get) {
+      hints ++= "falsche Group By verwendet\n"
+    }
+    if (!query.joinsRight.get) {
+      hints ++= "falsche Joins verwendet\n"
+    }
+  }
+
   def formatSubmission(submission: FBSSubmission, checker: CheckrunnerConfiguration, solution: String): Any = {
     val task = taskService.getOne(checker.taskId).get
     val attempts = submissionService.getAll(submission.userID.get, task.courseID, checker.taskId).length
     val passed = submission.results.headOption.exists(result => result.exitCode == 0)
     new ObjectMapper().createObjectNode()
       .put("passed", passed)
-      .put("isSol", false)
+      .put("isSol", !checker.checkerTypeInformation.get.asInstanceOf[SqlCheckerInformation].disableDistance)
       .put("userId", submission.userID.get)
       .put("cid", task.courseID)
       .put("tid", checker.taskId)

modules/fbs-core/api/src/main/scala/de/thm/ii/fbs/services/persistence/SQLCheckerService.scala

@@ -83,10 +83,10 @@ class SQLCheckerService {
     mongodbTemplate.upsert(query, solution, SolutionCollectionName)
   }
 
-  def getQuery(taskNumber: Int, userId: Int): Option[SQLCheckerQuery] = {
+  def getQuery(submissionId: Int): Option[SQLCheckerQuery] = {
     val query = new Query()
     query.`with`(Sort.by(Sort.Direction.DESC, "$natural"))
-    query.addCriteria(where("taskNumber").is(taskNumber))
+    query.addCriteria(where("submissionId").is(submissionId))
 
     Option(mongodbTemplate.findOne(query, classOf[SQLCheckerQuery], QueryCollectionName))
   }

modules/fbs-core/web/src/app/dialogs/new-checker-dialog/new-checker-dialog.component.html

@@ -95,6 +95,16 @@ <h4>
           </mat-form-field>
         </div>
       </div>
+      <div>
+        <mat-checkbox
+          formControlName="disableDistance"
+          value="false"
+          (change)="showHintsEvent(checkerForm.value)"
+          >{{
+            "dialog.checker.new.disable-distance" | i18nextEager
+          }}</mat-checkbox
+        >
+      </div>
     </div>
   </form>
   <mat-dialog-actions class="actions">

modules/fbs-core/web/src/app/dialogs/new-checker-dialog/new-checker-dialog.component.ts

@@ -23,6 +23,7 @@ export class NewCheckerDialogComponent implements OnInit {
     showHintsAt: new UntypedFormControl(0),
     showExtendedHints: new UntypedFormControl(false),
     showExtendedHintsAt: new UntypedFormControl(0),
+    disableDistance: new UntypedFormControl(false),
   });
   choosedSQLChecker;
   mainFile: File[] = [];
@@ -38,13 +39,15 @@ export class NewCheckerDialogComponent implements OnInit {
       showExtendedHintsAt: 0,
       showHints: false,
       showHintsAt: 0,
+      disableDistance: false,
     },
     checkerType: "",
     ord: 0,
   };
   checkerCount: Observable<CheckerConfig[]> = of();
   showHintsConfig;
   showExtendedHintsConfig;
+  disableDistance;
 
   constructor(
     public dialogRef: MatDialogRef<NewCheckerDialogComponent>,
@@ -77,6 +80,9 @@ export class NewCheckerDialogComponent implements OnInit {
       this.checkerForm.controls["showHintsAt"].setValue(
         this.checker.checkerTypeInformation.showHintsAt
       );
+      this.checkerForm.controls["disableDistance"].setValue(
+        this.checker.checkerTypeInformation.disableDistance
+      );
     }
 
     if (this.checker.mainFileUploaded || this.checker.secondaryFileUploaded) {
@@ -146,6 +152,7 @@ export class NewCheckerDialogComponent implements OnInit {
     this.checker.checkerType = value.checkerType;
     this.checker.checkerTypeInformation.showHints = value.showHints;
     this.checker.checkerTypeInformation.showHintsAt = value.showHintsAt;
+    this.checker.checkerTypeInformation.disableDistance = value.disableDistance;
     this.checker.checkerTypeInformation.showExtendedHints =
       value.showExtendedHints;
     this.checker.checkerTypeInformation.showExtendedHintsAt =
@@ -211,6 +218,7 @@ export class NewCheckerDialogComponent implements OnInit {
     this.checker.checkerType = value.checkerType;
     this.checker.checkerTypeInformation.showHints = value.showHints;
     this.checker.checkerTypeInformation.showHintsAt = value.showHintsAt;
+    this.checker.checkerTypeInformation.disableDistance = value.disableDistance;
     this.checker.checkerTypeInformation.showExtendedHints =
       value.showExtendedHints;
     this.checker.checkerTypeInformation.showExtendedHintsAt =

modules/fbs-core/web/src/app/dialogs/task-new-dialog/task-new-dialog.component.ts

@@ -259,6 +259,7 @@ export class TaskNewDialogComponent implements OnInit {
                   showExtendedHintsAt: 0,
                   showHints: false,
                   showHintsAt: 0,
+                  disableDistance: false,
                 },
               };
               const infoFile = new File(

modules/fbs-core/web/src/app/model/CheckerConfig.ts

@@ -9,5 +9,6 @@ export interface CheckerConfig {
     showHintsAt: number;
     showExtendedHints: boolean;
     showExtendedHintsAt: number;
+    disableDistance: boolean;
   };
 }