diff --git a/sarif/src/test/java/com/jetbrains/qodana/sarif/BaselineFingerprintTest.kt b/sarif/src/test/java/com/jetbrains/qodana/sarif/BaselineFingerprintTest.kt new file mode 100644 index 0000000..f78e48e --- /dev/null +++ b/sarif/src/test/java/com/jetbrains/qodana/sarif/BaselineFingerprintTest.kt @@ -0,0 +1,90 @@ +package com.jetbrains.qodana.sarif + +import com.jetbrains.qodana.sarif.baseline.BaselineCalculation +import com.jetbrains.qodana.sarif.model.Result.BaselineState +import com.jetbrains.qodana.sarif.model.SarifReport +import org.junit.jupiter.api.Assertions +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.function.Executable +import java.nio.file.Paths +import kotlin.streams.asStream + +class BaselineFingerprintTest { + private fun read(path: String) = SarifUtil.readReport(Paths.get(path)) + + // The findings in both reports are the same except for the human-readable message + private fun readBaseline(fingerprintVersion: String) = + read("src/test/resources/testData/fingerprinting/before.sarif_$fingerprintVersion.json") + + private fun readReport(fingerprintVersion: String) = + read("src/test/resources/testData/fingerprinting/after.sarif_$fingerprintVersion.json") + + private fun test( + report: SarifReport, + baseline: SarifReport, + expect: Map, + ) { + val res = BaselineCalculation.compare(report, baseline, BaselineCalculation.Options(true)) + + val byState = report.runs.orEmpty().asSequence() + .flatMap { it.results.orEmpty() } + .groupingBy { it.baselineState } + .eachCount() + + sequenceOf(BaselineState.UNCHANGED, BaselineState.ABSENT, BaselineState.NEW) + .flatMap { + val inReport = byState[it] ?: 0 + val inResult = when (it) { + BaselineState.NEW -> res.newResults + BaselineState.UNCHANGED -> res.unchangedResults + BaselineState.ABSENT -> res.absentResults + BaselineState.UPDATED -> error("Not in source") + } + val expected = expect[it] ?: 0 + sequenceOf( + Executable { Assertions.assertEquals(expected, inReport, "[$it] in report") }, + Executable { Assertions.assertEquals(expected, inResult, "[$it] in result") }, + ) + } + .asStream() + .let(Assertions::assertAll) + } + + private val expectBaselineMismatch = mapOf( + BaselineState.UNCHANGED to 1, + BaselineState.ABSENT to 17, + BaselineState.NEW to 17, + ) + + private val expectBaselineMatch = mapOf(BaselineState.UNCHANGED to 18) + + @Test + fun `both are v1`() { + // V1 fingerprint checks message => results are all absent / new + test(readReport("v1"), readBaseline("v1"), expectBaselineMismatch) + } + + @Test + fun `both are v2`() { + // V2 fingerprint ignores message => all results are unchanged + test(readReport("v2"), readBaseline("v2"), expectBaselineMatch) + } + + @Test + fun `backward compat -- report v1_v2 - baseline v1`() { + // we can only compare v1 because baseline doesn't have v2 + test(readReport("v1_v2"), readBaseline("v1"), expectBaselineMismatch) + } + + @Test + fun `forward compat -- report v1 - baseline v1_v2`() { + // we can only compare v1 because report doesn't have v2 + test(readReport("v1"), readBaseline("v1_v2"), expectBaselineMismatch) + } + + @Test + fun `v1_v2 to v1_v2`() { + // while V1 differs, V2 is the same, so expect match + test(readReport("v1_v2"), readBaseline("v1_v2"), expectBaselineMatch) + } +} diff --git a/sarif/src/test/java/com/jetbrains/qodana/sarif/BaselineTest.java b/sarif/src/test/java/com/jetbrains/qodana/sarif/BaselineTest.java index 8521fde..7cb9a6f 100644 --- a/sarif/src/test/java/com/jetbrains/qodana/sarif/BaselineTest.java +++ b/sarif/src/test/java/com/jetbrains/qodana/sarif/BaselineTest.java @@ -20,6 +20,7 @@ public class BaselineTest { private static final String QODANA_REPORT_JSON = "src/test/resources/testData/readWriteTest/qodanaReport.json"; private static final String QODANA_REPORT_JSON_2 = "src/test/resources/testData/readWriteTest/qodanaReport2.json"; + private static final BaselineCalculation.Options INCLUDE_ABSENT = new BaselineCalculation.Options(true); @Test public void testSameReport() throws IOException { SarifReport report = readReport(); @@ -59,7 +60,7 @@ public void testCompareWithOneAbsent() throws IOException { Result newResult = new Result(new Message().withText("new result")); baseline.getRuns().get(0).getResults().add(newResult); - doTest(report, baseline, problemsCount(report), 1, 0, new BaselineCalculation.Options(true)); + doTest(report, baseline, problemsCount(report), 1, 0, INCLUDE_ABSENT); assertEquals(Result.BaselineState.ABSENT, newResult.getBaselineState()); } @@ -70,11 +71,11 @@ public void testCompareWithOneAbsentInBaseline() throws IOException { Result newResult = new Result(new Message().withText("new result")); baseline.getRuns().get(0).getResults().add(newResult); - BaselineCalculation.compare(report, baseline, new BaselineCalculation.Options(true)); + BaselineCalculation.compare(report, baseline, INCLUDE_ABSENT); SarifReport newReport = readReport(); - doTest(newReport, report, problemsCount(newReport), 0, 0, new BaselineCalculation.Options(true)); + doTest(newReport, report, problemsCount(newReport), 0, 0, INCLUDE_ABSENT); } @Test @@ -86,11 +87,11 @@ public void testCompareWithOneAbsentInReport() throws IOException { baseline.getRuns().get(0).getResults().add(newResult); - doTest(report, baseline, 0, problemsCount(baseline), 0, new BaselineCalculation.Options(true)); + doTest(report, baseline, 0, problemsCount(baseline), 0, INCLUDE_ABSENT); SarifReport newBaseline = SarifUtil.emptyReport(toolName); - doTest(report, newBaseline, 0, 0, 0, new BaselineCalculation.Options(true)); + doTest(report, newBaseline, 0, 0, 0, INCLUDE_ABSENT); } @Test @@ -170,7 +171,7 @@ public void testDifferentToolName() throws IOException { SarifReport report = readReport(); SarifReport baseline = readReport(QODANA_REPORT_JSON_2); int problemsCount = problemsCount(report); - doTest(report, baseline, problemsCount, 1, 0, new BaselineCalculation.Options(true)); + doTest(report, baseline, problemsCount, 1, 0, INCLUDE_ABSENT); //assertEquals(Result.BaselineState.NEW, newResult.getBaselineState()); } @@ -179,7 +180,7 @@ public void testAbsentResultWithChangedIdAndSameVersion() throws IOException { SarifReport report = readReport("src/test/resources/testData/AbsentBaselineTest/report.json"); SarifReport baseline = readReport("src/test/resources/testData/AbsentBaselineTest/baseline.json"); - doTest(report, baseline, 1, 17, 18, new BaselineCalculation.Options(true)); + doTest(report, baseline, 1, 17, 18, INCLUDE_ABSENT); Set knownDescriptorIds = RuleUtil.allRules(report) .map(ReportingDescriptor::getId) @@ -200,7 +201,7 @@ public void testAbsentResultWithChangedIdAndOldVersion() throws IOException { SarifReport report = readReport("src/test/resources/testData/AbsentBaselineTest/report.json"); SarifReport baseline = readReport("src/test/resources/testData/AbsentBaselineTest/baseline_old.json"); - doTest(report, baseline, 0, 18, 19, new BaselineCalculation.Options(true)); + doTest(report, baseline, 0, 18, 19, INCLUDE_ABSENT); Set knownDescriptorIds = RuleUtil.allRules(report) .map(ReportingDescriptor::getId) diff --git a/sarif/src/test/resources/testData/fingerprinting/after.sarif_v1.json b/sarif/src/test/resources/testData/fingerprinting/after.sarif_v1.json new file mode 100644 index 0000000..13746c7 --- /dev/null +++ b/sarif/src/test/resources/testData/fingerprinting/after.sarif_v1.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:191b629f725ffd6f059b46e7ad0431cda76e19955b7b6dd34384878d2c853042 +size 6098343 diff --git a/sarif/src/test/resources/testData/fingerprinting/after.sarif_v1_v2.json b/sarif/src/test/resources/testData/fingerprinting/after.sarif_v1_v2.json new file mode 100644 index 0000000..695cdaa --- /dev/null +++ b/sarif/src/test/resources/testData/fingerprinting/after.sarif_v1_v2.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7afcb59078ce31864a39aa0379430e1d3af8cfa5d61f8b174f9f1abffbb2e146 +size 6099296 diff --git a/sarif/src/test/resources/testData/fingerprinting/after.sarif_v2.json b/sarif/src/test/resources/testData/fingerprinting/after.sarif_v2.json new file mode 100644 index 0000000..aed0ba0 --- /dev/null +++ b/sarif/src/test/resources/testData/fingerprinting/after.sarif_v2.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0b6583671bb7efef2dcceade0efc93c36be49f00ef6ea163876d08841aab05a4 +size 6097479 diff --git a/sarif/src/test/resources/testData/fingerprinting/before.sarif_v1.json b/sarif/src/test/resources/testData/fingerprinting/before.sarif_v1.json new file mode 100644 index 0000000..47de6f0 --- /dev/null +++ b/sarif/src/test/resources/testData/fingerprinting/before.sarif_v1.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1a8c26e5027c770ad13161629ab7c9130844a3a2c0807f139b6b7cd501c38e46 +size 6098343 diff --git a/sarif/src/test/resources/testData/fingerprinting/before.sarif_v1_v2.json b/sarif/src/test/resources/testData/fingerprinting/before.sarif_v1_v2.json new file mode 100644 index 0000000..fd36376 --- /dev/null +++ b/sarif/src/test/resources/testData/fingerprinting/before.sarif_v1_v2.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7bf6890a701d15f0f4a914d7c79aa19ed2247dfb77870d4ff564c215cec070f3 +size 6099296 diff --git a/sarif/src/test/resources/testData/fingerprinting/before.sarif_v2.json b/sarif/src/test/resources/testData/fingerprinting/before.sarif_v2.json new file mode 100644 index 0000000..bad1254 --- /dev/null +++ b/sarif/src/test/resources/testData/fingerprinting/before.sarif_v2.json @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8597fa71ce50e5ce71139d753ef60ab9f78d706c24e091a171a7d13ca1077b4e +size 6097479