Run conformance tests on samples

eisop · Jul 10, 2024 · 503ee7a · 503ee7a
1 parent df6d156
commit 503ee7a
Show file tree

Hide file tree

Showing 3 changed files with 600 additions and 0 deletions.
diff --git a/build.gradle b/build.gradle
@@ -113,6 +113,16 @@ test {
     ])
   }
 
+  // Conformance tests on samples
+  inputs.files("${projectDir}/src/test/resources/jspecify-conformance-test-on-samples-report.txt")
+
+  doFirst {
+    systemProperties([
+      "ConformanceTest.samples.inputs": "${buildDir}/jspecify-conformance-tests/samples",
+      "ConformanceTest.samples.report": "${projectDir}/src/test/resources/jspecify-conformance-test-on-samples-report.txt",
+    ])
+  }
+
   testLogging {
     showStackTraces = false
     showStandardStreams = true

diff --git a/src/test/java/conformance/NullnessJSpecifyConformanceTest.java b/src/test/java/conformance/NullnessJSpecifyConformanceTest.java
@@ -31,6 +31,12 @@ public final class NullnessJSpecifyConformanceTest {
   /** Location of the report. */
   private final Path reportPath;
 
+  /** Directory of the JSpecify Conformance Tests for samples. */
+  private final Path testDirSamples;
+
+  /** Location of the report for samples. */
+  private final Path reportPathSamples;
+
   /** JSpecify conformance test dependencies. */
   private final ImmutableList<Path> deps;
 
@@ -42,6 +48,8 @@ public final class NullnessJSpecifyConformanceTest {
   public NullnessJSpecifyConformanceTest() {
     this.testDir = getSystemPropertyPath("ConformanceTest.inputs");
     this.reportPath = getSystemPropertyPath("ConformanceTest.report");
+    this.testDirSamples = getSystemPropertyPath("ConformanceTest.samples.inputs");
+    this.reportPathSamples = getSystemPropertyPath("ConformanceTest.samples.report");
     this.deps =
         Splitter.on(":").splitToList(System.getProperty("ConformanceTest.deps")).stream()
             .map(dep -> Paths.get(dep))
@@ -69,6 +77,14 @@ public void conformanceTests() throws IOException {
     runner.checkConformance(testDir, deps, reportPath);
   }
 
+  /** Run the conformance tests on the samples. */
+  @Test
+  public void conformanceTestsOnSamples() throws IOException {
+    ConformanceTestRunner runner =
+        new ConformanceTestRunner(NullnessJSpecifyConformanceTest::analyze);
+    runner.checkConformance(testDirSamples, deps, reportPathSamples);
+  }
+
   /**
    * Analyze the conformance tests by comparing reported facts against expected facts.
    *