feat: check for test regression

Signed-off-by: nerodesu017 <[email protected]>

.github/workflows/ci.yml

@@ -34,7 +34,7 @@ jobs:
       - name: Build
         run: cargo build --verbose
       - name: Run tests
-        run: cargo test --verbose -- --nocapture
+        run: cargo test
 
   conventional_commit_check:
     name: Conventional Commits

.github/workflows/rust-test-runner.yml

@@ -0,0 +1,56 @@
+name: Rust Testsuite CI
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+    branches:
+      - main
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout main branch
+        uses: actions/checkout@v4
+        with:
+          submodules: true
+
+      - name: Set up Rust
+        uses: actions-rs/toolchain@v1
+        with:
+          toolchain: stable
+          override: true
+
+      - name: Checkout gh-pages branch
+        uses: actions/checkout@v4
+        with:
+          ref: gh-pages
+          path: gh-pages
+
+      # - name: Fetch previous testsuite_results.json
+      #   run: |
+      #     cp gh-pages/main/testsuite_results.json testsuite_results_original.json
+
+      - name: Run tests
+        run: cargo test --package wasm-interpreter --test wasm_spec_testsuite --verbose -- --nocapture
+
+      - name: Save testsuite_results.json to gh-pages
+        if: github.ref == 'refs/heads/main'
+        run: |
+          cp testsuite_results.json gh-pages/main/testsuite_results.json
+
+      - name: Commit and push results to gh-pages
+        # if: github.ref == 'refs/heads/main'
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          cd gh-pages
+          git config --global user.name "github-actions[bot]"
+          git config --global user.email "github-actions[bot]@users.noreply.github.com"
+          git add main/testsuite_results.json
+          git commit -m "Update test suite results" || echo "No changes to commit"
+          git pull --rebase origin gh-pages
+          git push origin gh-pages

Cargo.toml

@@ -31,6 +31,8 @@ wat = "1.0.83"
 wast = "212.0.0"
 criterion = { version = "0.5.1", features = ["html_reports"] }
 hexf = "0.2.1"
+serde_json = "1.0.138"
+serde = "1.0.217"
 
 [features]
 default = ["hooks"]

tests/specification/mod.rs

@@ -1,9 +1,12 @@
 use std::{
+    collections::HashMap,
     fs,
+    io::Write,
     path::{Path, PathBuf},
 };
 
 use reports::WastTestReport;
+use serde::{Deserialize, Serialize};
 
 mod reports;
 mod run;
@@ -107,7 +110,7 @@ pub fn spec_tests() {
 
     let mut final_status: String = String::new();
     // Printing success rate per file for those that did NOT error out when compiling
-    for report in no_compile_errors_reports {
+    for report in &no_compile_errors_reports {
         final_status += format!(
             "Report for {:filename_width$}: Tests: {:passed_width$} Passed, {:failed_width$} Failed --- {:percentage_width$.2}%\n",
             report.filename,
@@ -146,6 +149,123 @@ pub fn spec_tests() {
         "Tests: {} Passed, {} Failed, {} Compilation Errors",
         successful_reports, failed_reports, compile_error_reports
     );
+
+    let result = execute_ci_instructions(
+        "testsuite_results",
+        TestResults {
+            total: total_mini_tests,
+            successful: successful_mini_tests,
+            failed: total_mini_tests - successful_mini_tests,
+            tests: no_compile_errors_reports
+                .iter()
+                .map(|el| {
+                    (
+                        el.filename.clone(),
+                        TestResult {
+                            failed: el.failed,
+                            successful: el.successful,
+                            total: el.total,
+                        },
+                    )
+                })
+                .collect::<HashMap<String, TestResult>>(),
+        },
+    );
+
+    match result {
+        Err(e) => {
+            println!("\n\n\n{}\n\n\n", e);
+            std::process::exit(-1);
+        }
+        Ok(..) => {}
+    };
+}
+
+#[derive(Deserialize, Serialize)]
+struct TestResult {
+    // name: String,
+    total: usize,
+    successful: usize,
+    failed: usize,
+}
+
+#[derive(Deserialize, Serialize)]
+struct TestResults {
+    total: usize,
+    successful: usize,
+    failed: usize,
+    tests: HashMap<String, TestResult>,
+    // tests: Vec<TestResult>,
+}
+
+fn execute_ci_instructions(file_name: &str, results: TestResults) -> Result<(), String> {
+    let original_file = std::fs::File::open(String::from(file_name) + "_original.json").ok();
+
+    let mut file = std::fs::OpenOptions::new()
+        .write(true)
+        .create(true)
+        .truncate(true) // This ensures the file is truncated if it exists
+        .open(String::from(file_name) + ".json")
+        .unwrap();
+
+    let stringified: String = serde_json::to_string(&results).unwrap();
+
+    file.write(stringified.as_bytes()).unwrap();
+
+    match original_file {
+        None => Ok(()),
+        Some(original_file) => {
+            let parsed_original_file: TestResults =
+                serde_json::from_reader(original_file).map_err(|e| e.to_string())?;
+
+            diff(parsed_original_file, results)
+        }
+    }
+}
+
+fn diff(original: TestResults, current: TestResults) -> Result<(), String> {
+    if original.total > current.total {
+        return Err(format!(
+            "Fewer total tests in the current run ({}) than in the original ({})",
+            current.total, original.total
+        ));
+    }
+    if original.successful > current.successful {
+        return Err(format!(
+            "Fewer successful tests in the current run ({}) than in the original ({})",
+            current.successful, original.successful
+        ));
+    }
+
+    for (original_test_name, original_test) in original.tests {
+        let current_test = current
+            .tests
+            .iter()
+            .find(|test| *test.0 == original_test_name);
+
+        match current_test {
+            None => return Err(format!("Test with name '{}' not found", original_test_name)),
+            Some(current_test) => {
+                let current_test = current_test.1;
+
+                if original_test.total > current_test.total {
+                    return Err(format!(
+                        "TEST '{}': Fewer total tests ({}) than in the original ({})",
+                        original_test_name, current_test.total, original_test.total
+                    ));
+                }
+
+                if original_test.successful > current_test.successful {
+                    return Err(format!(
+                        "TEST '{}': Fewer successful tests ({}) than in the original ({})",
+                        original_test_name, current_test.successful, original_test.successful
+                    ));
+                }
+            }
+        }
+    }
+
+    Ok(())
 }
 
 #[allow(dead_code)]