feat(db): add database version management system

.db-versions.yml

@@ -0,0 +1,4 @@
+current_version: 0
+versions:
+  - version: 0
+    pr: 372

.github/labels.yml

@@ -2,6 +2,9 @@
 - name: "breaking-change"
   color: ee0701
   description: "A change that changes the API or breaks backward compatibility for users."
+- name: "bump_db"
+  color: ee0701
+  description: "Changes requiring a database version increment."
 - name: "bugfix"
   color: ee0701
   description:

.github/workflows/db-version.yml

@@ -0,0 +1,45 @@
+---
+name: DB Version Management
+
+on:
+  workflow_dispatch:
+  workflow_call:
+
+jobs:
+  update-db-version:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Install yq
+        run: sudo apt-get install -y yq
+
+      - name: Check if PR already bumped
+        id: check_bump
+        run: |
+          PR_NUM="${{ github.event.pull_request.number }}"
+          if yq -e ".versions[] | select(.pr == ${PR_NUM})" .db-versions.yml > /dev/null 2>&1; then
+            echo "already_bumped=true" >> $GITHUB_OUTPUT
+          else
+            echo "already_bumped=false" >> $GITHUB_OUTPUT
+          fi
+
+      - name: Configure Git
+        if: steps.check_bump.outputs.already_bumped == 'false'
+        run: |
+          git config user.name 'github-actions[bot]'
+          git config user.email 'github-actions[bot]@users.noreply.github.com'
+
+      - name: Update DB Version
+        if: steps.check_bump.outputs.already_bumped == 'false'
+        run: |
+          ./scripts/update-db-version.sh "${{ github.event.pull_request.number }}"
+
+      - name: Commit and Push
+        if: steps.check_bump.outputs.already_bumped == 'false'
+        run: |
+          if [[ -n "$(git status --porcelain)" ]]; then
+            git add .db-versions.toml
+            git commit -m "chore: bump db version"
+            git push origin HEAD
+          fi

.github/workflows/pull-request.yml

@@ -19,6 +19,12 @@ jobs:
     if: github.event.pull_request.draft == false
     uses: ./.github/workflows/changelog.yml
 
+  update_db_version:
+    name: Update DB Version
+    if: contains(github.event.pull_request.labels.*.name, 'bump_db') && github.event.pull_request.draft == false
+    uses: ./.github/workflows/db-version.yml
+    needs: changelog
+
   linters:
     name: Run linters
     if: github.event.pull_request.draft == false

CHANGELOG.md

@@ -2,6 +2,7 @@
 
 ## Next release
 
+- feat(db): add database version management system
 - fix: oracle need condition
 - fix(block_production): continue pending block now reexecutes the previous transactions
 - feat(services): reworked Madara services for better cancellation control

crates/client/db/build.rs

@@ -0,0 +1,114 @@
+use std::borrow::Cow;
+use std::env;
+use std::fs;
+use std::path::{Path, PathBuf};
+
+const DB_VERSION_FILE: &str = ".db-versions.yml";
+const PARENT_LEVELS: usize = 3;
+
+#[allow(clippy::print_stderr)]
+fn main() {
+    if let Err(e) = get_db_version() {
+        eprintln!("Failed to get DB version: {}", e);
+        std::process::exit(1);
+    }
+}
+
+#[derive(Debug)]
+enum BuildError {
+    EnvVar(env::VarError),
+    Io(std::io::Error),
+    Parse(Cow<'static, str>),
+}
+
+impl std::fmt::Display for BuildError {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        match self {
+            BuildError::EnvVar(e) => write!(f, "Environment variable error: {}", e),
+            BuildError::Io(e) => write!(f, "IO error: {}", e),
+            BuildError::Parse(msg) => write!(f, "Parse error: {}", msg),
+        }
+    }
+}
+
+impl From<env::VarError> for BuildError {
+    fn from(e: env::VarError) -> Self {
+        BuildError::EnvVar(e)
+    }
+}
+
+impl From<std::io::Error> for BuildError {
+    fn from(e: std::io::Error) -> Self {
+        BuildError::Io(e)
+    }
+}
+
+fn get_db_version() -> Result<(), BuildError> {
+    let manifest_dir = env::var("CARGO_MANIFEST_DIR")?;
+    let root_dir = get_parents(&PathBuf::from(manifest_dir), PARENT_LEVELS)?;
+    let file_path = root_dir.join(DB_VERSION_FILE);
+
+    let content = fs::read_to_string(&file_path).map_err(|e| {
+        BuildError::Io(std::io::Error::new(e.kind(), format!("Failed to read {}: {}", file_path.display(), e)))
+    })?;
+
+    let current_version = parse_version(&content)?;
+
+    println!("cargo:rerun-if-changed={}", DB_VERSION_FILE);
+    println!("cargo:rustc-env=DB_VERSION={}", current_version);
+
+    Ok(())
+}
+
+fn parse_version(content: &str) -> Result<u32, BuildError> {
+    content
+        .lines()
+        .find(|line| line.starts_with("current_version:"))
+        .ok_or_else(|| BuildError::Parse(Cow::Borrowed("Could not find current_version")))?
+        .split(':')
+        .nth(1)
+        .ok_or_else(|| BuildError::Parse(Cow::Borrowed("Invalid current_version format")))?
+        .trim()
+        .parse()
+        .map_err(|_| BuildError::Parse(Cow::Borrowed("Could not parse current_version as u32")))
+}
+
+fn get_parents(path: &Path, n: usize) -> Result<PathBuf, BuildError> {
+    let mut path = path.to_path_buf();
+    for _ in 0..n {
+        path = path
+            .parent()
+            .ok_or(BuildError::Io(std::io::Error::new(std::io::ErrorKind::NotFound, "Parent not found")))?
+            .to_path_buf();
+    }
+    Ok(path)
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use std::fs;
+    use tempfile::TempDir;
+
+    #[test]
+    fn test_parse_version_valid() {
+        let content = "current_version: 42\nother: stuff";
+        assert_eq!(parse_version(content).unwrap(), 42);
+    }
+
+    #[test]
+    fn test_parse_version_invalid_format() {
+        let content = "wrong_format";
+        assert!(matches!(parse_version(content), Err(BuildError::Parse(_))));
+    }
+
+    #[test]
+    fn test_get_parents() {
+        let temp = TempDir::new().unwrap();
+        let path = temp.path().join("a").join("b").join("c");
+        fs::create_dir_all(&path).unwrap();
+
+        let result = get_parents(&path, 2).unwrap();
+        assert_eq!(result, temp.path().join("a"));
+    }
+}

crates/client/db/src/db_version.rs

@@ -0,0 +1,40 @@
+use std::path::Path;
+
+const REQUIRED_DB_VERSION: &str = env!("DB_VERSION");
+
+#[derive(Debug, thiserror::Error)]
+pub enum DbVersionError {
+    #[error(
+        "Database version {db_version} is not compatible with current binary. Expected version {required_version}"
+    )]
+    IncompatibleVersion { db_version: u32, required_version: u32 },
+    #[error("Failed to read database version: {0}")]
+    VersionReadError(String),
+}
+
+pub fn check_db_version(path: &Path) -> Result<Option<u32>, DbVersionError> {
+    let required_db_version =
+        REQUIRED_DB_VERSION.parse::<u32>().expect("REQUIRED_DB_VERSION is checked at compile time");
+
+    if !path.exists() {
+        std::fs::create_dir_all(path).map_err(|e| DbVersionError::VersionReadError(e.to_string()))?;
+    }
+
+    let file_path = path.join(".db-version");
+    if !file_path.exists() {
+        // No version file, create it with the current version
+        std::fs::write(file_path, REQUIRED_DB_VERSION).map_err(|e| DbVersionError::VersionReadError(e.to_string()))?;
+        Ok(None)
+    } else {
+        let version =
+            std::fs::read_to_string(file_path).map_err(|e| DbVersionError::VersionReadError(e.to_string()))?;
+        let version = version.parse::<u32>().map_err(|_| DbVersionError::VersionReadError(version))?;
+        if version != required_db_version {
+            return Err(DbVersionError::IncompatibleVersion {
+                db_version: version,
+                required_version: required_db_version,
+            });
+        }
+        Ok(Some(version))
+    }
+}

crates/client/db/src/lib.rs

@@ -19,6 +19,7 @@ use std::sync::Arc;
 use std::{fmt, fs};
 use tokio::sync::{mpsc, oneshot};
 
+mod db_version;
 mod error;
 mod rocksdb_options;
 mod rocksdb_snapshot;
@@ -398,6 +399,12 @@ impl MadaraBackend {
         chain_config: Arc<ChainConfig>,
         trie_log_config: TrieLogConfig,
     ) -> anyhow::Result<Arc<MadaraBackend>> {
+        // check if the db version is compatible with the current binary
+        tracing::debug!("checking db version");
+        if let Some(db_version) = db_version::check_db_version(&db_config_dir).context("Checking database version")? {
+            tracing::debug!("version of existing db is {db_version}");
+        }
+
         let db_path = db_config_dir.join("db");
 
         // when backups are enabled, a thread is spawned that owns the rocksdb BackupEngine (it is not thread safe) and it receives backup requests using a mpsc channel

scripts/update-db-version.sh

@@ -0,0 +1,31 @@
+#!/bin/sh
+FILE=".db-versions.yml"
+if [ $# -eq 0 ]; then
+    echo "Usage: $0 PR_NUMBER" 
+    exit 1
+fi
+set -euo pipefail
+
+PR_NUMBER="$1"
+
+# Check if the file exists
+if [ ! -f "$FILE" ]; then
+    echo "Error: $FILE not found"
+    exit 1
+fi
+
+# Read and validate the current version
+CURRENT_VERSION=$(yq '.current_version' "$FILE")
+if ! [[ "$CURRENT_VERSION" =~ ^[0-9]+$ ]]; then
+    echo "Error: Failed to read current_version from $FILE"
+    exit 1
+fi
+
+# Increment the version
+NEW_VERSION=$((CURRENT_VERSION + 1))
+
+# Update the file
+yq -i ".current_version = $NEW_VERSION" "$FILE"
+yq -i ".versions = [{ \"version\": $NEW_VERSION, \"pr\": $PR_NUMBER }] + .versions" "$FILE"
+
+echo "Successfully updated DB version to ${NEW_VERSION} (PR #${PR_NUMBER})"