change(db): Make the first stable release forward-compatible with planned state changes

Cargo.lock

@@ -5960,6 +5960,7 @@ dependencies = [
  "regex",
  "rlimit",
  "rocksdb",
+ "semver 1.0.17",
  "serde",
  "serde_json",
  "spandoc",

zebra-state/Cargo.toml

@@ -46,6 +46,7 @@ mset = "0.1.1"
 regex = "1.8.3"
 rlimit = "0.9.1"
 rocksdb = { version = "0.21.0", default_features = false, features = ["lz4"] }
+semver = "1.0.17"
 serde = { version = "1.0.163", features = ["serde_derive"] }
 tempfile = "3.5.0"
 thiserror = "1.0.40"

zebra-state/src/config.rs

@@ -1,16 +1,26 @@
 //! Cached state configuration for Zebra.
 
 use std::{
-    fs::{canonicalize, remove_dir_all, DirEntry, ReadDir},
+    fs::{self, canonicalize, remove_dir_all, DirEntry, ReadDir},
+    io::ErrorKind,
     path::{Path, PathBuf},
 };
 
+use semver::Version;
 use serde::{Deserialize, Serialize};
 use tokio::task::{spawn_blocking, JoinHandle};
 use tracing::Span;
 
 use zebra_chain::parameters::Network;
 
+use crate::{
+    constants::{
+        DATABASE_FORMAT_MINOR_VERSION, DATABASE_FORMAT_PATCH_VERSION, DATABASE_FORMAT_VERSION,
+        DATABASE_FORMAT_VERSION_FILE_NAME,
+    },
+    BoxError,
+};
+
 /// Configuration for the state service.
 #[derive(Clone, Debug, Deserialize, Serialize)]
 #[serde(deny_unknown_fields, default)]
@@ -119,6 +129,15 @@ impl Config {
         }
     }
 
+    /// Returns the path of the database format version file.
+    pub fn version_file_path(&self, network: Network) -> PathBuf {
+        let mut version_path = self.db_path(network);
+
+        version_path.push(DATABASE_FORMAT_VERSION_FILE_NAME);
+
+        version_path
+    }
+
     /// Construct a config for an ephemeral database
     pub fn ephemeral() -> Config {
         Config {
@@ -261,8 +280,83 @@ fn parse_dir_name(entry: &DirEntry) -> Option<String> {
 /// Parse the state version number from `dir_name`.
 ///
 /// Returns `None` if parsing fails, or the directory name is not in the expected format.
-fn parse_version_number(dir_name: &str) -> Option<u32> {
+fn parse_version_number(dir_name: &str) -> Option<u64> {
     dir_name
         .strip_prefix('v')
         .and_then(|version| version.parse().ok())
 }
+
+/// Returns the full semantic version of the currently running database format code.
+///
+/// This is the version implemented by the Zebra code that's currently running,
+/// the minor and patch versions on disk can be different.
+pub fn database_format_version_in_code() -> Version {
+    Version::new(
+        DATABASE_FORMAT_VERSION,
+        DATABASE_FORMAT_MINOR_VERSION,
+        DATABASE_FORMAT_PATCH_VERSION,
+    )
+}
+
+/// Returns the full semantic version of the on-disk database.
+/// If there is no existing on-disk database, returns `Ok(None)`.
+///
+/// This is the format of the data on disk, the minor and patch versions
+/// implemented by the running Zebra code can be different.
+pub fn database_format_version_on_disk(
+    config: &Config,
+    network: Network,
+) -> Result<Option<Version>, BoxError> {
+    let version_path = config.version_file_path(network);
+
+    let version = match fs::read_to_string(version_path) {
+        Ok(version) => version,
+        Err(e) if e.kind() == ErrorKind::NotFound => {
+            // If the version file doesn't exist, don't guess the version.
+            // (It will end up being the version in code, once the database is created.)
+            return Ok(None);
+        }
+        Err(e) => Err(e)?,
+    };
+
+    let (minor, patch) = version
+        .split_once('.')
+        .ok_or("invalid database format version file")?;
+
+    Ok(Some(Version::new(
+        DATABASE_FORMAT_VERSION,
+        minor.parse()?,
+        patch.parse()?,
+    )))
+}
+
+/// Writes the currently running semantic database version to the on-disk database.
+///
+/// # Correctness
+///
+/// This should only be called after all running format upgrades are complete.
+///
+/// # Concurrency
+///
+/// This must only be called while RocksDB has an open database for `config`.
+/// Otherwise, multiple Zebra processes could write the version at the same time,
+/// corrupting the file.
+pub fn write_database_format_version_to_disk(
+    config: &Config,
+    network: Network,
+) -> Result<(), BoxError> {
+    let version_path = config.version_file_path(network);
+
+    // The major version is already in the directory path.
+    let version = format!(
+        "{}.{}",
+        DATABASE_FORMAT_MINOR_VERSION, DATABASE_FORMAT_PATCH_VERSION
+    );
+
+    // # Concurrency
+    //
+    // The caller handles locking for this file write.
+    fs::write(version_path, version.as_bytes())?;
+
+    Ok(())
+}

zebra-state/src/constants.rs

@@ -1,4 +1,11 @@
-//! Definitions of constants.
+//! Constants that impact state behaviour.
+
+use lazy_static::lazy_static;
+use regex::Regex;
+
+// For doc comment links
+#[allow(unused_imports)]
+use crate::config::{database_format_version_in_code, database_format_version_on_disk};
 
 pub use zebra_chain::transparent::MIN_TRANSPARENT_COINBASE_MATURITY;
 
@@ -19,13 +26,42 @@ pub use zebra_chain::transparent::MIN_TRANSPARENT_COINBASE_MATURITY;
 // TODO: change to HeightDiff
 pub const MAX_BLOCK_REORG_HEIGHT: u32 = MIN_TRANSPARENT_COINBASE_MATURITY - 1;
 
-/// The database format version, incremented each time the database format changes.
-pub const DATABASE_FORMAT_VERSION: u32 = 25;
+/// The database format major version, incremented each time the on-disk database format has a
+/// breaking data format change.
+///
+/// Breaking changes include:
+/// - deleting a column family, or
+/// - changing a column family's data format in an incompatible way.
+///
+/// Breaking changes become minor version changes if:
+/// - we previously added compatibility code, and
+/// - it's available in all supported Zebra versions.
+///
+/// Use [`database_format_version_in_code()`] or [`database_format_version_on_disk()`]
+/// to get the full semantic format version.
+pub const DATABASE_FORMAT_VERSION: u64 = 25;
+
+/// The database format minor version, incremented each time the on-disk database format has a
+/// significant data format change.
+///
+/// Significant changes include:
+/// - adding new column families,
+/// - changing the format of a column family in a compatible way, or
+/// - breaking changes with compatibility code in all supported Zebra versions.
+pub const DATABASE_FORMAT_MINOR_VERSION: u64 = 0;
+
+/// The database format patch version, incremented each time the on-disk database format has a
+/// significant format compatibility fix.
+pub const DATABASE_FORMAT_PATCH_VERSION: u64 = 1;
+
+/// The name of the file containing the minor and patch database versions.
+pub const DATABASE_FORMAT_VERSION_FILE_NAME: &str = "version";
 
 /// The maximum number of blocks to check for NU5 transactions,
 /// before we assume we are on a pre-NU5 legacy chain.
 ///
-/// Zebra usually only has to check back a few blocks, but on testnet it can be a long time between v5 transactions.
+/// Zebra usually only has to check back a few blocks on mainnet, but on testnet it can be a long
+/// time between v5 transactions.
 pub const MAX_LEGACY_CHAIN_BLOCKS: usize = 100_000;
 
 /// The maximum number of non-finalized chain forks Zebra will track.
@@ -58,9 +94,6 @@ const MAX_FIND_BLOCK_HEADERS_RESULTS_FOR_PROTOCOL: u32 = 160;
 pub const MAX_FIND_BLOCK_HEADERS_RESULTS_FOR_ZEBRA: u32 =
     MAX_FIND_BLOCK_HEADERS_RESULTS_FOR_PROTOCOL - 2;
 
-use lazy_static::lazy_static;
-use regex::Regex;
-
 lazy_static! {
     /// Regex that matches the RocksDB error when its lock file is already open.
     pub static ref LOCK_FILE_ERROR: Regex = Regex::new("(lock file).*(temporarily unavailable)|(in use)|(being used by another process)").expect("regex is valid");