diff --git a/buildpacks/ruby/src/layers.rs b/buildpacks/ruby/src/layers.rs index 4c00f500..1037d54a 100644 --- a/buildpacks/ruby/src/layers.rs +++ b/buildpacks/ruby/src/layers.rs @@ -2,3 +2,4 @@ pub(crate) mod bundle_download_layer; pub(crate) mod bundle_install_layer; pub(crate) mod metrics_agent_install; pub(crate) mod ruby_install_layer; +mod shared; diff --git a/buildpacks/ruby/src/layers/ruby_install_layer.rs b/buildpacks/ruby/src/layers/ruby_install_layer.rs index 7f9c0851..e0444a67 100644 --- a/buildpacks/ruby/src/layers/ruby_install_layer.rs +++ b/buildpacks/ruby/src/layers/ruby_install_layer.rs @@ -11,22 +11,19 @@ //! //! When the Ruby version changes, invalidate and re-run. //! -use bullet_stream::state::SubBullet; -use bullet_stream::{style, Print}; -use commons::display::SentenceList; -use libcnb::data::layer_name; -use libcnb::layer::{ - CachedLayerDefinition, EmptyLayerCause, InvalidMetadataAction, LayerState, RestoredLayerAction, -}; -use libcnb::layer_env::LayerEnv; -use magic_migrate::{try_migrate_deserializer_chain, TryMigrate}; - +use crate::layers::shared::{cached_layer_write_metadata, MetadataDiff}; use crate::{ target_id::{TargetId, TargetIdError}, RubyBuildpack, RubyBuildpackError, }; +use bullet_stream::state::SubBullet; +use bullet_stream::{style, Print}; use commons::gemfile_lock::ResolvedRubyVersion; use flate2::read::GzDecoder; +use libcnb::data::layer_name; +use libcnb::layer::{EmptyLayerCause, LayerState}; +use libcnb::layer_env::LayerEnv; +use magic_migrate::{try_migrate_deserializer_chain, TryMigrate}; use serde::{Deserialize, Deserializer, Serialize}; use std::convert::Infallible; use std::io::{self, Stdout}; @@ -38,67 +35,26 @@ use url::Url; pub(crate) fn handle( context: &libcnb::build::BuildContext, mut bullet: Print>, - metadata: Metadata, + metadata: &Metadata, ) -> libcnb::Result<(Print>, LayerEnv), RubyBuildpackError> { - let layer_ref = context.cached_layer( - layer_name!("ruby"), - CachedLayerDefinition { - build: true, - launch: true, - invalid_metadata_action: &|old| match Metadata::try_from_str_migrations( - &toml::to_string(old).expect("TOML deserialization of GenericMetadata"), - ) { - Some(Ok(migrated)) => ( - InvalidMetadataAction::ReplaceMetadata(migrated), - "replaced metadata".to_string(), - ), - Some(Err(error)) => ( - InvalidMetadataAction::DeleteLayer, - format!("metadata migration error {error}"), - ), - None => ( - InvalidMetadataAction::DeleteLayer, - "invalid metadata".to_string(), - ), - }, - restored_layer_action: &|old: &Metadata, _| { - let diff = metadata_diff(old, &metadata); - if diff.is_empty() { - ( - RestoredLayerAction::KeepLayer, - "using cached version".to_string(), - ) - } else { - ( - RestoredLayerAction::DeleteLayer, - format!( - "due to {changes}: {differences}", - changes = if diff.len() > 1 { "changes" } else { "change" }, - differences = SentenceList::new(&diff) - ), - ) - } - }, - }, - )?; + let layer_ref = cached_layer_write_metadata(layer_name!("ruby"), context, metadata)?; match &layer_ref.state { - LayerState::Restored { cause: _ } => { - bullet = bullet.sub_bullet("Using cached Ruby version"); + LayerState::Restored { cause } => { + bullet = bullet.sub_bullet(cause); } LayerState::Empty { cause } => { match cause { EmptyLayerCause::NewlyCreated => {} EmptyLayerCause::InvalidMetadataAction { cause } | EmptyLayerCause::RestoredLayerAction { cause } => { - bullet = bullet.sub_bullet(format!("Clearing cache {cause}")); + bullet = bullet.sub_bullet(cause); } } let timer = bullet.start_timer("Installing"); - install_ruby(&metadata, &layer_ref.path())?; + install_ruby(metadata, &layer_ref.path())?; bullet = timer.done(); } } - layer_ref.write_metadata(metadata)?; Ok((bullet, layer_ref.read_env()?)) } @@ -170,44 +126,39 @@ impl TryFrom for MetadataV2 { } } -fn metadata_diff(old: &Metadata, metadata: &Metadata) -> Vec { - let mut differences = Vec::new(); - let Metadata { - distro_name, - distro_version, - cpu_architecture, - ruby_version, - } = old; - if ruby_version != &metadata.ruby_version { - differences.push(format!( - "Ruby version ({old} to {now})", - old = style::value(ruby_version.to_string()), - now = style::value(metadata.ruby_version.to_string()) - )); - } - if distro_name != &metadata.distro_name { - differences.push(format!( - "distro name ({old} to {now})", - old = style::value(distro_name), - now = style::value(&metadata.distro_name) - )); - } - if distro_version != &metadata.distro_version { - differences.push(format!( - "distro version ({old} to {now})", - old = style::value(distro_version), - now = style::value(&metadata.distro_version) - )); - } - if cpu_architecture != &metadata.cpu_architecture { - differences.push(format!( - "CPU architecture ({old} to {now})", - old = style::value(cpu_architecture), - now = style::value(&metadata.cpu_architecture) - )); - } +impl MetadataDiff for Metadata { + fn diff(&self, old: &Self) -> Vec { + let mut differences = Vec::new(); + let Metadata { + distro_name, + distro_version, + cpu_architecture, + ruby_version, + } = old; + if ruby_version != &self.ruby_version { + differences.push(format!( + "Ruby version ({old} to {now})", + old = style::value(ruby_version.to_string()), + now = style::value(self.ruby_version.to_string()) + )); + } + if distro_name != &self.distro_name || distro_version != &self.distro_version { + differences.push(format!( + "Distribution ({old} to {now})", + old = style::value(format!("{distro_name} {distro_version}")), + now = style::value(format!("{} {}", self.distro_name, self.distro_version)) + )); + } + if cpu_architecture != &self.cpu_architecture { + differences.push(format!( + "CPU architecture ({old} to {now})", + old = style::value(cpu_architecture), + now = style::value(&self.cpu_architecture) + )); + } - differences + differences + } } fn download_url( @@ -291,6 +242,8 @@ pub(crate) enum RubyInstallError { #[cfg(test)] mod tests { + use crate::layers::shared::{strip_ansi, temp_build_context}; + use super::*; /// If this test fails due to a change you'll need to @@ -362,4 +315,86 @@ version = "3.1.3" "https://heroku-buildpack-ruby.s3.us-east-1.amazonaws.com/heroku-22/ruby-2.7.4.tgz", ); } + + #[test] + fn metadata_diff_messages() { + let old = Metadata { + ruby_version: ResolvedRubyVersion("3.5.3".to_string()), + distro_name: "ubuntu".to_string(), + distro_version: "20.04".to_string(), + cpu_architecture: "amd64".to_string(), + }; + assert_eq!(old.diff(&old), Vec::::new()); + + let diff = Metadata { + ruby_version: ResolvedRubyVersion("3.5.5".to_string()), + distro_name: old.distro_name.clone(), + distro_version: old.distro_version.clone(), + cpu_architecture: old.cpu_architecture.clone(), + } + .diff(&old); + assert_eq!( + diff.iter().map(strip_ansi).collect::>(), + vec!["Ruby version (`3.5.3` to `3.5.5`)".to_string()] + ); + + let diff = Metadata { + ruby_version: old.ruby_version.clone(), + distro_name: "alpine".to_string(), + distro_version: "3.20.0".to_string(), + cpu_architecture: old.cpu_architecture.clone(), + } + .diff(&old); + + assert_eq!( + diff.iter().map(strip_ansi).collect::>(), + vec!["Distribution (`ubuntu 20.04` to `alpine 3.20.0`)".to_string()] + ); + + let diff = Metadata { + ruby_version: old.ruby_version.clone(), + distro_name: old.distro_name.clone(), + distro_version: old.distro_version.clone(), + cpu_architecture: "arm64".to_string(), + } + .diff(&old); + assert_eq!( + diff.iter().map(strip_ansi).collect::>(), + vec!["CPU architecture (`amd64` to `arm64`)".to_string()] + ); + } + + #[test] + fn test_ruby_version_difference_clears_cache() { + let temp = tempfile::tempdir().unwrap(); + let context = temp_build_context::(temp.path()); + let old = Metadata { + ruby_version: ResolvedRubyVersion("2.7.2".to_string()), + distro_name: "ubuntu".to_string(), + distro_version: "20.04".to_string(), + cpu_architecture: "x86_64".to_string(), + }; + let differences = old.diff(&old); + assert_eq!(differences, Vec::::new()); + + cached_layer_write_metadata(layer_name!("ruby"), &context, &old).unwrap(); + let result = cached_layer_write_metadata(layer_name!("ruby"), &context, &old).unwrap(); + let actual = result.state; + assert!(matches!(actual, LayerState::Restored { .. })); + + let now = Metadata { + ruby_version: ResolvedRubyVersion("3.0.0".to_string()), + ..old.clone() + }; + let differences = now.diff(&old); + assert_eq!(differences.len(), 1); + + let result = cached_layer_write_metadata(layer_name!("ruby"), &context, &now).unwrap(); + assert!(matches!( + result.state, + LayerState::Empty { + cause: EmptyLayerCause::RestoredLayerAction { .. } + } + )); + } } diff --git a/buildpacks/ruby/src/layers/shared.rs b/buildpacks/ruby/src/layers/shared.rs new file mode 100644 index 00000000..48152aed --- /dev/null +++ b/buildpacks/ruby/src/layers/shared.rs @@ -0,0 +1,314 @@ +use commons::display::SentenceList; +use libcnb::build::BuildContext; +use libcnb::layer::{CachedLayerDefinition, InvalidMetadataAction, LayerRef, RestoredLayerAction}; + +/// Default behavior for a cached layer, ensures new metadata is always written +/// +/// The metadadata must implement `MetadataDiff` and `TryMigrate` in addition +/// to the typical `Serialize` and `Debug` traits +pub(crate) fn cached_layer_write_metadata( + layer_name: libcnb::data::layer::LayerName, + context: &BuildContext, + metadata: &'_ M, +) -> libcnb::Result, B::Error> +where + B: libcnb::Buildpack, + M: MetadataDiff + magic_migrate::TryMigrate + serde::ser::Serialize + std::fmt::Debug, + ::Error: std::fmt::Display, +{ + let layer_ref = context.cached_layer( + layer_name, + CachedLayerDefinition { + build: true, + launch: true, + invalid_metadata_action: &invalid_metadata_action, + restored_layer_action: &|old: &M, _| restored_layer_action(old, metadata), + }, + )?; + layer_ref.write_metadata(metadata)?; + Ok(layer_ref) +} + +/// Given another metadata object, returns a list of differences between the two +/// +/// If no differences, return an empty list +pub(crate) trait MetadataDiff { + fn diff(&self, old: &Self) -> Vec; +} + +/// Standardizes formatting for layer cache clearing behavior +/// +/// If the diff is empty, there are no changes and the layer is kept +/// If the diff is not empty, the layer is deleted and the changes are listed +pub(crate) fn restored_layer_action(old: &T, now: &T) -> (RestoredLayerAction, String) +where + T: MetadataDiff, +{ + let diff = now.diff(old); + if diff.is_empty() { + (RestoredLayerAction::KeepLayer, "Using cache".to_string()) + } else { + ( + RestoredLayerAction::DeleteLayer, + format!( + "Clearing cache due to {changes}: {differences}", + changes = if diff.len() > 1 { "changes" } else { "change" }, + differences = SentenceList::new(&diff) + ), + ) + } +} + +/// Standardizes formatting for invalid metadata behavior +/// +/// If the metadata can be migrated, it is replaced with the migrated version +/// If an error occurs, the layer is deleted and the error displayed +/// If no migration is possible, the layer is deleted and the invalid metadata is displayed +pub(crate) fn invalid_metadata_action(invalid: &S) -> (InvalidMetadataAction, String) +where + T: magic_migrate::TryMigrate, + S: serde::ser::Serialize + std::fmt::Debug, + // TODO: Enforce Display + Debug in the library + ::Error: std::fmt::Display, +{ + let invalid = toml::to_string(invalid); + match invalid { + Ok(toml) => match T::try_from_str_migrations(&toml) { + Some(Ok(migrated)) => ( + InvalidMetadataAction::ReplaceMetadata(migrated), + "Replaced metadata".to_string(), + ), + Some(Err(error)) => ( + InvalidMetadataAction::DeleteLayer, + format!("Clearing cache due to metadata migration error: {error}"), + ), + None => ( + InvalidMetadataAction::DeleteLayer, + format!( + "Clearing cache due to invalid metadata ({toml})", + toml = toml.trim() + ), + ), + }, + Err(error) => ( + InvalidMetadataAction::DeleteLayer, + format!("Clearing cache due to invalid metadata serialization error: {error}"), + ), + } +} + +/// Removes ANSI control characters from a string +#[cfg(test)] +pub(crate) fn strip_ansi(input: impl AsRef) -> String { + let re = regex::Regex::new(r"\x1b\[[0-9;]*[a-zA-Z]").expect("Clippy checked"); + re.replace_all(input.as_ref(), "").to_string() +} + +/// Takes in a directory and returns a minimal build context for use in testing shared caching behavior +/// +/// Intented only for use with this buildpack, but meant to be used by multiple layers to assert caching behavior. +#[cfg(test)] +pub(crate) fn temp_build_context( + from_dir: impl AsRef, +) -> BuildContext { + let base_dir = from_dir.as_ref().to_path_buf(); + let layers_dir = base_dir.join("layers"); + let app_dir = base_dir.join("app_dir"); + let platform_dir = base_dir.join("platform_dir"); + let buildpack_dir = base_dir.join("buildpack_dir"); + for dir in [&app_dir, &layers_dir, &buildpack_dir, &platform_dir] { + std::fs::create_dir_all(dir).unwrap(); + } + + let target = libcnb::Target { + os: String::new(), + arch: String::new(), + arch_variant: None, + distro_name: String::new(), + distro_version: String::new(), + }; + let buildpack_toml_string = include_str!("../../buildpack.toml"); + let platform = + <::Platform as libcnb::Platform>::from_path(&platform_dir).unwrap(); + let buildpack_descriptor: libcnb::data::buildpack::ComponentBuildpackDescriptor< + ::Metadata, + > = toml::from_str(buildpack_toml_string).unwrap(); + let buildpack_plan = libcnb::data::buildpack_plan::BuildpackPlan { + entries: Vec::::new(), + }; + let store = None; + + BuildContext { + layers_dir, + app_dir, + buildpack_dir, + target, + platform, + buildpack_plan, + buildpack_descriptor, + store, + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::RubyBuildpack; + use core::panic; + use libcnb::data::layer_name; + use libcnb::layer::{EmptyLayerCause, LayerState}; + use magic_migrate::{migrate_toml_chain, try_migrate_deserializer_chain, Migrate, TryMigrate}; + use serde::Deserializer; + use std::convert::Infallible; + + /// Struct for asserting the behavior of `cached_layer_write_metadata` + #[derive(Debug, serde::Serialize, serde::Deserialize)] + struct TestMetadata { + value: String, + } + impl MetadataDiff for TestMetadata { + fn diff(&self, old: &Self) -> Vec { + if self.value == old.value { + vec![] + } else { + vec![format!("value ({} to {})", old.value, self.value)] + } + } + } + migrate_toml_chain! {TestMetadata} + + #[test] + fn test_cached_layer_write_metadata_restored_layer_action() { + let temp = tempfile::tempdir().unwrap(); + let context = temp_build_context::(temp.path()); + + // First write + let result = cached_layer_write_metadata( + layer_name!("testing"), + &context, + &TestMetadata { + value: "hello".to_string(), + }, + ) + .unwrap(); + assert!(matches!( + result.state, + LayerState::Empty { + cause: EmptyLayerCause::NewlyCreated + } + )); + + // Second write, preserve the contents + let result = cached_layer_write_metadata( + layer_name!("testing"), + &context, + &TestMetadata { + value: "hello".to_string(), + }, + ) + .unwrap(); + let LayerState::Restored { cause } = &result.state else { + panic!("Expected restored layer") + }; + assert_eq!(cause, "Using cache"); + + // Third write, change the data + let result = cached_layer_write_metadata( + layer_name!("testing"), + &context, + &TestMetadata { + value: "world".to_string(), + }, + ) + .unwrap(); + + let LayerState::Empty { + cause: EmptyLayerCause::RestoredLayerAction { cause }, + } = &result.state + else { + panic!("Expected empty layer with restored layer action"); + }; + assert_eq!( + cause, + "Clearing cache due to change: value (hello to world)" + ); + } + + /// Struct for asserting the behavior of `invalid_metadata_action` + #[derive(serde::Deserialize, serde::Serialize, Debug)] + #[serde(deny_unknown_fields)] + struct PersonV1 { + name: String, + } + /// Struct for asserting the behavior of `invalid_metadata_action` + #[derive(serde::Deserialize, serde::Serialize, Debug)] + #[serde(deny_unknown_fields)] + struct PersonV2 { + name: String, + updated_at: String, + } + // First define how to map from one struct to another + impl TryFrom for PersonV2 { + type Error = NotRichard; + fn try_from(value: PersonV1) -> Result { + if &value.name == "Schneems" { + Ok(PersonV2 { + name: value.name.clone(), + updated_at: "unknown".to_string(), + }) + } else { + Err(NotRichard { + name: value.name.clone(), + }) + } + } + } + #[derive(Debug, Eq, PartialEq)] + struct NotRichard { + name: String, + } + impl From for PersonMigrationError { + fn from(value: NotRichard) -> Self { + PersonMigrationError::NotRichard(value) + } + } + #[derive(Debug, Eq, PartialEq, thiserror::Error)] + enum PersonMigrationError { + #[error("Not Richard")] + NotRichard(NotRichard), + } + try_migrate_deserializer_chain!( + deserializer: toml::Deserializer::new, + error: PersonMigrationError, + chain: [PersonV1, PersonV2], + ); + + #[test] + fn test_invalid_metadata_action() { + let (action, message) = invalid_metadata_action::(&PersonV1 { + name: "schneems".to_string(), + }); + assert!(matches!(action, InvalidMetadataAction::ReplaceMetadata(_))); + assert_eq!(message, "Replaced metadata".to_string()); + + let (action, message) = invalid_metadata_action::(&PersonV1 { + name: "not_richard".to_string(), + }); + assert!(matches!(action, InvalidMetadataAction::DeleteLayer)); + assert_eq!( + message, + "Clearing cache due to metadata migration error: Not Richard".to_string() + ); + + let (action, message) = invalid_metadata_action::(&TestMetadata { + value: "world".to_string(), + }); + assert!(matches!(action, InvalidMetadataAction::DeleteLayer)); + assert_eq!( + message, + "Clearing cache due to invalid metadata (value = \"world\")".to_string() + ); + + // Unable to produce this error at will: "Clearing cache due to invalid metadata serialization error: {error}" + } +} diff --git a/buildpacks/ruby/src/main.rs b/buildpacks/ruby/src/main.rs index 141e33e8..fcd1e8f3 100644 --- a/buildpacks/ruby/src/main.rs +++ b/buildpacks/ruby/src/main.rs @@ -158,7 +158,7 @@ impl Buildpack for RubyBuildpack { let (bullet, layer_env) = layers::ruby_install_layer::handle( &context, bullet, - layers::ruby_install_layer::Metadata { + &layers::ruby_install_layer::Metadata { distro_name: context.target.distro_name.clone(), distro_version: context.target.distro_version.clone(), cpu_architecture: context.target.arch.clone(),