From 439d2660e06f10b7be0058bd3cd77940182a2dd8 Mon Sep 17 00:00:00 2001 From: Richard Schneeman Date: Thu, 13 Jul 2023 15:29:26 -0700 Subject: [PATCH 01/40] Introduce new metrics buildpack The classic `heroku/metrics` buildpack sets up a statsd daemon that the dyno can use to send Heroku language specific metrics. For Ruby the biggest one is Puma Pool Capacity. This code will download the metrics client and put it on the path, however there's an issue affecting our ability to actually boot a daemon. ## Context Classic buildpacks (Heroku v2) have acces to a `profile.d` directory they can use to write scripts which execute at boot time. This was largely needed to set build time environment variables because the directory structure of build and runtime was different. It was eventually used by the `heroku/metrics` buildpack to download an run a metrics daemon in the background. Cloud Native Buildpacks (CNB) removed support for bash in an effort to simplify the spec and allow buildpacks to target extremely lightweight distributions that do not include `bash` (for example if you are running a pure JVM application, then there's no runtime need to have bash on the system, removing it reduces the image size and can be viewed as a reduction in security surface area). As part of this work to remove bash from the spec the `profile.d`-like interface was removed from the CNB spec. ## Problem Without a spec supported way to execute a script at startup there is no way for a buildpack to provide a background daemon. ## Possible solutions - TBD --- Cargo.lock | 191 +++++++++++------ Cargo.toml | 1 + buildpacks/metrics-agent/CHANGELOG.md | 4 + buildpacks/metrics-agent/Cargo.toml | 23 ++ buildpacks/metrics-agent/buildpack.toml | 23 ++ .../src/layers/download_agentmon.rs | 199 ++++++++++++++++++ buildpacks/metrics-agent/src/layers/mod.rs | 1 + buildpacks/metrics-agent/src/main.rs | 130 ++++++++++++ 8 files changed, 505 insertions(+), 67 deletions(-) create mode 100644 buildpacks/metrics-agent/CHANGELOG.md create mode 100644 buildpacks/metrics-agent/Cargo.toml create mode 100644 buildpacks/metrics-agent/buildpack.toml create mode 100644 buildpacks/metrics-agent/src/layers/download_agentmon.rs create mode 100644 buildpacks/metrics-agent/src/layers/mod.rs create mode 100644 buildpacks/metrics-agent/src/main.rs diff --git a/Cargo.lock b/Cargo.lock index 6da50e22..45755750 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -25,9 +25,9 @@ checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" [[package]] name = "base64" -version = "0.21.2" +version = "0.21.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "604178f6c5c21f02dc555784810edfb88d34ac2c73b2eae109655649ee73ce3d" +checksum = "414dcefbc63d77c526a76b3afcf6fbb9b5e2791c19c3aa2297733208750c6e53" [[package]] name = "bit-set" @@ -99,6 +99,20 @@ dependencies = [ "serde", ] +[[package]] +name = "cargo_metadata" +version = "0.15.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eee4243f1f26fc7a42710e7439c149e2b10b05472f88090acce52632f231a73a" +dependencies = [ + "camino", + "cargo-platform", + "semver", + "serde", + "serde_json", + "thiserror", +] + [[package]] name = "cargo_metadata" version = "0.17.0" @@ -115,9 +129,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.0.82" +version = "1.0.83" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "305fe645edc1442a0fa8b6726ba61d422798d37a52e12eaecf4b022ebbb88f01" +checksum = "f1174fb0b6ec23863f8b971027804a42614e347eafb0a95bf0b12cdae21fc4d0" dependencies = [ "libc", ] @@ -140,7 +154,7 @@ dependencies = [ "glob", "indoc", "lazy_static", - "libcnb", + "libcnb 0.14.0", "libherokubuildpack", "regex", "serde", @@ -365,12 +379,6 @@ version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b" -[[package]] -name = "hashbrown" -version = "0.12.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" - [[package]] name = "hashbrown" version = "0.14.0" @@ -392,7 +400,7 @@ dependencies = [ "fs-err", "glob", "indoc", - "libcnb", + "libcnb 0.14.0", "libcnb-test", "rand", "regex", @@ -405,6 +413,26 @@ dependencies = [ "url", ] +[[package]] +name = "heroku-statsd-metrics" +version = "0.0.0" +dependencies = [ + "commons", + "flate2", + "fs-err", + "glob", + "indoc", + "libcnb 0.13.0", + "rand", + "regex", + "serde", + "tar", + "tempfile", + "thiserror", + "ureq", + "url", +] + [[package]] name = "idna" version = "0.4.0" @@ -415,16 +443,6 @@ dependencies = [ "unicode-normalization", ] -[[package]] -name = "indexmap" -version = "1.9.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99" -dependencies = [ - "autocfg", - "hashbrown 0.12.3", -] - [[package]] name = "indexmap" version = "2.0.0" @@ -432,7 +450,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d5477fe2230a79769d8dc68e0eabf5437907c0457a5614a9e8dddb67f65eb65d" dependencies = [ "equivalent", - "hashbrown 0.14.0", + "hashbrown", ] [[package]] @@ -486,17 +504,44 @@ version = "0.2.147" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b4668fb0ea861c1df094127ac5f1da3409a82116a4ba74fca2e58ef927159bb3" +[[package]] +name = "libcnb" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39d5e0e5f0ea6fe91d867457289d88c4f56631e37fac072d11676ff970715012" +dependencies = [ + "libcnb-data 0.13.0", + "libcnb-proc-macros 0.13.0", + "serde", + "thiserror", + "toml", +] + [[package]] name = "libcnb" version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5132851c82d808e6b42edd1cc9e7cb9b16b0274c325b25fdb42660fae9b2e88b" dependencies = [ - "libcnb-data", - "libcnb-proc-macros", + "libcnb-data 0.14.0", + "libcnb-proc-macros 0.14.0", + "serde", + "thiserror", + "toml", +] + +[[package]] +name = "libcnb-data" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "631bda3e80115baf38894609cde58b796d3b3fc0f47cca369321c230df53d563" +dependencies = [ + "fancy-regex", + "libcnb-proc-macros 0.13.0", "serde", "thiserror", "toml", + "uriparse", ] [[package]] @@ -506,7 +551,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4bed8b0f2676aebeb216a7f7872151fbf74ff3706c7027449573c03c9d7f3393" dependencies = [ "fancy-regex", - "libcnb-proc-macros", + "libcnb-proc-macros 0.14.0", "serde", "thiserror", "toml", @@ -519,20 +564,32 @@ version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "412f8c3ee7e2fcaff1acd1926a238cac271e86d7258c51038053fac17d85d144" dependencies = [ - "cargo_metadata", - "libcnb-data", + "cargo_metadata 0.17.0", + "libcnb-data 0.14.0", "petgraph", "toml", "which", ] +[[package]] +name = "libcnb-proc-macros" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ab33c1d63ffd280516abc7ada744fc1b653a888c439f5e2962d5371d0aecaf7" +dependencies = [ + "cargo_metadata 0.15.4", + "fancy-regex", + "quote", + "syn", +] + [[package]] name = "libcnb-proc-macros" version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c715cec438b3a02c3564e9b9c20a78c54b9c71874249bec1e3d45fcd2537cfcf" dependencies = [ - "cargo_metadata", + "cargo_metadata 0.17.0", "fancy-regex", "quote", "syn", @@ -544,10 +601,10 @@ version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "92ee152780ab4ad6e4aca8ec12767c090677294b7aa4ffc1a90633d38b429c9e" dependencies = [ - "cargo_metadata", + "cargo_metadata 0.17.0", "fastrand", "fs_extra", - "libcnb-data", + "libcnb-data 0.14.0", "libcnb-package", "tempfile", ] @@ -560,7 +617,7 @@ checksum = "999689d1a9f8cbea478ae7c4ce6136601e7abe9512ccfc0409f5525949a41457" dependencies = [ "crossbeam-utils", "flate2", - "libcnb", + "libcnb 0.14.0", "pathdiff", "sha2", "tar", @@ -595,9 +652,9 @@ checksum = "b5e6163cb8c49088c2c36f57875e58ccd8c87c7427f7fbd50ea6710b2f3f2e8f" [[package]] name = "memchr" -version = "2.5.0" +version = "2.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2dffe52ecf27772e601905b7522cb4ef790d2cc203488bbd0e2fe85fcb74566d" +checksum = "76fc44e2588d5b436dbc3c6cf62aef290f90dab6235744a93dfe1cc18f451e2c" [[package]] name = "memoffset" @@ -644,9 +701,9 @@ checksum = "dd8b5dd2ae5ed71462c540258bedcb51965123ad7e7ccf4b9a8cafaa4a63576d" [[package]] name = "ordered-float" -version = "3.9.0" +version = "3.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "126d3e6f3926bfb0fb24495b4f4da50626f547e54956594748e3d8882a0320b4" +checksum = "2a54938017eacd63036332b4ae5c8a49fc8c0c1d6d629893057e4f13609edd06" dependencies = [ "num-traits", ] @@ -665,12 +722,12 @@ checksum = "9b2a4787296e9989611394c33f193f676704af1686e70b8f8033ab5ba9a35a94" [[package]] name = "petgraph" -version = "0.6.3" +version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4dd7d28ee937e54fe3080c91faa1c3a46c06de6252988a7f4592ba2310ef22a4" +checksum = "e1d3afd2628e69da2be385eb6f2fd57c8ac7977ceeff6dc166ff1657b0e386a9" dependencies = [ "fixedbitset", - "indexmap 1.9.3", + "indexmap", ] [[package]] @@ -766,9 +823,9 @@ dependencies = [ [[package]] name = "regex" -version = "1.9.3" +version = "1.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "81bc1d4caf89fac26a70747fe603c130093b53c773888797a6329091246d651a" +checksum = "12de2eff854e5fa4b1295edd650e227e9d8fb0c9e90b12e7f36d6a6811791a29" dependencies = [ "aho-corasick", "memchr", @@ -778,9 +835,9 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.3.6" +version = "0.3.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fed1ceff11a1dddaee50c9dc8e4938bd106e9d89ae372f192311e7da498e3b69" +checksum = "49530408a136e16e5b486e883fbb6ba058e8e4e8ae6621a77b048b314336e629" dependencies = [ "aho-corasick", "memchr", @@ -789,9 +846,9 @@ dependencies = [ [[package]] name = "regex-syntax" -version = "0.7.4" +version = "0.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5ea92a5b6195c6ef2a0295ea818b312502c6fc94dde986c5553242e18fd4ce2" +checksum = "dbb5fb1acd8a1a18b3dd5be62d25485eb770e05afb408a9627d14d451bae12da" [[package]] name = "ring" @@ -810,9 +867,9 @@ dependencies = [ [[package]] name = "rustix" -version = "0.38.8" +version = "0.38.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19ed4fa021d81c8392ce04db050a3da9a60299050b7ae1cf482d862b54a7218f" +checksum = "9bfe0f2582b4931a45d1fa608f8a8722e8b3c7ac54dd6d5f3b3212791fedef49" dependencies = [ "bitflags 2.4.0", "errno", @@ -823,21 +880,21 @@ dependencies = [ [[package]] name = "rustls" -version = "0.21.6" +version = "0.21.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d1feddffcfcc0b33f5c6ce9a29e341e4cd59c3f78e7ee45f4a40c038b1d6cbb" +checksum = "cd8d6c9f025a446bc4d18ad9632e69aec8f287aa84499ee335599fabd20c3fd8" dependencies = [ "log", "ring", - "rustls-webpki 0.101.3", + "rustls-webpki 0.101.4", "sct", ] [[package]] name = "rustls-webpki" -version = "0.100.1" +version = "0.100.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d6207cd5ed3d8dca7816f8f3725513a34609c0c765bf652b8c3cb4cfd87db46b" +checksum = "e98ff011474fa39949b7e5c0428f9b4937eda7da7848bbb947786b7be0b27dab" dependencies = [ "ring", "untrusted", @@ -845,9 +902,9 @@ dependencies = [ [[package]] name = "rustls-webpki" -version = "0.101.3" +version = "0.101.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "261e9e0888cba427c3316e6322805653c9425240b6fd96cee7cb671ab70ab8d0" +checksum = "7d93931baf2d282fff8d3a532bbfd7653f734643161b87e3e01e59a04439bf0d" dependencies = [ "ring", "untrusted", @@ -895,18 +952,18 @@ dependencies = [ [[package]] name = "serde" -version = "1.0.183" +version = "1.0.188" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32ac8da02677876d532745a130fc9d8e6edfa81a269b107c5b00829b91d8eb3c" +checksum = "cf9e0fcba69a370eed61bcf2b728575f726b50b55cba78064753d708ddc7549e" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.183" +version = "1.0.188" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aafe972d60b0b9bee71a91b92fee2d4fb3c9d7e8f6b179aa99f27203d99a4816" +checksum = "4eca7ac642d82aa35b60049a6eccb4be6be75e599bd2e9adb5f875a737654af2" dependencies = [ "proc-macro2", "quote", @@ -980,9 +1037,9 @@ dependencies = [ [[package]] name = "tempfile" -version = "3.7.1" +version = "3.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc02fddf48964c42031a0b3fe0428320ecf3a73c401040fc0096f97794310651" +checksum = "cb94d2f3cc536af71caac6b6fcebf65860b347e7ce0cc9ebe8f70d3e521054ef" dependencies = [ "cfg-if", "fastrand", @@ -1062,7 +1119,7 @@ version = "0.19.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8123f27e969974a3dfba720fdb560be359f57b44302d280ba72e76a74480e8a" dependencies = [ - "indexmap 2.0.0", + "indexmap", "serde", "serde_spanned", "toml_datetime", @@ -1113,7 +1170,7 @@ dependencies = [ "log", "once_cell", "rustls", - "rustls-webpki 0.100.1", + "rustls-webpki 0.100.2", "url", "webpki-roots", ] @@ -1130,9 +1187,9 @@ dependencies = [ [[package]] name = "url" -version = "2.4.0" +version = "2.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "50bff7831e19200a85b17131d085c25d7811bc4e186efdaf54bbd132994a88cb" +checksum = "143b538f18257fac9cad154828a57c6bf5157e1aa604d4816b5995bf6de87ae5" dependencies = [ "form_urlencoded", "idna", @@ -1243,7 +1300,7 @@ version = "0.23.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b03058f88386e5ff5310d9111d53f48b17d732b401aeb83a8d5190f2ac459338" dependencies = [ - "rustls-webpki 0.100.1", + "rustls-webpki 0.100.2", ] [[package]] @@ -1369,9 +1426,9 @@ checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" [[package]] name = "winnow" -version = "0.5.14" +version = "0.5.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d09770118a7eb1ccaf4a594a221334119a44a814fcb0d31c5b85e83e97227a97" +checksum = "7c2e3184b9c4e92ad5167ca73039d0c42476302ab603e2fec4487511f38ccefc" dependencies = [ "memchr", ] diff --git a/Cargo.toml b/Cargo.toml index 1b1eec91..03e9109b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,7 @@ [workspace] resolver = "2" members = [ + "buildpacks/metrics-agent", "buildpacks/ruby", "commons" ] diff --git a/buildpacks/metrics-agent/CHANGELOG.md b/buildpacks/metrics-agent/CHANGELOG.md new file mode 100644 index 00000000..1410f16b --- /dev/null +++ b/buildpacks/metrics-agent/CHANGELOG.md @@ -0,0 +1,4 @@ +# Changelog + +## [Unreleased] + diff --git a/buildpacks/metrics-agent/Cargo.toml b/buildpacks/metrics-agent/Cargo.toml new file mode 100644 index 00000000..4da3a149 --- /dev/null +++ b/buildpacks/metrics-agent/Cargo.toml @@ -0,0 +1,23 @@ +[package] +name = "heroku-statsd-metrics" +# This crate is not published, so the only version that is used is the one in buildpack.toml. +version = "0.0.0" +publish = false +edition = "2021" +rust-version = "1.66" + +[dependencies] +commons = { path = "../../commons" } +flate2 = "1" +fs-err = "2" +indoc = "2" +libcnb = "0.13" +rand = "0.8" +regex = "1" +serde = "1" +tar = "0.4" +tempfile = "3" +thiserror = "1" +ureq = "2" +url = "2" +glob = "0.3" diff --git a/buildpacks/metrics-agent/buildpack.toml b/buildpacks/metrics-agent/buildpack.toml new file mode 100644 index 00000000..c5fcc2b7 --- /dev/null +++ b/buildpacks/metrics-agent/buildpack.toml @@ -0,0 +1,23 @@ +api = "0.9" + +[buildpack] +id = "heroku/statsd-metrics" +version = "2.0.0" +name = "Statsd Metrics" +homepage = "https://github.com/heroku/buildpacks-ruby" +description = "Installs an agent to send language specific metrics back to heroku via statsd" +keywords = ["statsd", "metrics"] + +[[stacks]] +id = "heroku-20" + +[[stacks]] +id = "heroku-22" + +[[buildpack.licenses]] +type = "BSD-3-Clause" + +[metadata] +[metadata.release] +[metadata.release.docker] +# repository = "docker.io/heroku/buildpack-ruby" diff --git a/buildpacks/metrics-agent/src/layers/download_agentmon.rs b/buildpacks/metrics-agent/src/layers/download_agentmon.rs new file mode 100644 index 00000000..cd80ed8e --- /dev/null +++ b/buildpacks/metrics-agent/src/layers/download_agentmon.rs @@ -0,0 +1,199 @@ +use crate::{MetricsAgentBuildpack, MetricsAgentError}; +use libcnb::{ + generic::GenericMetadata, + layer::{Layer, LayerResultBuilder}, +}; +use std::os::unix::fs::PermissionsExt; +use url::Url; + +use crate::build_output; +use flate2::read::GzDecoder; +use libcnb::data::layer_content_metadata::LayerTypes; +use std::path::Path; +use tar::Archive; +use tempfile::NamedTempFile; +// use url::Url; + +#[derive(Debug)] +pub(crate) struct DownloadAgentmon { + pub(crate) section: build_output::Section, +} + +#[derive(thiserror::Error, Debug)] +pub(crate) enum DownloadAgentmonError { + #[error("Could not read file permissions {0}")] + PermissionError(std::io::Error), + + #[error("Could not parse url {0}")] + UrlParseError(url::ParseError), + + #[error("Could not open file: {0}")] + CouldNotOpenFile(std::io::Error), + + #[error("Could not untar: {0}")] + CouldNotUnpack(std::io::Error), + + // Boxed to prevent `large_enum_variant` errors since `ureq::Error` is massive. + #[error("Download error: {0}")] + RequestError(Box), + + #[error("Could not create file: {0}")] + CouldNotCreateDestinationFile(std::io::Error), + + #[error("Could not write file: {0}")] + CouldNotWriteDestinationFile(std::io::Error), +} + +impl Layer for DownloadAgentmon { + type Buildpack = MetricsAgentBuildpack; + type Metadata = GenericMetadata; + + fn types(&self) -> libcnb::data::layer_content_metadata::LayerTypes { + LayerTypes { + build: true, + launch: true, + cache: true, + } + } + + fn create( + &self, + _context: &libcnb::build::BuildContext, + layer_path: &std::path::Path, + ) -> Result< + libcnb::layer::LayerResult, + ::Error, + > { + let mut timer = self.section.say_with_inline_timer("Installing"); + + let agentmon_tgz = NamedTempFile::new() + .map_err(DownloadAgentmonError::CouldNotCreateDestinationFile) + .map_err(MetricsAgentError::DownloadAgentmonError)?; + + let url = Url::parse("https://agentmon-releases.s3.amazonaws.com/latest") + .map_err(DownloadAgentmonError::UrlParseError) + .map_err(MetricsAgentError::DownloadAgentmonError)?; + + download(url.as_ref(), agentmon_tgz.path()) + .map_err(MetricsAgentError::DownloadAgentmonError)?; + + let destination = layer_path.join("bin"); + untar(agentmon_tgz.path(), &destination) + .map_err(MetricsAgentError::DownloadAgentmonError)?; + + chmod_plus_x(&destination.join("agentmon")) + .map_err(DownloadAgentmonError::PermissionError) + .map_err(MetricsAgentError::DownloadAgentmonError)?; + + timer.done(); + + LayerResultBuilder::new(GenericMetadata::default()).build() + } + + fn existing_layer_strategy( + &self, + _context: &libcnb::build::BuildContext, + _layer_data: &libcnb::layer::LayerData, + ) -> Result::Error> + { + // TODO caching logic + // + // The classic buildpack actually downloads this binary on dyno boot every time + // We could use content headers to check if it needs to be re-downloaded. + Ok(libcnb::layer::ExistingLayerStrategy::Recreate) + } + + fn migrate_incompatible_metadata( + &self, + _context: &libcnb::build::BuildContext, + _metadata: &GenericMetadata, + ) -> Result< + libcnb::layer::MetadataMigration, + ::Error, + > { + self.section + .say_with_details("Clearing cache", "invalid metadata"); + + Ok(libcnb::layer::MetadataMigration::RecreateLayer) + } +} + +pub(crate) fn untar( + path: impl AsRef, + destination: impl AsRef, +) -> Result<(), DownloadAgentmonError> { + let file = + fs_err::File::open(path.as_ref()).map_err(DownloadAgentmonError::CouldNotOpenFile)?; + + Archive::new(GzDecoder::new(file)) + .unpack(destination.as_ref()) + .map_err(DownloadAgentmonError::CouldNotUnpack) +} + +/// Sets file permissions on the given path to 7xx (similar to `chmod +x `) +fn chmod_plus_x(path: &Path) -> Result<(), std::io::Error> { + let mut perms = fs_err::metadata(path)?.permissions(); + let mut mode = perms.mode(); + octal_executable_permission(&mut mode); + perms.set_mode(mode); + + fs_err::set_permissions(path, perms) +} + +/// Ensures the provided octal number's executable +/// bit is enabled. +/// +/// i.e. chmod +x will ensure that the first digit +/// of the file permission is 7 on unix so if you pass +/// in 0o455 it would be mutated to 0o755 +fn octal_executable_permission(mode: &mut u32) { + *mode |= 0o700; +} + +pub(crate) fn download( + uri: impl AsRef, + destination: impl AsRef, +) -> Result<(), DownloadAgentmonError> { + let mut response_reader = ureq::get(uri.as_ref()) + .call() + .map_err(|err| DownloadAgentmonError::RequestError(Box::new(err)))? + .into_reader(); + + let mut destination_file = fs_err::File::create(destination.as_ref()) + .map_err(DownloadAgentmonError::CouldNotCreateDestinationFile)?; + + std::io::copy(&mut response_reader, &mut destination_file) + .map_err(DownloadAgentmonError::CouldNotWriteDestinationFile)?; + + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_chmod() { + let tmp = tempfile::tempdir().unwrap(); + let file = tmp.into_path().join("file"); + std::fs::write(&file, "lol").unwrap(); + + let before = file.metadata().unwrap().permissions().mode(); + chmod_plus_x(&file).unwrap(); + let after = file.metadata().unwrap().permissions().mode(); + assert!(before != after); + } + + #[test] + fn test_executable_logic() { + // Sets executable bit + let mut mode = 0o455; + octal_executable_permission(&mut mode); + assert_eq!(0o755, mode); + + // Does not affect already executable + let mut mode = 0o745; + octal_executable_permission(&mut mode); + assert_eq!(0o745, mode); + } +} diff --git a/buildpacks/metrics-agent/src/layers/mod.rs b/buildpacks/metrics-agent/src/layers/mod.rs new file mode 100644 index 00000000..33f09cad --- /dev/null +++ b/buildpacks/metrics-agent/src/layers/mod.rs @@ -0,0 +1 @@ +pub(crate) mod download_agentmon; diff --git a/buildpacks/metrics-agent/src/main.rs b/buildpacks/metrics-agent/src/main.rs new file mode 100644 index 00000000..81c5ac5f --- /dev/null +++ b/buildpacks/metrics-agent/src/main.rs @@ -0,0 +1,130 @@ +mod layers; + +use crate::layers::download_agentmon::{DownloadAgentmon, DownloadAgentmonError}; +use commons::build_output::{self, fmt::ErrorInfo}; +use indoc::formatdoc; +use libcnb::{ + build::{BuildContext, BuildResult, BuildResultBuilder}, + buildpack_main, + data::{build_plan::BuildPlanBuilder, layer_name}, + detect::{DetectContext, DetectResult, DetectResultBuilder}, + Buildpack, +}; + +#[derive(Debug)] +enum MetricsAgentError { + DownloadAgentmonError(DownloadAgentmonError), +} + +impl From for libcnb::Error { + fn from(error: MetricsAgentError) -> Self { + libcnb::Error::BuildpackError(error) + } +} + +buildpack_main!(MetricsAgentBuildpack); + +pub(crate) struct MetricsAgentBuildpack; + +impl Buildpack for MetricsAgentBuildpack { + type Platform = libcnb::generic::GenericPlatform; + type Metadata = libcnb::generic::GenericMetadata; + type Error = MetricsAgentError; + + fn detect(&self, context: DetectContext) -> libcnb::Result { + let plan_builder = BuildPlanBuilder::new().provides("heroku-statsd-metrics-agent"); + + if let Ok(true) = fs_err::read_to_string(context.app_dir.join("Gemfile.lock")) + .map(|lockfile| lockfile.contains("barnes")) + { + DetectResultBuilder::pass() + .build_plan(plan_builder.requires("heroku-statsd-metrics-agent").build()) + .build() + } else { + DetectResultBuilder::pass() + .build_plan(plan_builder.build()) + .build() + } + } + + fn build(&self, context: BuildContext) -> libcnb::Result { + let build_duration = build_output::buildpack_name("Heroku Statsd Metrics Agent"); + + let section = build_output::section("Metrics agent"); + context.handle_layer( + layer_name!("statsd-metrics-agent"), + DownloadAgentmon { section }, + )?; + + // TODO write launch script + // + // if [[ "${AGENTMON_DEBUG}" = "true" ]]; then + // AGENTMON_FLAGS+=("-debug") + // fi + + // if [[ -x "${BUILD_DIR}/bin/agentmon" ]]; then + // (while true; do + // ${BUILD_DIR}/bin/agentmon "${AGENTMON_FLAGS[@]}" "${HEROKU_METRICS_URL}" + // echo "agentmon completed with status=${?}. Restarting" + // sleep 1 + // done) & + // else + // echo "No agentmon executable found. Not starting." + // fi + + build_duration.done(); + BuildResultBuilder::new().build() + } + + fn on_error(&self, err: libcnb::Error) { + on_error(err); + } +} + +#[derive(Debug)] +enum Cause { + OurError(MetricsAgentError), + FrameworkError(libcnb::Error), +} + +fn cause(err: libcnb::Error) -> Cause { + match err { + libcnb::Error::BuildpackError(err) => Cause::OurError(err), + err => Cause::FrameworkError(err), + } +} + +pub(crate) fn on_error(err: libcnb::Error) { + match cause(err) { + Cause::OurError(error) => log_our_error(error), + Cause::FrameworkError(error) => ErrorInfo::header_body_details( + "heroku/buildpack-ruby internal buildpack error", + formatdoc! {" + An unexpected internal error was reported by the framework used + by this buildpack. + + If the issue persists, consider opening an issue on the GitHub + repository. If you are unable to deploy to Heroku as a result + of this issue, consider opening a ticket for additional support. + "}, + error, + ) + .print(), + }; +} + +fn log_our_error(error: MetricsAgentError) { + match error { + MetricsAgentError::DownloadAgentmonError(error) => ErrorInfo::header_body_details( + formatdoc! { + "Could not install Statsd agent" + }, + formatdoc! { + "An error occured while downloading and installing the metrics agent + the buildpack cannot continue" + }, + error, + ) + .print(), + } +} From 5f449e6a6d00d29edbefe8e4745e27f7705f8ba6 Mon Sep 17 00:00:00 2001 From: Richard Schneeman Date: Wed, 19 Jul 2023 13:46:03 -0500 Subject: [PATCH 02/40] Update build output Errors in the middle of an `Installing ... ` section looked weird because it would start rendering on the same line: ``` # Heroku Statsd Metrics Agent - Metrics agent - Installing .! ERROR: Could not install Statsd agent ! ! An error occured while downloading and installing the metrics agent ! the buildpack cannot continue ! Debug information: Could not untar: failed to iterate over archive ERROR: failed to build: exit status 1 ERROR: failed to build: executing lifecycle: failed with status code: 51 ``` I changed it so it's now like this: ``` # Heroku Statsd Metrics Agent - Metrics agent - Downloading .. ! ERROR: Could not install Statsd agent ! ! An error occured while downloading and installing the metrics agent ! the buildpack cannot continue ! ! Debug information: Could not untar: failed to iterate over archive ERROR: failed to build: exit status 1 ERROR: failed to build: executing lifecycle: failed with status code: 51 ``` I think it would look better to indent the "debug information" at least. Maybe add an extra newline. --- commons/src/build_output.rs | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/commons/src/build_output.rs b/commons/src/build_output.rs index 1bc6cb91..10cb8596 100644 --- a/commons/src/build_output.rs +++ b/commons/src/build_output.rs @@ -533,14 +533,15 @@ pub mod fmt { let noun = noun.as_ref(); let header = header.as_ref(); let body = help_url(body, url); - colorize( + + let contents = colorize( color, bangify(formatdoc! {" {noun} {header} - {body} - "}), - ) + {body}"}), + ); + format!("\n{contents}") } #[must_use] @@ -600,11 +601,18 @@ pub mod fmt { debug_details, } = info; - let body = look_at_me(ERROR_COLOR, "ERROR:", header, body, url); if let Some(details) = debug_details { - format!("{body}\n\nDebug information: {details}") + let message = look_at_me( + ERROR_COLOR, + "ERROR:", + header, + format!("{body}\n\nDebug information: "), + url, + ); + + format!("{message}\n{details}\n") } else { - body + look_at_me(ERROR_COLOR, "ERROR:", header, body, url) } } From 32a611b1ddd7f1e8f83fdee7813b9442b5aed46b Mon Sep 17 00:00:00 2001 From: Richard Schneeman Date: Tue, 18 Jul 2023 17:40:11 -0500 Subject: [PATCH 03/40] Attempting workaround with exec.d MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Since exec.d will run any program given to it I want to intentionally leak a bash process that will continue to run in the background. It works locally: ``` $ cat lol.sh #!/usr/bin/env bash echo "spawning agentmon" & while true; do echo "pretend agentmon"; sleep 2; done & ⛄️ 3.1.4 🚀 /tmp $ ./lol.sh spawning agentmon pretend agentmon ⛄️ 3.1.4 🚀 /tmp $ echo "pretend agentmon it works" it works ⛄️ 3.1.4 🚀 /tmp $ pretend agentmon echo "lpretend agentmon ol" lol ⛄️ 3.1.4 🚀 /tmp $ pretend agentmon pretend agentmon epretend agentmon echo "pretend agentmon It keeps runnipretend agentmon ng even apretend agentmon s I can pretend agentmon use the systpretend agentmon em" It keeps running even as I can use the system ⛄️ 3.1.4 🚀 /tmp $ pretend agentmon ``` However when I try to replicate this with exec.d it never seems to end the `exec.d` program and never yields control to a different process: ``` $ cargo libcnb package; docker rmi my-image; pack build my-image --buildpack /Users/rschneeman/Documents/projects/work/buildpacks/buildpacks-ruby/target/buildpack/debug/heroku_statsd-metrics --path /tmp/47b6249d5e0a353f91910f848a700061 --pull-policy never 🔍 Locating buildpacks... 📦 [1/2] Building heroku/ruby Determining automatic cross-compile settings... Building binaries (x86_64-unknown-linux-musl)... Finished dev [unoptimized] target(s) in 0.89s Writing buildpack directory... Successfully wrote buildpack directory: ../../target/buildpack/debug/heroku_ruby (12.64 MiB) 📦 [2/2] Building heroku/statsd-metrics Determining automatic cross-compile settings... Building binaries (x86_64-unknown-linux-musl)... Blocking waiting for file lock on package cache Blocking waiting for file lock on package cache Blocking waiting for file lock on package cache Compiling heroku-statsd-metrics v0.0.0 (/Users/rschneeman/Documents/projects/work/buildpacks/buildpacks-ruby/buildpacks/metrics-agent) warning: unused import: `DownloadAgentmon` --> buildpacks/metrics-agent/src/main.rs:3:40 | 3 | use crate::layers::download_agentmon::{DownloadAgentmon, DownloadAgentmonError}; | ^^^^^^^^^^^^^^^^ | = note: `#[warn(unused_imports)]` on by default warning: `heroku-statsd-metrics` (bin "heroku-statsd-metrics") generated 1 warning (run `cargo fix --bin "heroku-statsd-metrics"` to apply 1 suggestion) Finished dev [unoptimized] target(s) in 7.67s Writing buildpack directory... Successfully wrote buildpack directory: ../../target/buildpack/debug/heroku_statsd-metrics (6.58 MiB) ✨ Packaging successfully finished! 💡 To test your buildpack locally with pack, run: pack build my-image-name \ --buildpack /Users/rschneeman/Documents/projects/work/buildpacks/buildpacks-ruby/target/buildpack/debug/heroku_ruby \ --buildpack /Users/rschneeman/Documents/projects/work/buildpacks/buildpacks-ruby/target/buildpack/debug/heroku_statsd-metrics \ --path /path/to/application /Users/rschneeman/Documents/projects/work/buildpacks/buildpacks-ruby/target/buildpack/debug/heroku_ruby /Users/rschneeman/Documents/projects/work/buildpacks/buildpacks-ruby/target/buildpack/debug/heroku_statsd-metrics Error: No such image: my-image ===> ANALYZING Image with name "my-image" not found ===> DETECTING heroku/statsd-metrics 2.0.0 ===> RESTORING ===> BUILDING # Heroku Statsd Metrics Agent - Done ===> EXPORTING Adding layer 'heroku/statsd-metrics:layer_name' Adding layer 'buildpacksio/lifecycle:launch.sbom' Adding 1/1 app layer(s) Adding layer 'buildpacksio/lifecycle:launcher' Adding layer 'buildpacksio/lifecycle:config' Adding label 'io.buildpacks.lifecycle.metadata' Adding label 'io.buildpacks.build.metadata' Adding label 'io.buildpacks.project.metadata' no default process type Saving my-image... *** Images (516b6641ecda): my-image Successfully built image my-image ⛄️ 3.1.4 🚀 /Users/rschneeman/Documents/projects/work/buildpacks/buildpacks-ruby (schneems/metrics-agent-download-rebase) $ docker run -it --rm --entrypoint='/cnb/lifecycle/launcher' my-image 'sleep 60' spawning agentmon pretend agentmon pretend agentmon pretend agentmon pretend agentmon pretend agentmon pretend agentmon pretend agentmon pretend agentmon pretend agentmon pretend agentmon pretend agentmon pretend agentmon pretend agentmon pretend agentmon pretend agentmon pretend agentmon pretend agentmon pretend agentmon ^[pretend agentmon ^C% ⛄️ 3.1.4 🚀 /Users/rschneeman/Documents/projects/work/buildpacks/buildpacks-ruby (schneems/metrics-agent-download-rebase) $ docker run -it --rm --entrypoint='/cnb/lifecycle/launcher' my-image 'bash' spawning agentmon pretend agentmon lol pretend agentmon echo lopretend agentmon l pretend agentmon ls pretend agentmon cd echpretend agentmon o "it never pretend agentmon ends" pretend agentmon pretend agentmon pretend agentmon pretend agentmon pretend agentmon pretend agentmon pretend agentmon pretend agentmon pretend agentmon ``` Here's the test directory (needs barnes in the Gemfile.lock to execute) ``` $ ls /tmp/47b6249d5e0a353f91910f848a700061 Gemfile Gemfile.lock ⛄️ 3.1.4 🚀 /tmp/47b6249d5e0a353f91910f848a700061 $ cat /tmp/47b6249d5e0a353f91910f848a700061/Gemfile source "https://rubygems.org" gem "barnes" ⛄️ 3.1.4 🚀 /tmp/47b6249d5e0a353f91910f848a700061 $ cat /tmp/47b6249d5e0a353f91910f848a700061/Gemfile.lock GEM remote: https://rubygems.org/ specs: barnes (0.0.9) multi_json (~> 1) statsd-ruby (~> 1.1) multi_json (1.15.0) statsd-ruby (1.5.0) PLATFORMS x86_64-darwin-22 DEPENDENCIES barnes BUNDLED WITH 2.4.15 ``` --- .../src/layers/download_agentmon.rs | 2 +- buildpacks/metrics-agent/src/layers/mod.rs | 1 + .../src/layers/spawn_agemntmon_execd.rs | 50 +++++++++++++++++++ buildpacks/metrics-agent/src/main.rs | 14 +++--- 4 files changed, 60 insertions(+), 7 deletions(-) create mode 100644 buildpacks/metrics-agent/src/layers/spawn_agemntmon_execd.rs diff --git a/buildpacks/metrics-agent/src/layers/download_agentmon.rs b/buildpacks/metrics-agent/src/layers/download_agentmon.rs index cd80ed8e..35831e0b 100644 --- a/buildpacks/metrics-agent/src/layers/download_agentmon.rs +++ b/buildpacks/metrics-agent/src/layers/download_agentmon.rs @@ -131,7 +131,7 @@ pub(crate) fn untar( } /// Sets file permissions on the given path to 7xx (similar to `chmod +x `) -fn chmod_plus_x(path: &Path) -> Result<(), std::io::Error> { +pub fn chmod_plus_x(path: &Path) -> Result<(), std::io::Error> { let mut perms = fs_err::metadata(path)?.permissions(); let mut mode = perms.mode(); octal_executable_permission(&mut mode); diff --git a/buildpacks/metrics-agent/src/layers/mod.rs b/buildpacks/metrics-agent/src/layers/mod.rs index 33f09cad..8517bf47 100644 --- a/buildpacks/metrics-agent/src/layers/mod.rs +++ b/buildpacks/metrics-agent/src/layers/mod.rs @@ -1 +1,2 @@ pub(crate) mod download_agentmon; +pub(crate) mod spawn_agemntmon_execd; diff --git a/buildpacks/metrics-agent/src/layers/spawn_agemntmon_execd.rs b/buildpacks/metrics-agent/src/layers/spawn_agemntmon_execd.rs new file mode 100644 index 00000000..88cb1b5c --- /dev/null +++ b/buildpacks/metrics-agent/src/layers/spawn_agemntmon_execd.rs @@ -0,0 +1,50 @@ +use crate::MetricsAgentBuildpack; +use libcnb::build::BuildContext; +use libcnb::data::layer_content_metadata::LayerTypes; +use libcnb::generic::GenericMetadata; +use libcnb::layer::{Layer, LayerResult, LayerResultBuilder}; +use libcnb::Buildpack; +use std::path::Path; + +use super::download_agentmon; + +pub(crate) struct SpawnAgentmonExecd; + +impl Layer for SpawnAgentmonExecd { + type Buildpack = MetricsAgentBuildpack; + type Metadata = GenericMetadata; + + fn types(&self) -> LayerTypes { + LayerTypes { + launch: true, + build: false, + cache: false, + } + } + + fn create( + &self, + _context: &BuildContext, + layer_path: &Path, + ) -> Result, ::Error> { + let path = layer_path.join("lol"); + // Intentionally leak background process + let script = r#"#!/usr/bin/env bash + + #!/usr/bin/env bash + + echo "spawning agentmon" & + + # Intentional leak of process + # https://superuser.com/questions/448445/run-bash-script-in-background-and-exit-terminal + while true; do echo "pretend agentmon"; sleep 2; done & + "#; + + fs_err::write(&path, script).unwrap(); + download_agentmon::chmod_plus_x(&path).unwrap(); + + LayerResultBuilder::new(GenericMetadata::default()) + .exec_d_program("spawn agentmon", path) + .build() + } +} diff --git a/buildpacks/metrics-agent/src/main.rs b/buildpacks/metrics-agent/src/main.rs index 81c5ac5f..597335c8 100644 --- a/buildpacks/metrics-agent/src/main.rs +++ b/buildpacks/metrics-agent/src/main.rs @@ -1,6 +1,7 @@ mod layers; use crate::layers::download_agentmon::{DownloadAgentmon, DownloadAgentmonError}; +use crate::layers::spawn_agemntmon_execd::SpawnAgentmonExecd; use commons::build_output::{self, fmt::ErrorInfo}; use indoc::formatdoc; use libcnb::{ @@ -50,11 +51,12 @@ impl Buildpack for MetricsAgentBuildpack { fn build(&self, context: BuildContext) -> libcnb::Result { let build_duration = build_output::buildpack_name("Heroku Statsd Metrics Agent"); - let section = build_output::section("Metrics agent"); - context.handle_layer( - layer_name!("statsd-metrics-agent"), - DownloadAgentmon { section }, - )?; + // let section = build_output::section("Metrics agent"); + // context.handle_layer( + // layer_name!("statsd-metrics-agent"), + // DownloadAgentmon { section }, + // )?; + context.handle_layer(layer_name!("layer_name"), SpawnAgentmonExecd)?; // TODO write launch script // @@ -72,7 +74,7 @@ impl Buildpack for MetricsAgentBuildpack { // echo "No agentmon executable found. Not starting." // fi - build_duration.done(); + build_duration.done_timed(); BuildResultBuilder::new().build() } From 707480611affe5f29b172649e20e5948fb8663b1 Mon Sep 17 00:00:00 2001 From: Richard Schneeman Date: Wed, 19 Jul 2023 13:45:15 -0500 Subject: [PATCH 04/40] Run agentmon with Ubuntu's start-stop-daemon ## Fix URL logic On the plane I assumed the URL was a redirect to the most current file. It is not. It returns a plain text body with the most recent URL. I fixed this logic. ## Cache download logic The URL is versioned. For example: ``` https://agentmon-releases.s3.amazonaws.com/agentmon-0.3.1-linux-amd64.tar.gz ``` So I used it as a cache key to not have to download the binary when the URL hasn't changed. ## Background process While my initial experiment wasn't successful David suggested to try out `start-stop-daemon` (though it is only supported on Ubuntu). The `start-stop-daemon` can successfully boot a background process and allow `execd` to return. For example this works: ```rust let background_script = layer_path.join("agentmon_script"); let execd_script = layer_path.join("agentmon_exec.d"); write_bash_script( &background_script, r#" while true; do echo 'pretend agentmon' >> /tmp/agentmon.txt sleep 2 done "#, ) .unwrap(); // Intentionally leak background process let background_script = background_script.canonicalize().unwrap(); let background_script = background_script.display(); write_bash_script( &execd_script, format!( r#"start-stop-daemon --start --background --exec "{background_script}""# ), ) .unwrap(); ``` With this output: ``` $ docker run -it --rm --entrypoint='/cnb/lifecycle/launcher' my-image 'bash' heroku@e3b22af1a126:/workspace$ heroku@e3b22af1a126:/workspace$ heroku@e3b22af1a126:/workspace$ ls /tmp agentmon.txt heroku@e3b22af1a126:/workspace$ tail -f /tmp/agentmon.txt pretend agentmon pretend agentmon pretend agentmon pretend agentmon pretend agentmon ``` As to why this is needed. From David: > [...] it appears that Go waits for all processes where PPID = launcher's PID (1), instead of the spawned PID. Still digging to understand why, since from my understanding of the code so far, that should not be the case. > launcher uses Cmd.Run() which does Cmd.Start() and then Cmd.Wait() > For Wait(), internally, Go does wait4(pid, &status, 0, &rusage): https://cs.opensource.google/go/go/+/refs/tags/go1.20.6:src/os/exec_unix.go;l=16-60 But as you can see, before that, it first calls blockUntilWaitable() which is implemented here for systems that have waitid(), such as Linux: https://cs.opensource.google/go/go/+/refs/tags/go1.20.6:src/os/wait_waitid.go The signature for that is int waitid(idtype_t idtype, id_t id, siginfo_t *infop, int options);, see https://man7.org/linux/man-pages/man2/waitpid.2.html > I am starting to wonder if this instead has to do with open FDs which the sleep() would also inherit, since launcher obviously reads from the process. I don't know why one works and the other doesn't. That needs more investigation. My immediate goal is to get agentmon working on CNBs. TODO: - Add an integration test - Move the while-loop agentmon bash script to a Rust script --- Cargo.lock | 310 +++++++++++++++- buildpacks/metrics-agent/Cargo.toml | 3 +- .../src/layers/download_agentmon.rs | 199 ----------- .../src/layers/install_agentmon.rs | 336 ++++++++++++++++++ buildpacks/metrics-agent/src/layers/mod.rs | 3 +- .../src/layers/spawn_agemntmon_execd.rs | 50 --- buildpacks/metrics-agent/src/main.rs | 30 +- 7 files changed, 648 insertions(+), 283 deletions(-) delete mode 100644 buildpacks/metrics-agent/src/layers/download_agentmon.rs create mode 100644 buildpacks/metrics-agent/src/layers/install_agentmon.rs delete mode 100644 buildpacks/metrics-agent/src/layers/spawn_agemntmon_execd.rs diff --git a/Cargo.lock b/Cargo.lock index 45755750..4643b048 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,6 +2,15 @@ # It is not intended for manual editing. version = 3 +[[package]] +name = "addr2line" +version = "0.21.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a30b2e23b9e17a9f90641c7ab1549cd9b44f296d3ccbf309d2863cfe398a0cb" +dependencies = [ + "gimli", +] + [[package]] name = "adler" version = "1.0.2" @@ -17,12 +26,38 @@ dependencies = [ "memchr", ] +[[package]] +name = "async-trait" +version = "0.1.73" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc00ceb34980c03614e35a3a4e218276a0a824e911d07651cd0d858a51e8c0f0" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.29", +] + [[package]] name = "autocfg" version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" +[[package]] +name = "backtrace" +version = "0.3.69" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2089b7e3f35b9dd2d0ed921ead4f6d318c27680d4a5bd167b3ee120edb105837" +dependencies = [ + "addr2line", + "cc", + "cfg-if", + "libc", + "miniz_oxide", + "object", + "rustc-demangle", +] + [[package]] name = "base64" version = "0.21.3" @@ -81,6 +116,42 @@ dependencies = [ "utf8-width", ] +[[package]] +name = "cached" +version = "0.44.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b195e4fbc4b6862bbd065b991a34750399c119797efff72492f28a5864de8700" +dependencies = [ + "async-trait", + "cached_proc_macro", + "cached_proc_macro_types", + "futures", + "hashbrown 0.13.2", + "instant", + "once_cell", + "thiserror", + "tokio", +] + +[[package]] +name = "cached_proc_macro" +version = "0.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b48814962d2fd604c50d2b9433c2a41a0ab567779ee2c02f7fba6eca1221f082" +dependencies = [ + "cached_proc_macro_types", + "darling", + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "cached_proc_macro_types" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a4f925191b4367301851c6d99b09890311d74b0d43f274c0b34c86d308a3663" + [[package]] name = "camino" version = "1.1.6" @@ -237,6 +308,41 @@ dependencies = [ "typenum", ] +[[package]] +name = "darling" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b750cb3417fd1b327431a470f388520309479ab0bf5e323505daf0290cd3850" +dependencies = [ + "darling_core", + "darling_macro", +] + +[[package]] +name = "darling_core" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "109c1ca6e6b7f82cc233a97004ea8ed7ca123a9af07a8230878fcfda9b158bf0" +dependencies = [ + "fnv", + "ident_case", + "proc-macro2", + "quote", + "strsim", + "syn 1.0.109", +] + +[[package]] +name = "darling_macro" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4aab4dbc9f7611d8b55048a3a16d2d010c2c8334e46304b40ac1cc14bf3b48e" +dependencies = [ + "darling_core", + "quote", + "syn 1.0.109", +] + [[package]] name = "digest" version = "0.10.7" @@ -352,6 +458,67 @@ version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" +[[package]] +name = "futures" +version = "0.3.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23342abe12aba583913b2e62f22225ff9c950774065e4bfb61a19cd9770fec40" +dependencies = [ + "futures-channel", + "futures-core", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-channel" +version = "0.3.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "955518d47e09b25bbebc7a18df10b81f0c766eaf4c4f1cccef2fca5f2a4fb5f2" +dependencies = [ + "futures-core", + "futures-sink", +] + +[[package]] +name = "futures-core" +version = "0.3.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4bca583b7e26f571124fe5b7561d49cb2868d79116cfa0eefce955557c6fee8c" + +[[package]] +name = "futures-io" +version = "0.3.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fff74096e71ed47f8e023204cfd0aa1289cd54ae5430a9523be060cdb849964" + +[[package]] +name = "futures-sink" +version = "0.3.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f43be4fe21a13b9781a69afa4985b0f6ee0e1afab2c6f454a8cf30e2b2237b6e" + +[[package]] +name = "futures-task" +version = "0.3.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "76d3d132be6c0e6aa1534069c705a74a5997a356c0dc2f86a47765e5617c5b65" + +[[package]] +name = "futures-util" +version = "0.3.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26b01e40b772d54cf6c6d721c1d1abd0647a0106a12ecaa1c186273392a69533" +dependencies = [ + "futures-core", + "futures-sink", + "futures-task", + "pin-project-lite", + "pin-utils", +] + [[package]] name = "generic-array" version = "0.14.7" @@ -373,12 +540,24 @@ dependencies = [ "wasi", ] +[[package]] +name = "gimli" +version = "0.28.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6fb8d784f27acf97159b40fc4db5ecd8aa23b9ad5ef69cdd136d3bc80665f0c0" + [[package]] name = "glob" version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b" +[[package]] +name = "hashbrown" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43a3c133739dddd0d2990f9a4bdf8eb4b21ef50e4851ca85ab661199821d510e" + [[package]] name = "hashbrown" version = "0.14.0" @@ -417,6 +596,7 @@ dependencies = [ name = "heroku-statsd-metrics" version = "0.0.0" dependencies = [ + "cached", "commons", "flate2", "fs-err", @@ -433,6 +613,12 @@ dependencies = [ "url", ] +[[package]] +name = "ident_case" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" + [[package]] name = "idna" version = "0.4.0" @@ -450,7 +636,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d5477fe2230a79769d8dc68e0eabf5437907c0457a5614a9e8dddb67f65eb65d" dependencies = [ "equivalent", - "hashbrown", + "hashbrown 0.14.0", ] [[package]] @@ -459,6 +645,15 @@ version = "2.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2c785eefb63ebd0e33416dfcb8d6da0bf27ce752843a45632a67bf10d4d4b5c4" +[[package]] +name = "instant" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a5bbe824c507c5da5956355e86a746d82e0e1464f65d862cc5e71da70e94b2c" +dependencies = [ + "cfg-if", +] + [[package]] name = "is_executable" version = "1.0.1" @@ -580,7 +775,7 @@ dependencies = [ "cargo_metadata 0.15.4", "fancy-regex", "quote", - "syn", + "syn 2.0.29", ] [[package]] @@ -592,7 +787,7 @@ dependencies = [ "cargo_metadata 0.17.0", "fancy-regex", "quote", - "syn", + "syn 2.0.29", ] [[package]] @@ -644,6 +839,16 @@ version = "0.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "57bcfdad1b858c2db7c38303a6d2ad4dfaf5eb53dfeb0910128b2c26d6158503" +[[package]] +name = "lock_api" +version = "0.4.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c1cc9717a20b1bb222f333e6a92fd32f7d8a18ddc5a3191a11af45dcbf4dcd16" +dependencies = [ + "autocfg", + "scopeguard", +] + [[package]] name = "log" version = "0.4.20" @@ -693,6 +898,15 @@ dependencies = [ "libc", ] +[[package]] +name = "object" +version = "0.32.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77ac5bbd07aea88c60a577a1ce218075ffd59208b2d7ca97adf9bfc5aeb21ebe" +dependencies = [ + "memchr", +] + [[package]] name = "once_cell" version = "1.18.0" @@ -708,6 +922,29 @@ dependencies = [ "num-traits", ] +[[package]] +name = "parking_lot" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3742b2c103b9f06bc9fff0a37ff4912935851bee6d36f3c02bcc755bcfec228f" +dependencies = [ + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93f00c865fe7cabf650081affecd3871070f26767e7b2070a3ffae14c654b447" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall", + "smallvec", + "windows-targets", +] + [[package]] name = "pathdiff" version = "0.2.1" @@ -730,6 +967,18 @@ dependencies = [ "indexmap", ] +[[package]] +name = "pin-project-lite" +version = "0.2.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8afb450f006bf6385ca15ef45d71d2288452bc3683ce2e2cacc0d18e4be60b58" + +[[package]] +name = "pin-utils" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" + [[package]] name = "pkg-config" version = "0.3.27" @@ -865,6 +1114,12 @@ dependencies = [ "winapi", ] +[[package]] +name = "rustc-demangle" +version = "0.1.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d626bb9dae77e28219937af045c257c28bfd3f69333c512553507f5f9798cb76" + [[package]] name = "rustix" version = "0.38.9" @@ -967,7 +1222,7 @@ checksum = "4eca7ac642d82aa35b60049a6eccb4be6be75e599bd2e9adb5f875a737654af2" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.29", ] [[package]] @@ -1001,6 +1256,12 @@ dependencies = [ "digest", ] +[[package]] +name = "smallvec" +version = "1.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62bb4feee49fdd9f707ef802e22365a35de4b7b299de4763d44bfea899442ff9" + [[package]] name = "spin" version = "0.5.2" @@ -1013,6 +1274,17 @@ version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" +[[package]] +name = "syn" +version = "1.0.109" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + [[package]] name = "syn" version = "2.0.29" @@ -1074,7 +1346,7 @@ checksum = "6bb623b56e39ab7dcd4b1b98bb6c8f8d907ed255b18de254088016b27a8ee19b" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.29", ] [[package]] @@ -1092,6 +1364,29 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" +[[package]] +name = "tokio" +version = "1.32.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "17ed6077ed6cd6c74735e21f37eb16dc3935f96878b1fe961074089cc80893f9" +dependencies = [ + "backtrace", + "parking_lot", + "pin-project-lite", + "tokio-macros", +] + +[[package]] +name = "tokio-macros" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "630bdcf245f78637c13ec01ffae6187cca34625e8c63150d424b59e55af2675e" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.29", +] + [[package]] name = "toml" version = "0.7.6" @@ -1194,6 +1489,7 @@ dependencies = [ "form_urlencoded", "idna", "percent-encoding", + "serde", ] [[package]] @@ -1251,7 +1547,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn", + "syn 2.0.29", "wasm-bindgen-shared", ] @@ -1273,7 +1569,7 @@ checksum = "54681b18a46765f095758388f2d0cf16eb8d4169b639ab575a8f5693af210c7b" dependencies = [ "proc-macro2", "quote", - "syn", + "syn 2.0.29", "wasm-bindgen-backend", "wasm-bindgen-shared", ] diff --git a/buildpacks/metrics-agent/Cargo.toml b/buildpacks/metrics-agent/Cargo.toml index 4da3a149..d1f89bac 100644 --- a/buildpacks/metrics-agent/Cargo.toml +++ b/buildpacks/metrics-agent/Cargo.toml @@ -19,5 +19,6 @@ tar = "0.4" tempfile = "3" thiserror = "1" ureq = "2" -url = "2" +url = { version = "2", features = ["serde"] } glob = "0.3" +cached = "0.44.0" diff --git a/buildpacks/metrics-agent/src/layers/download_agentmon.rs b/buildpacks/metrics-agent/src/layers/download_agentmon.rs deleted file mode 100644 index 35831e0b..00000000 --- a/buildpacks/metrics-agent/src/layers/download_agentmon.rs +++ /dev/null @@ -1,199 +0,0 @@ -use crate::{MetricsAgentBuildpack, MetricsAgentError}; -use libcnb::{ - generic::GenericMetadata, - layer::{Layer, LayerResultBuilder}, -}; -use std::os::unix::fs::PermissionsExt; -use url::Url; - -use crate::build_output; -use flate2::read::GzDecoder; -use libcnb::data::layer_content_metadata::LayerTypes; -use std::path::Path; -use tar::Archive; -use tempfile::NamedTempFile; -// use url::Url; - -#[derive(Debug)] -pub(crate) struct DownloadAgentmon { - pub(crate) section: build_output::Section, -} - -#[derive(thiserror::Error, Debug)] -pub(crate) enum DownloadAgentmonError { - #[error("Could not read file permissions {0}")] - PermissionError(std::io::Error), - - #[error("Could not parse url {0}")] - UrlParseError(url::ParseError), - - #[error("Could not open file: {0}")] - CouldNotOpenFile(std::io::Error), - - #[error("Could not untar: {0}")] - CouldNotUnpack(std::io::Error), - - // Boxed to prevent `large_enum_variant` errors since `ureq::Error` is massive. - #[error("Download error: {0}")] - RequestError(Box), - - #[error("Could not create file: {0}")] - CouldNotCreateDestinationFile(std::io::Error), - - #[error("Could not write file: {0}")] - CouldNotWriteDestinationFile(std::io::Error), -} - -impl Layer for DownloadAgentmon { - type Buildpack = MetricsAgentBuildpack; - type Metadata = GenericMetadata; - - fn types(&self) -> libcnb::data::layer_content_metadata::LayerTypes { - LayerTypes { - build: true, - launch: true, - cache: true, - } - } - - fn create( - &self, - _context: &libcnb::build::BuildContext, - layer_path: &std::path::Path, - ) -> Result< - libcnb::layer::LayerResult, - ::Error, - > { - let mut timer = self.section.say_with_inline_timer("Installing"); - - let agentmon_tgz = NamedTempFile::new() - .map_err(DownloadAgentmonError::CouldNotCreateDestinationFile) - .map_err(MetricsAgentError::DownloadAgentmonError)?; - - let url = Url::parse("https://agentmon-releases.s3.amazonaws.com/latest") - .map_err(DownloadAgentmonError::UrlParseError) - .map_err(MetricsAgentError::DownloadAgentmonError)?; - - download(url.as_ref(), agentmon_tgz.path()) - .map_err(MetricsAgentError::DownloadAgentmonError)?; - - let destination = layer_path.join("bin"); - untar(agentmon_tgz.path(), &destination) - .map_err(MetricsAgentError::DownloadAgentmonError)?; - - chmod_plus_x(&destination.join("agentmon")) - .map_err(DownloadAgentmonError::PermissionError) - .map_err(MetricsAgentError::DownloadAgentmonError)?; - - timer.done(); - - LayerResultBuilder::new(GenericMetadata::default()).build() - } - - fn existing_layer_strategy( - &self, - _context: &libcnb::build::BuildContext, - _layer_data: &libcnb::layer::LayerData, - ) -> Result::Error> - { - // TODO caching logic - // - // The classic buildpack actually downloads this binary on dyno boot every time - // We could use content headers to check if it needs to be re-downloaded. - Ok(libcnb::layer::ExistingLayerStrategy::Recreate) - } - - fn migrate_incompatible_metadata( - &self, - _context: &libcnb::build::BuildContext, - _metadata: &GenericMetadata, - ) -> Result< - libcnb::layer::MetadataMigration, - ::Error, - > { - self.section - .say_with_details("Clearing cache", "invalid metadata"); - - Ok(libcnb::layer::MetadataMigration::RecreateLayer) - } -} - -pub(crate) fn untar( - path: impl AsRef, - destination: impl AsRef, -) -> Result<(), DownloadAgentmonError> { - let file = - fs_err::File::open(path.as_ref()).map_err(DownloadAgentmonError::CouldNotOpenFile)?; - - Archive::new(GzDecoder::new(file)) - .unpack(destination.as_ref()) - .map_err(DownloadAgentmonError::CouldNotUnpack) -} - -/// Sets file permissions on the given path to 7xx (similar to `chmod +x `) -pub fn chmod_plus_x(path: &Path) -> Result<(), std::io::Error> { - let mut perms = fs_err::metadata(path)?.permissions(); - let mut mode = perms.mode(); - octal_executable_permission(&mut mode); - perms.set_mode(mode); - - fs_err::set_permissions(path, perms) -} - -/// Ensures the provided octal number's executable -/// bit is enabled. -/// -/// i.e. chmod +x will ensure that the first digit -/// of the file permission is 7 on unix so if you pass -/// in 0o455 it would be mutated to 0o755 -fn octal_executable_permission(mode: &mut u32) { - *mode |= 0o700; -} - -pub(crate) fn download( - uri: impl AsRef, - destination: impl AsRef, -) -> Result<(), DownloadAgentmonError> { - let mut response_reader = ureq::get(uri.as_ref()) - .call() - .map_err(|err| DownloadAgentmonError::RequestError(Box::new(err)))? - .into_reader(); - - let mut destination_file = fs_err::File::create(destination.as_ref()) - .map_err(DownloadAgentmonError::CouldNotCreateDestinationFile)?; - - std::io::copy(&mut response_reader, &mut destination_file) - .map_err(DownloadAgentmonError::CouldNotWriteDestinationFile)?; - - Ok(()) -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn test_chmod() { - let tmp = tempfile::tempdir().unwrap(); - let file = tmp.into_path().join("file"); - std::fs::write(&file, "lol").unwrap(); - - let before = file.metadata().unwrap().permissions().mode(); - chmod_plus_x(&file).unwrap(); - let after = file.metadata().unwrap().permissions().mode(); - assert!(before != after); - } - - #[test] - fn test_executable_logic() { - // Sets executable bit - let mut mode = 0o455; - octal_executable_permission(&mut mode); - assert_eq!(0o755, mode); - - // Does not affect already executable - let mut mode = 0o745; - octal_executable_permission(&mut mode); - assert_eq!(0o745, mode); - } -} diff --git a/buildpacks/metrics-agent/src/layers/install_agentmon.rs b/buildpacks/metrics-agent/src/layers/install_agentmon.rs new file mode 100644 index 00000000..29b0505d --- /dev/null +++ b/buildpacks/metrics-agent/src/layers/install_agentmon.rs @@ -0,0 +1,336 @@ +use crate::build_output; +use crate::{MetricsAgentBuildpack, MetricsAgentError}; +use cached::proc_macro::cached; +use flate2::read::GzDecoder; +use libcnb::data::layer_content_metadata::LayerTypes; +use libcnb::layer::ExistingLayerStrategy; +use libcnb::{ + generic::GenericMetadata, + layer::{Layer, LayerResultBuilder}, +}; +use serde::{Deserialize, Serialize}; +use std::os::unix::fs::PermissionsExt; +use std::path::Path; +use std::path::PathBuf; +use tar::Archive; +use tempfile::NamedTempFile; +use url::Url; + +#[derive(Debug)] +pub(crate) struct InstallAgentmon { + pub(crate) section: build_output::Section, +} + +#[derive(Deserialize, Serialize, Debug, Clone)] +pub(crate) struct Metadata { + download_url: Option, +} + +// All cloneable subtypes to make cachable +#[derive(thiserror::Error, Debug, Clone)] +pub(crate) enum GetUrlError { + #[error("Response successful, but body not in the form of a URL: {0}")] + CannotConvertResponseToString(String), + + #[error("Cannot parse url: {0}")] + UrlParseError(url::ParseError), + + // Boxed to prevent `large_enum_variant` errors since `ureq::Error` is massive. + #[error("Network error while retrieving the url: {0}")] + RequestError(String), +} + +#[derive(thiserror::Error, Debug)] +pub(crate) enum DownloadAgentmonError { + #[error("Could not determine the url of the latest agentmont release.\n{0}")] + CannotGetLatestUrl(GetUrlError), + + #[error("Could not read file permissions {0}")] + PermissionError(std::io::Error), + + #[error("Could not open file: {0}")] + CouldNotOpenFile(std::io::Error), + + #[error("Could not untar: {0}")] + CouldNotUnpack(std::io::Error), + + // Boxed to prevent `large_enum_variant` errors since `ureq::Error` is massive. + #[error("Download error: {0}")] + RequestError(Box), + + #[error("Could not create file: {0}")] + CouldNotCreateDestinationFile(std::io::Error), + + #[error("Could not write file: {0}")] + CouldNotWriteDestinationFile(std::io::Error), +} + +impl Layer for InstallAgentmon { + type Buildpack = MetricsAgentBuildpack; + type Metadata = Metadata; + + fn types(&self) -> libcnb::data::layer_content_metadata::LayerTypes { + LayerTypes { + build: true, + launch: true, + cache: true, + } + } + + fn create( + &self, + _context: &libcnb::build::BuildContext, + layer_path: &std::path::Path, + ) -> Result< + libcnb::layer::LayerResult, + ::Error, + > { + let destination_dir = layer_path.join("bin"); + let executable = destination_dir.join("agentmon"); + + let mut timer = self.section.say_with_inline_timer("Downloading"); + + let url = get_latest_url() + .map_err(DownloadAgentmonError::CannotGetLatestUrl) + .map_err(MetricsAgentError::DownloadAgentmonError)?; + + download_to_dir(&destination_dir, &url) + .map_err(MetricsAgentError::DownloadAgentmonError)?; + timer.done(); + + self.section.say("Writing scripts"); + let execd = write_execd(&executable, layer_path) + .map_err(MetricsAgentError::DownloadAgentmonError)?; + + LayerResultBuilder::new(Metadata { + download_url: Some(url), + }) + .exec_d_program("spawn agentmon", execd) + .build() + } + + fn existing_layer_strategy( + &self, + _context: &libcnb::build::BuildContext, + layer_data: &libcnb::layer::LayerData, + ) -> Result::Error> + { + if let Some(old_url) = &layer_data.content_metadata.metadata.download_url { + let url = get_latest_url() + .map_err(DownloadAgentmonError::CannotGetLatestUrl) + .map_err(MetricsAgentError::DownloadAgentmonError)?; + + if old_url == &url { + self.section.say("Using cache"); + Ok(ExistingLayerStrategy::Keep) + } else { + let url = build_output::fmt::value(url); + self.section + .say_with_details("Clearing cache", format!("Updated url {url}")); + Ok(ExistingLayerStrategy::Recreate) + } + } else { + self.section + .say_with_details("Clearing cache", "No url found in metadata"); + + Ok(ExistingLayerStrategy::Recreate) + } + } + + fn migrate_incompatible_metadata( + &self, + _context: &libcnb::build::BuildContext, + _metadata: &GenericMetadata, + ) -> Result< + libcnb::layer::MetadataMigration, + ::Error, + > { + self.section + .say_with_details("Clearing cache", "invalid metadata"); + + Ok(libcnb::layer::MetadataMigration::RecreateLayer) + } +} + +fn write_execd(agentmon_path: &Path, layer_path: &Path) -> Result { + let agentmon_path = agentmon_path + .canonicalize() + .map_err(DownloadAgentmonError::CouldNotOpenFile)?; + let agentmon_path = agentmon_path.display(); + + // This script boots and runs agentmon in a loop + let background_script = { + let script = layer_path.join("agentmon_script"); + write_bash_script( + &script, + // Curly braces need to be escaped with another curly + format!( + r#" + setup_metrics() {{ + if [[ -z "$HEROKU_METRICS_URL" ]] || [[ "${{DYNO}}" = run\.* ]]; then + return 0 + fi + + AGENTMON_FLAGS=("-statsd-addr=:${{PORT}}") + + if [[ "${{AGENTMON_DEBUG}}" = "true" ]]; then + AGENTMON_FLAGS+=("-debug") + fi + + if [[ -x "{agentmon_path}" ]]; then + (while true; do + {agentmon_path} "${{AGENTMON_FLAGS[@]}}" "${{HEROKU_METRICS_URL}}" + echo "agentmon completed with status=${{?}}. Restarting" + sleep 1 + done) & + else + echo "No agentmon executable found. Not starting." + fi + }} + + setup_metrics + "# + ), + ) + .map_err(DownloadAgentmonError::CouldNotWriteDestinationFile)?; + + script + .canonicalize() + .map_err(DownloadAgentmonError::CouldNotOpenFile)? + }; + + // We use the exec.d to boot a process. This script MUST exit though as otherwise + // The container would never boot. To handle this we intentionally leak a process + let execd_script = { + let script = layer_path.join("agentmon_exec.d"); + + let background_script = background_script.display(); + write_bash_script( + &script, + format!(r#"start-stop-daemon --start --background --exec "{background_script}""#), + ) + .map_err(DownloadAgentmonError::CouldNotWriteDestinationFile)?; + + script + }; + + Ok(execd_script) +} + +#[cached] +fn get_latest_url() -> Result { + // This file on S3 stores a raw string that holds the URL to the latest agentmon release + // It's not a redirect to the latest file, it's a string body that contains a URL. + let base = Url::parse("https://agentmon-releases.s3.amazonaws.com/latest") + .expect("Internal error: Bad url"); + + let body = ureq::get(base.as_ref()) + .call() + .map_err(|err| GetUrlError::RequestError(err.to_string()))? + .into_string() + .map_err(|error| GetUrlError::CannotConvertResponseToString(error.to_string()))?; + + Url::parse(body.as_str().trim()).map_err(GetUrlError::UrlParseError) +} + +fn download_to_dir(destination: &Path, url: &Url) -> Result<(), DownloadAgentmonError> { + let agentmon_tgz = + NamedTempFile::new().map_err(DownloadAgentmonError::CouldNotCreateDestinationFile)?; + + download(url.as_ref(), agentmon_tgz.path())?; + + untar(agentmon_tgz.path(), destination)?; + + chmod_plus_x(&destination.join("agentmon")).map_err(DownloadAgentmonError::PermissionError)?; + + Ok(()) +} + +pub(crate) fn untar( + path: impl AsRef, + destination: impl AsRef, +) -> Result<(), DownloadAgentmonError> { + let file = + fs_err::File::open(path.as_ref()).map_err(DownloadAgentmonError::CouldNotOpenFile)?; + + Archive::new(GzDecoder::new(file)) + .unpack(destination.as_ref()) + .map_err(DownloadAgentmonError::CouldNotUnpack) +} + +/// Sets file permissions on the given path to 7xx (similar to `chmod +x `) +pub fn chmod_plus_x(path: &Path) -> Result<(), std::io::Error> { + let mut perms = fs_err::metadata(path)?.permissions(); + let mut mode = perms.mode(); + octal_executable_permission(&mut mode); + perms.set_mode(mode); + + fs_err::set_permissions(path, perms) +} + +/// Write a script to the target path while adding a bash shebang line and setting execution permissions +fn write_bash_script(path: &Path, script: impl AsRef) -> std::io::Result<()> { + let script = script.as_ref(); + fs_err::write(path, format!("#!/usr/bin/env bash\n\n{script}"))?; + chmod_plus_x(path)?; + + Ok(()) +} + +/// Ensures the provided octal number's executable +/// bit is enabled. +/// +/// i.e. chmod +x will ensure that the first digit +/// of the file permission is 7 on unix so if you pass +/// in 0o455 it would be mutated to 0o755 +fn octal_executable_permission(mode: &mut u32) { + *mode |= 0o700; +} + +pub(crate) fn download( + uri: impl AsRef, + destination: impl AsRef, +) -> Result<(), DownloadAgentmonError> { + let mut response_reader = ureq::get(uri.as_ref()) + .call() + .map_err(|err| DownloadAgentmonError::RequestError(Box::new(err)))? + .into_reader(); + + let mut destination_file = fs_err::File::create(destination.as_ref()) + .map_err(DownloadAgentmonError::CouldNotCreateDestinationFile)?; + + std::io::copy(&mut response_reader, &mut destination_file) + .map_err(DownloadAgentmonError::CouldNotWriteDestinationFile)?; + + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_chmod() { + let tmp = tempfile::tempdir().unwrap(); + let file = tmp.into_path().join("file"); + std::fs::write(&file, "lol").unwrap(); + + let before = file.metadata().unwrap().permissions().mode(); + chmod_plus_x(&file).unwrap(); + let after = file.metadata().unwrap().permissions().mode(); + assert!(before != after); + } + + #[test] + fn test_executable_logic() { + // Sets executable bit + let mut mode = 0o455; + octal_executable_permission(&mut mode); + assert_eq!(0o755, mode); + + // Does not affect already executable + let mut mode = 0o745; + octal_executable_permission(&mut mode); + assert_eq!(0o745, mode); + } +} diff --git a/buildpacks/metrics-agent/src/layers/mod.rs b/buildpacks/metrics-agent/src/layers/mod.rs index 8517bf47..96155e8c 100644 --- a/buildpacks/metrics-agent/src/layers/mod.rs +++ b/buildpacks/metrics-agent/src/layers/mod.rs @@ -1,2 +1 @@ -pub(crate) mod download_agentmon; -pub(crate) mod spawn_agemntmon_execd; +pub(crate) mod install_agentmon; diff --git a/buildpacks/metrics-agent/src/layers/spawn_agemntmon_execd.rs b/buildpacks/metrics-agent/src/layers/spawn_agemntmon_execd.rs deleted file mode 100644 index 88cb1b5c..00000000 --- a/buildpacks/metrics-agent/src/layers/spawn_agemntmon_execd.rs +++ /dev/null @@ -1,50 +0,0 @@ -use crate::MetricsAgentBuildpack; -use libcnb::build::BuildContext; -use libcnb::data::layer_content_metadata::LayerTypes; -use libcnb::generic::GenericMetadata; -use libcnb::layer::{Layer, LayerResult, LayerResultBuilder}; -use libcnb::Buildpack; -use std::path::Path; - -use super::download_agentmon; - -pub(crate) struct SpawnAgentmonExecd; - -impl Layer for SpawnAgentmonExecd { - type Buildpack = MetricsAgentBuildpack; - type Metadata = GenericMetadata; - - fn types(&self) -> LayerTypes { - LayerTypes { - launch: true, - build: false, - cache: false, - } - } - - fn create( - &self, - _context: &BuildContext, - layer_path: &Path, - ) -> Result, ::Error> { - let path = layer_path.join("lol"); - // Intentionally leak background process - let script = r#"#!/usr/bin/env bash - - #!/usr/bin/env bash - - echo "spawning agentmon" & - - # Intentional leak of process - # https://superuser.com/questions/448445/run-bash-script-in-background-and-exit-terminal - while true; do echo "pretend agentmon"; sleep 2; done & - "#; - - fs_err::write(&path, script).unwrap(); - download_agentmon::chmod_plus_x(&path).unwrap(); - - LayerResultBuilder::new(GenericMetadata::default()) - .exec_d_program("spawn agentmon", path) - .build() - } -} diff --git a/buildpacks/metrics-agent/src/main.rs b/buildpacks/metrics-agent/src/main.rs index 597335c8..4e8f2345 100644 --- a/buildpacks/metrics-agent/src/main.rs +++ b/buildpacks/metrics-agent/src/main.rs @@ -1,7 +1,6 @@ mod layers; -use crate::layers::download_agentmon::{DownloadAgentmon, DownloadAgentmonError}; -use crate::layers::spawn_agemntmon_execd::SpawnAgentmonExecd; +use crate::layers::install_agentmon::{DownloadAgentmonError, InstallAgentmon}; use commons::build_output::{self, fmt::ErrorInfo}; use indoc::formatdoc; use libcnb::{ @@ -51,28 +50,11 @@ impl Buildpack for MetricsAgentBuildpack { fn build(&self, context: BuildContext) -> libcnb::Result { let build_duration = build_output::buildpack_name("Heroku Statsd Metrics Agent"); - // let section = build_output::section("Metrics agent"); - // context.handle_layer( - // layer_name!("statsd-metrics-agent"), - // DownloadAgentmon { section }, - // )?; - context.handle_layer(layer_name!("layer_name"), SpawnAgentmonExecd)?; - - // TODO write launch script - // - // if [[ "${AGENTMON_DEBUG}" = "true" ]]; then - // AGENTMON_FLAGS+=("-debug") - // fi - - // if [[ -x "${BUILD_DIR}/bin/agentmon" ]]; then - // (while true; do - // ${BUILD_DIR}/bin/agentmon "${AGENTMON_FLAGS[@]}" "${HEROKU_METRICS_URL}" - // echo "agentmon completed with status=${?}. Restarting" - // sleep 1 - // done) & - // else - // echo "No agentmon executable found. Not starting." - // fi + let section = build_output::section("Metrics agent"); + context.handle_layer( + layer_name!("statsd-metrics-agent"), + InstallAgentmon { section }, + )?; build_duration.done_timed(); BuildResultBuilder::new().build() From 669ef6f874149343da8eef64b105c25661ff2c4b Mon Sep 17 00:00:00 2001 From: Richard Schneeman Date: Wed, 19 Jul 2023 13:58:04 -0500 Subject: [PATCH 05/40] Fix spelling --- buildpacks/ruby/src/user_errors.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/buildpacks/ruby/src/user_errors.rs b/buildpacks/ruby/src/user_errors.rs index 0af7e170..81a39dc9 100644 --- a/buildpacks/ruby/src/user_errors.rs +++ b/buildpacks/ruby/src/user_errors.rs @@ -65,7 +65,7 @@ fn log_our_error(error: RubyBuildpackError) { RubyBuildpackError::InAppDirCacheError(error) => ErrorInfo::header_body_details( "Internal cache error", formatdoc! {" - An internal error occured while caching files. + An internal error occurred while caching files. "}, error, ) @@ -95,7 +95,7 @@ fn log_our_error(error: RubyBuildpackError) { RubyBuildpackError::RakeAssetsPrecompileFailed(error) => ErrorInfo::header_body_details( "Asset compilation failed", formatdoc! {" - An error occured while compiling assets via rake command. + An error occurred while compiling assets via rake command. "}, error, ) From f247694e5107a4d0b38706415dd145848b1ece91 Mon Sep 17 00:00:00 2001 From: Richard Schneeman Date: Wed, 19 Jul 2023 17:12:17 -0500 Subject: [PATCH 06/40] Move background loop logic from bash to Rust MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ``` $ cargo libcnb package && docker rmi my-image --force && pack build my-image --buildpack target/buildpack/debug/heroku_statsd-metrics --path /tmp/47b6249d5e0a353f91910f848a700061/ --pull-policy never 🔍 Locating buildpacks... 📦 [1/2] Building heroku/ruby Determining automatic cross-compile settings... Building binaries (x86_64-unknown-linux-musl)... Finished dev [unoptimized] target(s) in 0.48s Writing buildpack directory... Successfully wrote buildpack directory: ../../target/buildpack/debug/heroku_ruby (12.64 MiB) 📦 [2/2] Building heroku/statsd-metrics Determining automatic cross-compile settings... Building binaries (x86_64-unknown-linux-musl)... Blocking waiting for file lock on package cache Blocking waiting for file lock on package cache Finished dev [unoptimized] target(s) in 0.68s Finished dev [unoptimized] target(s) in 0.51s Writing buildpack directory... Successfully wrote buildpack directory: ../../target/buildpack/debug/heroku_statsd-metrics (12.15 MiB) ✨ Packaging successfully finished! 💡 To test your buildpack locally with pack, run: pack build my-image-name \ --buildpack /Users/rschneeman/Documents/projects/work/buildpacks/buildpacks-ruby/target/buildpack/debug/heroku_ruby \ --buildpack /Users/rschneeman/Documents/projects/work/buildpacks/buildpacks-ruby/target/buildpack/debug/heroku_statsd-metrics \ --path /path/to/application /Users/rschneeman/Documents/projects/work/buildpacks/buildpacks-ruby/target/buildpack/debug/heroku_ruby /Users/rschneeman/Documents/projects/work/buildpacks/buildpacks-ruby/target/buildpack/debug/heroku_statsd-metrics Untagged: my-image:latest Deleted: sha256:b73cef10f9747c93c0589f68bd30b76298f57a178c9e3efb68dbe77a9bd70cdc Deleted: sha256:9c2fb27824ec55b3e82d5245a623de6ae85ffbcfd91d9ca93cda6e220fb4ae28 Deleted: sha256:0864c2235caabe94562658b4003c64d2834a8d15915bffdb080d9d6d39d37ad1 Deleted: sha256:e0f324b7867aecebbe166b6a773859cd8a9779aed246c0e6029430b5558bfc95 Deleted: sha256:c578d8034a4c45399cf1a55207eb25d07464681fe44caf88b94b6082dee74df8 Deleted: sha256:d45865c0091f9fde9ec70ba62d1d8dd48b3cbb6bcad730be8fed56344bc36ae3 ===> ANALYZING Image with name "my-image" not found ===> DETECTING heroku/statsd-metrics 2.0.0 ===> RESTORING ===> BUILDING # Heroku Statsd Metrics Agent - Metrics agent - Downloading ..... (2.596s) - Writing scripts - Done (finished in 2.602s) ===> EXPORTING Adding layer 'heroku/statsd-metrics:statsd-metrics-agent' Adding layer 'buildpacksio/lifecycle:launch.sbom' Adding 1/1 app layer(s) Adding layer 'buildpacksio/lifecycle:launcher' Adding layer 'buildpacksio/lifecycle:config' Adding label 'io.buildpacks.lifecycle.metadata' Adding label 'io.buildpacks.build.metadata' Adding label 'io.buildpacks.project.metadata' no default process type Saving my-image... *** Images (b73cef10f974): my-image Reusing cache layer 'heroku/statsd-metrics:statsd-metrics-agent' Successfully built image my-image ⛄️ 3.1.4 🚀 /Users/rschneeman/Documents/projects/work/buildpacks/buildpacks-ruby (schneems/metrics-agent-download-rebase) $ docker run -it --rm --entrypoint='/cnb/lifecycle/launcher' -e HEROKU_METRICS_URL=https://example.com my-image 'bash' heroku@1bb0e5cf43f5:/workspace$ ps -aux USER PID %CPU %MEM VSZ RSS TTY STAT START TIME COMMAND heroku 1 2.0 0.0 4624 3792 pts/0 Ss 22:18 0:00 bash heroku 14 11.3 0.0 2176 4 ? S 22:18 0:00 /layers/heroku_statsd-metrics/statsd-metrics-agent/agentmon_loop --path /layers/heroku_statsd-metrics/statsd-metrics-agent/b heroku 16 0.0 0.0 7060 1584 pts/0 R+ 22:18 0:00 ps -aux ``` Manually running the script seems to work: ``` heroku@1bb0e5cf43f5:/workspace$ PORT=3000 /layers/heroku_statsd-metrics/statsd-metrics-agent/agentmon_loop --path /layers/heroku_statsd-metrics/statsd-metrics-agent/bin/agentmon agentmon: Listening on :3000... ``` I'm not sure how we test this. --- Cargo.lock | 167 +++++++++++++++++- Cargo.toml | 8 + buildpacks/metrics-agent/Cargo.toml | 6 +- .../metrics-agent/src/bin/agentmon_loop.rs | 79 +++++++++ .../src/layers/install_agentmon.rs | 41 +---- 5 files changed, 264 insertions(+), 37 deletions(-) create mode 100644 buildpacks/metrics-agent/src/bin/agentmon_loop.rs diff --git a/Cargo.lock b/Cargo.lock index 4643b048..7119820d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -26,6 +26,54 @@ dependencies = [ "memchr", ] +[[package]] +name = "anstream" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1f58811cfac344940f1a400b6e6231ce35171f614f26439e80f8c1465c5cc0c" +dependencies = [ + "anstyle", + "anstyle-parse", + "anstyle-query", + "anstyle-wincon", + "colorchoice", + "utf8parse", +] + +[[package]] +name = "anstyle" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15c4c2c83f81532e5845a733998b6971faca23490340a418e9b72a3ec9de12ea" + +[[package]] +name = "anstyle-parse" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "938874ff5980b03a87c5524b3ae5b59cf99b1d6bc836848df7bc5ada9643c333" +dependencies = [ + "utf8parse", +] + +[[package]] +name = "anstyle-query" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5ca11d4be1bab0c8bc8734a9aa7bf4ee8316d462a08c6ac5052f888fef5b494b" +dependencies = [ + "windows-sys", +] + +[[package]] +name = "anstyle-wincon" +version = "2.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "58f54d10c6dfa51283a066ceab3ec1ab78d13fae00aa49243a45e4571fb79dfd" +dependencies = [ + "anstyle", + "windows-sys", +] + [[package]] name = "async-trait" version = "0.1.73" @@ -213,6 +261,53 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" +[[package]] +name = "clap" +version = "4.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c8d502cbaec4595d2e7d5f61e318f05417bd2b66fdc3809498f0d3fdf0bea27" +dependencies = [ + "clap_builder", + "clap_derive", + "once_cell", +] + +[[package]] +name = "clap_builder" +version = "4.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5891c7bc0edb3e1c2204fc5e94009affabeb1821c9e5fdc3959536c5c0bb984d" +dependencies = [ + "anstream", + "anstyle", + "clap_lex", + "strsim", +] + +[[package]] +name = "clap_derive" +version = "4.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c9fd1a5729c4548118d7d70ff234a44868d00489a4b6597b0b020918a0e91a1a" +dependencies = [ + "heck", + "proc-macro2", + "quote", + "syn 2.0.29", +] + +[[package]] +name = "clap_lex" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd7cc57abe963c6d3b9d8be5b06ba7c8957a930305ca90304f24ef040aa6f961" + +[[package]] +name = "colorchoice" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "acbf1af155f9b9ef647e42cdc158db4b64a1b61f743629225fde6f3e0be2a7c7" + [[package]] name = "commons" version = "0.1.0" @@ -226,7 +321,7 @@ dependencies = [ "indoc", "lazy_static", "libcnb 0.14.0", - "libherokubuildpack", + "libherokubuildpack 0.14.0", "regex", "serde", "sha2", @@ -564,6 +659,12 @@ version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2c6201b9ff9fd90a5a3bac2e56a830d0caa509576f0e503818ee82c181b3437a" +[[package]] +name = "heck" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" + [[package]] name = "hermit-abi" version = "0.3.2" @@ -597,12 +698,14 @@ name = "heroku-statsd-metrics" version = "0.0.0" dependencies = [ "cached", + "clap", "commons", "flate2", "fs-err", "glob", "indoc", "libcnb 0.13.0", + "libherokubuildpack 0.12.0", "rand", "regex", "serde", @@ -699,6 +802,19 @@ version = "0.2.147" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b4668fb0ea861c1df094127ac5f1da3409a82116a4ba74fca2e58ef927159bb3" +[[package]] +name = "libcnb" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "027cd4a736600564c4e7aebf124eabb9d7dc622bcfeefb414cc7c4c7d7ac6595" +dependencies = [ + "libcnb-data 0.12.0", + "libcnb-proc-macros 0.12.0", + "serde", + "thiserror", + "toml", +] + [[package]] name = "libcnb" version = "0.13.0" @@ -725,6 +841,19 @@ dependencies = [ "toml", ] +[[package]] +name = "libcnb-data" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e8840246c7aced3307fa193edc5d26482f92f992986a33860b6b3b523a67975" +dependencies = [ + "fancy-regex", + "libcnb-proc-macros 0.12.0", + "serde", + "thiserror", + "toml", +] + [[package]] name = "libcnb-data" version = "0.13.0" @@ -766,6 +895,18 @@ dependencies = [ "which", ] +[[package]] +name = "libcnb-proc-macros" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c9e31cc93a20d00f1d54ecb55dcff681669871e6d8a8b63ac0320e77fca1987c" +dependencies = [ + "cargo_metadata 0.15.4", + "fancy-regex", + "quote", + "syn 2.0.29", +] + [[package]] name = "libcnb-proc-macros" version = "0.13.0" @@ -804,6 +945,24 @@ dependencies = [ "tempfile", ] +[[package]] +name = "libherokubuildpack" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ee2764ebf688454c4fcbcd7c52ff277b5cb2e196c502ff4ce92de563cb10ea2" +dependencies = [ + "crossbeam-utils", + "flate2", + "libcnb 0.12.0", + "pathdiff", + "sha2", + "tar", + "termcolor", + "thiserror", + "toml", + "ureq", +] + [[package]] name = "libherokubuildpack" version = "0.14.0" @@ -1498,6 +1657,12 @@ version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5190c9442dcdaf0ddd50f37420417d219ae5261bbf5db120d0f9bab996c9cba1" +[[package]] +name = "utf8parse" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "711b9620af191e0cdc7468a8d14e709c3dcdb115b36f838e601583af800a370a" + [[package]] name = "vcpkg" version = "0.2.15" diff --git a/Cargo.toml b/Cargo.toml index 03e9109b..0a609cf1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -5,3 +5,11 @@ members = [ "buildpacks/ruby", "commons" ] + +[workspace.package] +rust-version = "1.64" +edition = "2021" +license = "BSD-3-Clause" + +[workspace.dependencies] +libherokubuildpack = "0.12" diff --git a/buildpacks/metrics-agent/Cargo.toml b/buildpacks/metrics-agent/Cargo.toml index d1f89bac..f6b7993d 100644 --- a/buildpacks/metrics-agent/Cargo.toml +++ b/buildpacks/metrics-agent/Cargo.toml @@ -3,8 +3,8 @@ name = "heroku-statsd-metrics" # This crate is not published, so the only version that is used is the one in buildpack.toml. version = "0.0.0" publish = false -edition = "2021" -rust-version = "1.66" +edition.workspace = true +rust-version.workspace = true [dependencies] commons = { path = "../../commons" } @@ -22,3 +22,5 @@ ureq = "2" url = { version = "2", features = ["serde"] } glob = "0.3" cached = "0.44.0" +clap = { version = "4.3.17", features = ["derive"] } +libherokubuildpack.workspace = true diff --git a/buildpacks/metrics-agent/src/bin/agentmon_loop.rs b/buildpacks/metrics-agent/src/bin/agentmon_loop.rs new file mode 100644 index 00000000..98067832 --- /dev/null +++ b/buildpacks/metrics-agent/src/bin/agentmon_loop.rs @@ -0,0 +1,79 @@ +// Enable Clippy lints that are disabled by default. +// https://rust-lang.github.io/rust-clippy/stable/index.html +#![warn(clippy::pedantic)] + +use clap::Parser; +use commons::fun_run::CmdMapExt; +use std::{ + ffi::OsString, + path::{Path, PathBuf}, + process::{exit, Command}, + time::Duration, +}; + +/// Simple program to greet a person +#[derive(Parser, Debug)] +struct Args { + /// Name of the person to greet + #[arg(short, long)] + path: PathBuf, +} + +fn main() { + let sleep_for = std::time::Duration::from_secs(1); + let heroku_metrics_url = if let Some(url) = std::env::var_os("HEROKU_METRICS_URL") { + url + } else { + eprintln!("Metrics agent exiting: 0 (HEROKU_METRICS_URL is not set)"); + exit(0) + }; + + if let Some(true) = + std::env::var_os("DYNO").map(|value| value.to_string_lossy().starts_with("run.")) + { + eprintln!("Metrics agent exiting: 0 (one off dyno detected i.e. DYNO=\"run.*\")"); + exit(0) + } + + let agentmon = Args::parse().path; + if !agentmon.exists() { + eprintln!("Path does not exist {}", agentmon.display()); + exit(1); + } + + loop { + run_agentmon(&agentmon, &heroku_metrics_url, &sleep_for); + } +} + +fn run_agentmon(agentmon: &Path, heroku_metrics_url: &OsString, sleep_for: &Duration) { + if let Some(port) = std::env::var_os("PORT") { + let statsd_addr = { + let mut string = OsString::from("statsd-addr=:"); + string.push(&port); + string + }; + + let result = Command::new(agentmon).cmd_map(|cmd| { + cmd.arg(&statsd_addr); + + if let Some(true) = std::env::var_os("AGENTMON_DEBUG").map(|value| value == *"true") { + cmd.arg("-debug"); + }; + cmd.arg(heroku_metrics_url); + + cmd.spawn().and_then(|mut child| child.wait()) + }); + + match result { + Ok(status) => eprintln!("agentmon completed with status=${status}. Restarting"), + Err(error) => { + eprintln!("agentmon could not be run due to error: {error}"); + eprintln!("Retrying"); + } + }; + } else { + eprintln!("PORT is not set, sleeping {sleep_for:?}"); + } + std::thread::sleep(*sleep_for); +} diff --git a/buildpacks/metrics-agent/src/layers/install_agentmon.rs b/buildpacks/metrics-agent/src/layers/install_agentmon.rs index 29b0505d..74aa0682 100644 --- a/buildpacks/metrics-agent/src/layers/install_agentmon.rs +++ b/buildpacks/metrics-agent/src/layers/install_agentmon.rs @@ -5,6 +5,7 @@ use flate2::read::GzDecoder; use libcnb::data::layer_content_metadata::LayerTypes; use libcnb::layer::ExistingLayerStrategy; use libcnb::{ + additional_buildpack_binary_path, generic::GenericMetadata, layer::{Layer, LayerResultBuilder}, }; @@ -160,39 +161,11 @@ fn write_execd(agentmon_path: &Path, layer_path: &Path) -> Result Result Date: Wed, 9 Aug 2023 11:37:53 -0500 Subject: [PATCH 07/40] Add metrics integration test with WIP libcnb-test feature Asserts that an app build with the heroku statsd metrics buildpack that has `barnes` in the Gemfile.lock will have a process `agentmon_loop` running in the background on dyne boot with both HEROKU_METRICS_URL and DYNO env vars are present at boot. --- Cargo.lock | 519 +++++++++++++++++- buildpacks/metrics-agent/Cargo.toml | 4 + buildpacks/metrics-agent/src/main.rs | 2 +- .../tests/fixtures/barnes_app/Gemfile | 4 + .../tests/fixtures/barnes_app/Gemfile.lock | 23 + .../tests/fixtures/barnes_app/config.ru | 1 + .../tests/fixtures/barnes_app/config/puma.rb | 9 + .../metrics-agent/tests/integration_test.rs | 41 ++ 8 files changed, 594 insertions(+), 9 deletions(-) create mode 100644 buildpacks/metrics-agent/tests/fixtures/barnes_app/Gemfile create mode 100644 buildpacks/metrics-agent/tests/fixtures/barnes_app/Gemfile.lock create mode 100644 buildpacks/metrics-agent/tests/fixtures/barnes_app/config.ru create mode 100644 buildpacks/metrics-agent/tests/fixtures/barnes_app/config/puma.rb create mode 100644 buildpacks/metrics-agent/tests/integration_test.rs diff --git a/Cargo.lock b/Cargo.lock index 7119820d..9d09fe1f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -26,6 +26,21 @@ dependencies = [ "memchr", ] +[[package]] +name = "android-tzdata" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e999941b234f3131b00bc13c22d06e8c5ff726d1b6318ac7eb276997bbb4fef0" + +[[package]] +name = "android_system_properties" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" +dependencies = [ + "libc", +] + [[package]] name = "anstream" version = "0.5.0" @@ -106,6 +121,12 @@ dependencies = [ "rustc-demangle", ] +[[package]] +name = "base64" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8" + [[package]] name = "base64" version = "0.21.3" @@ -148,6 +169,45 @@ dependencies = [ "generic-array", ] +[[package]] +name = "bollard" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af254ed2da4936ef73309e9597180558821cb16ae9bba4cb24ce6b612d8d80ed" +dependencies = [ + "base64 0.21.3", + "bollard-stubs", + "bytes", + "futures-core", + "futures-util", + "hex", + "http", + "hyper", + "hyperlocal", + "log", + "pin-project-lite", + "serde", + "serde_derive", + "serde_json", + "serde_repr", + "serde_urlencoded", + "thiserror", + "tokio", + "tokio-util", + "url", + "winapi", +] + +[[package]] +name = "bollard-stubs" +version = "1.42.0-rc.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "602bda35f33aeb571cef387dcd4042c643a8bf689d8aaac2cc47ea24cb7bc7e0" +dependencies = [ + "serde", + "serde_with", +] + [[package]] name = "bumpalo" version = "3.13.0" @@ -164,6 +224,12 @@ dependencies = [ "utf8-width", ] +[[package]] +name = "bytes" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "89b2fd2a0dcf38d7971e2194b6b6eebab45ae01067456a7fd93d5547a61b70be" + [[package]] name = "cached" version = "0.44.0" @@ -261,6 +327,19 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" +[[package]] +name = "chrono" +version = "0.4.26" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ec837a71355b28f6556dbd569b37b3f363091c0bd4b2e735674521b4c5fd9bc5" +dependencies = [ + "android-tzdata", + "iana-time-zone", + "num-traits", + "serde", + "winapi", +] + [[package]] name = "clap" version = "4.4.1" @@ -332,6 +411,12 @@ dependencies = [ "which_problem", ] +[[package]] +name = "core-foundation-sys" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e496a50fda8aacccc86d7529e2c1e0892dbd0f898a6b5645b5561b89c3210efa" + [[package]] name = "cpufeatures" version = "0.2.9" @@ -438,6 +523,15 @@ dependencies = [ "syn 1.0.109", ] +[[package]] +name = "deranged" +version = "0.3.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2696e8a945f658fd14dc3b87242e6b80cd0f36ff04ea560fa39082368847946" +dependencies = [ + "serde", +] + [[package]] name = "digest" version = "0.10.7" @@ -589,6 +683,17 @@ version = "0.3.28" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4fff74096e71ed47f8e023204cfd0aa1289cd54ae5430a9523be060cdb849964" +[[package]] +name = "futures-macro" +version = "0.3.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "89ca545a94061b6365f2c7355b4b32bd20df3ff95f02da9329b34ccc3bd6ee72" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.29", +] + [[package]] name = "futures-sink" version = "0.3.28" @@ -608,10 +713,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "26b01e40b772d54cf6c6d721c1d1abd0647a0106a12ecaa1c186273392a69533" dependencies = [ "futures-core", + "futures-macro", "futures-sink", "futures-task", "pin-project-lite", "pin-utils", + "slab", ] [[package]] @@ -647,6 +754,31 @@ version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b" +[[package]] +name = "h2" +version = "0.3.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91fc23aa11be92976ef4729127f1a74adf36d8436f7816b185d18df956790833" +dependencies = [ + "bytes", + "fnv", + "futures-core", + "futures-sink", + "futures-util", + "http", + "indexmap 1.9.3", + "slab", + "tokio", + "tokio-util", + "tracing", +] + +[[package]] +name = "hashbrown" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" + [[package]] name = "hashbrown" version = "0.13.2" @@ -681,7 +813,7 @@ dependencies = [ "glob", "indoc", "libcnb 0.14.0", - "libcnb-test", + "libcnb-test 0.14.0", "rand", "regex", "serde", @@ -705,6 +837,7 @@ dependencies = [ "glob", "indoc", "libcnb 0.13.0", + "libcnb-test 0.13.0", "libherokubuildpack 0.12.0", "rand", "regex", @@ -716,6 +849,106 @@ dependencies = [ "url", ] +[[package]] +name = "hex" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" + +[[package]] +name = "http" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd6effc99afb63425aff9b05836f029929e345a6148a14b7ecd5ab67af944482" +dependencies = [ + "bytes", + "fnv", + "itoa", +] + +[[package]] +name = "http-body" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d5f38f16d184e36f2408a55281cd658ecbd3ca05cce6d6510a176eca393e26d1" +dependencies = [ + "bytes", + "http", + "pin-project-lite", +] + +[[package]] +name = "httparse" +version = "1.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d897f394bad6a705d5f4104762e116a75639e470d80901eed05a860a95cb1904" + +[[package]] +name = "httpdate" +version = "1.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" + +[[package]] +name = "hyper" +version = "0.14.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ffb1cfd654a8219eaef89881fdb3bb3b1cdc5fa75ded05d6933b2b382e395468" +dependencies = [ + "bytes", + "futures-channel", + "futures-core", + "futures-util", + "h2", + "http", + "http-body", + "httparse", + "httpdate", + "itoa", + "pin-project-lite", + "socket2 0.4.9", + "tokio", + "tower-service", + "tracing", + "want", +] + +[[package]] +name = "hyperlocal" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fafdf7b2b2de7c9784f76e02c0935e65a8117ec3b768644379983ab333ac98c" +dependencies = [ + "futures-util", + "hex", + "hyper", + "pin-project", + "tokio", +] + +[[package]] +name = "iana-time-zone" +version = "0.1.57" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2fad5b825842d2b38bd206f3e81d6957625fd7f0a361e345c30e01a0ae2dd613" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "wasm-bindgen", + "windows", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" +dependencies = [ + "cc", +] + [[package]] name = "ident_case" version = "1.0.1" @@ -732,6 +965,17 @@ dependencies = [ "unicode-normalization", ] +[[package]] +name = "indexmap" +version = "1.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99" +dependencies = [ + "autocfg", + "hashbrown 0.12.3", + "serde", +] + [[package]] name = "indexmap" version = "2.0.0" @@ -821,8 +1065,8 @@ version = "0.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39d5e0e5f0ea6fe91d867457289d88c4f56631e37fac072d11676ff970715012" dependencies = [ - "libcnb-data 0.13.0", - "libcnb-proc-macros 0.13.0", + "libcnb-data 0.13.0 (registry+https://github.com/rust-lang/crates.io-index)", + "libcnb-proc-macros 0.13.0 (registry+https://github.com/rust-lang/crates.io-index)", "serde", "thiserror", "toml", @@ -861,7 +1105,20 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "631bda3e80115baf38894609cde58b796d3b3fc0f47cca369321c230df53d563" dependencies = [ "fancy-regex", - "libcnb-proc-macros 0.13.0", + "libcnb-proc-macros 0.13.0 (registry+https://github.com/rust-lang/crates.io-index)", + "serde", + "thiserror", + "toml", + "uriparse", +] + +[[package]] +name = "libcnb-data" +version = "0.13.0" +source = "git+https://github.com/heroku/libcnb.rs?branch=libcnb_test_local_and_meta_buildpack_support#2042df4578e9a908617b2aca656cba0557ee525b" +dependencies = [ + "fancy-regex", + "libcnb-proc-macros 0.13.0 (git+https://github.com/heroku/libcnb.rs?branch=libcnb_test_local_and_meta_buildpack_support)", "serde", "thiserror", "toml", @@ -882,6 +1139,18 @@ dependencies = [ "uriparse", ] +[[package]] +name = "libcnb-package" +version = "0.13.0" +source = "git+https://github.com/heroku/libcnb.rs?branch=libcnb_test_local_and_meta_buildpack_support#2042df4578e9a908617b2aca656cba0557ee525b" +dependencies = [ + "cargo_metadata 0.15.4", + "libcnb-data 0.13.0 (git+https://github.com/heroku/libcnb.rs?branch=libcnb_test_local_and_meta_buildpack_support)", + "petgraph", + "toml", + "which", +] + [[package]] name = "libcnb-package" version = "0.14.0" @@ -919,6 +1188,17 @@ dependencies = [ "syn 2.0.29", ] +[[package]] +name = "libcnb-proc-macros" +version = "0.13.0" +source = "git+https://github.com/heroku/libcnb.rs?branch=libcnb_test_local_and_meta_buildpack_support#2042df4578e9a908617b2aca656cba0557ee525b" +dependencies = [ + "cargo_metadata 0.15.4", + "fancy-regex", + "quote", + "syn 2.0.29", +] + [[package]] name = "libcnb-proc-macros" version = "0.14.0" @@ -931,6 +1211,22 @@ dependencies = [ "syn 2.0.29", ] +[[package]] +name = "libcnb-test" +version = "0.13.0" +source = "git+https://github.com/heroku/libcnb.rs?branch=libcnb_test_local_and_meta_buildpack_support#2042df4578e9a908617b2aca656cba0557ee525b" +dependencies = [ + "bollard", + "fastrand", + "fs_extra", + "libcnb-data 0.13.0 (git+https://github.com/heroku/libcnb.rs?branch=libcnb_test_local_and_meta_buildpack_support)", + "libcnb-package 0.13.0", + "serde", + "tempfile", + "tokio", + "tokio-stream", +] + [[package]] name = "libcnb-test" version = "0.14.0" @@ -941,7 +1237,7 @@ dependencies = [ "fastrand", "fs_extra", "libcnb-data 0.14.0", - "libcnb-package", + "libcnb-package 0.14.0", "tempfile", ] @@ -1038,6 +1334,17 @@ dependencies = [ "adler", ] +[[package]] +name = "mio" +version = "0.8.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "927a765cd3fc26206e66b296465fa9d3e5ab003e651c1b3c060e7956d96b19d2" +dependencies = [ + "libc", + "wasi", + "windows-sys", +] + [[package]] name = "num-traits" version = "0.2.16" @@ -1123,7 +1430,27 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e1d3afd2628e69da2be385eb6f2fd57c8ac7977ceeff6dc166ff1657b0e386a9" dependencies = [ "fixedbitset", - "indexmap", + "indexmap 2.0.0", +] + +[[package]] +name = "pin-project" +version = "1.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fda4ed1c6c173e3fc7a83629421152e01d7b1f9b7f65fb301e490e8cfc656422" +dependencies = [ + "pin-project-internal", +] + +[[package]] +name = "pin-project-internal" +version = "1.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4359fd9c9171ec6e8c62926d6faaf553a8dc3f64e1507e76da7911b4f6a04405" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.29", ] [[package]] @@ -1395,6 +1722,17 @@ dependencies = [ "serde", ] +[[package]] +name = "serde_repr" +version = "0.1.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8725e1dfadb3a50f7e5ce0b1a540466f6ed3fe7a0fca2ac2b8b831d31316bd00" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.29", +] + [[package]] name = "serde_spanned" version = "0.6.3" @@ -1404,6 +1742,33 @@ dependencies = [ "serde", ] +[[package]] +name = "serde_urlencoded" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" +dependencies = [ + "form_urlencoded", + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "serde_with" +version = "2.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07ff71d2c147a7b57362cead5e22f772cd52f6ab31cfcd9edcd7f6aeb2a0afbe" +dependencies = [ + "base64 0.13.1", + "chrono", + "hex", + "indexmap 1.9.3", + "serde", + "serde_json", + "time", +] + [[package]] name = "sha2" version = "0.10.7" @@ -1415,12 +1780,41 @@ dependencies = [ "digest", ] +[[package]] +name = "slab" +version = "0.4.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f92a496fb766b417c996b9c5e57daf2f7ad3b0bebe1ccfca4856390e3d3bb67" +dependencies = [ + "autocfg", +] + [[package]] name = "smallvec" version = "1.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "62bb4feee49fdd9f707ef802e22365a35de4b7b299de4763d44bfea899442ff9" +[[package]] +name = "socket2" +version = "0.4.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "64a4a911eed85daf18834cfaa86a79b7d266ff93ff5ba14005426219480ed662" +dependencies = [ + "libc", + "winapi", +] + +[[package]] +name = "socket2" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2538b18701741680e0322a2302176d3253a35388e2e62f172f64f4f16605f877" +dependencies = [ + "libc", + "windows-sys", +] + [[package]] name = "spin" version = "0.5.2" @@ -1508,6 +1902,34 @@ dependencies = [ "syn 2.0.29", ] +[[package]] +name = "time" +version = "0.3.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "17f6bb557fd245c28e6411aa56b6403c689ad95061f50e4be16c274e70a17e48" +dependencies = [ + "deranged", + "itoa", + "serde", + "time-core", + "time-macros", +] + +[[package]] +name = "time-core" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7300fbefb4dadc1af235a9cef3737cea692a9d97e1b9cbcd4ebdae6f8868e6fb" + +[[package]] +name = "time-macros" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a942f44339478ef67935ab2bbaec2fb0322496cf3cbe84b261e06ac3814c572" +dependencies = [ + "time-core", +] + [[package]] name = "tinyvec" version = "1.6.0" @@ -1530,9 +1952,15 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "17ed6077ed6cd6c74735e21f37eb16dc3935f96878b1fe961074089cc80893f9" dependencies = [ "backtrace", + "bytes", + "libc", + "mio", + "num_cpus", "parking_lot", "pin-project-lite", + "socket2 0.5.3", "tokio-macros", + "windows-sys", ] [[package]] @@ -1546,6 +1974,31 @@ dependencies = [ "syn 2.0.29", ] +[[package]] +name = "tokio-stream" +version = "0.1.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "397c988d37662c7dda6d2208364a706264bf3d6138b11d436cbac0ad38832842" +dependencies = [ + "futures-core", + "pin-project-lite", + "tokio", +] + +[[package]] +name = "tokio-util" +version = "0.7.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "806fe8c2c87eccc8b3267cbae29ed3ab2d0bd37fca70ab622e46aaa9375ddb7d" +dependencies = [ + "bytes", + "futures-core", + "futures-sink", + "pin-project-lite", + "tokio", + "tracing", +] + [[package]] name = "toml" version = "0.7.6" @@ -1573,13 +2026,45 @@ version = "0.19.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8123f27e969974a3dfba720fdb560be359f57b44302d280ba72e76a74480e8a" dependencies = [ - "indexmap", + "indexmap 2.0.0", "serde", "serde_spanned", "toml_datetime", "winnow", ] +[[package]] +name = "tower-service" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6bc1c9ce2b5135ac7f93c72918fc37feb872bdc6a5533a8b85eb4b86bfdae52" + +[[package]] +name = "tracing" +version = "0.1.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ce8c33a8d48bd45d624a6e523445fd21ec13d3653cd51f681abf67418f54eb8" +dependencies = [ + "cfg-if", + "pin-project-lite", + "tracing-core", +] + +[[package]] +name = "tracing-core" +version = "0.1.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0955b8137a1df6f1a2e9a37d8a6656291ff0297c1a97c24e0d8425fe2312f79a" +dependencies = [ + "once_cell", +] + +[[package]] +name = "try-lock" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3528ecfd12c466c6f163363caf2d02a71161dd5e1cc6ae7b34207ea2d42d81ed" + [[package]] name = "typenum" version = "1.16.0" @@ -1619,7 +2104,7 @@ version = "2.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b11c96ac7ee530603dcdf68ed1557050f374ce55a5a07193ebf8cbc9f8927e9" dependencies = [ - "base64", + "base64 0.21.3", "flate2", "log", "once_cell", @@ -1685,6 +2170,15 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "want" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e" +dependencies = [ + "try-lock", +] + [[package]] name = "wasi" version = "0.11.0+wasi-snapshot-preview1" @@ -1819,6 +2313,15 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" +[[package]] +name = "windows" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e686886bc078bc1b0b600cac0147aadb815089b6e4da64016cbd754b6342700f" +dependencies = [ + "windows-targets", +] + [[package]] name = "windows-sys" version = "0.48.0" diff --git a/buildpacks/metrics-agent/Cargo.toml b/buildpacks/metrics-agent/Cargo.toml index f6b7993d..7e47b0e9 100644 --- a/buildpacks/metrics-agent/Cargo.toml +++ b/buildpacks/metrics-agent/Cargo.toml @@ -24,3 +24,7 @@ glob = "0.3" cached = "0.44.0" clap = { version = "4.3.17", features = ["derive"] } libherokubuildpack.workspace = true + + +[dev-dependencies] +libcnb-test = { git = "https://github.com/heroku/libcnb.rs", branch = "libcnb_test_local_and_meta_buildpack_support" } diff --git a/buildpacks/metrics-agent/src/main.rs b/buildpacks/metrics-agent/src/main.rs index 4e8f2345..69b4e776 100644 --- a/buildpacks/metrics-agent/src/main.rs +++ b/buildpacks/metrics-agent/src/main.rs @@ -48,7 +48,7 @@ impl Buildpack for MetricsAgentBuildpack { } fn build(&self, context: BuildContext) -> libcnb::Result { - let build_duration = build_output::buildpack_name("Heroku Statsd Metrics Agent"); + let build_duration = build_output::buildpack_name("Heroku StatsD Metrics Agent"); let section = build_output::section("Metrics agent"); context.handle_layer( diff --git a/buildpacks/metrics-agent/tests/fixtures/barnes_app/Gemfile b/buildpacks/metrics-agent/tests/fixtures/barnes_app/Gemfile new file mode 100644 index 00000000..45bc623e --- /dev/null +++ b/buildpacks/metrics-agent/tests/fixtures/barnes_app/Gemfile @@ -0,0 +1,4 @@ +source "https://rubygems.org" + +gem "barnes" +gem "puma" diff --git a/buildpacks/metrics-agent/tests/fixtures/barnes_app/Gemfile.lock b/buildpacks/metrics-agent/tests/fixtures/barnes_app/Gemfile.lock new file mode 100644 index 00000000..502408fa --- /dev/null +++ b/buildpacks/metrics-agent/tests/fixtures/barnes_app/Gemfile.lock @@ -0,0 +1,23 @@ +GEM + remote: https://rubygems.org/ + specs: + barnes (0.0.9) + multi_json (~> 1) + statsd-ruby (~> 1.1) + multi_json (1.15.0) + nio4r (2.5.9) + puma (6.3.0) + nio4r (~> 2.0) + statsd-ruby (1.5.0) + +PLATFORMS + ruby + x86_64-darwin-22 + x86_64-linux + +DEPENDENCIES + barnes + puma + +BUNDLED WITH + 2.4.17 diff --git a/buildpacks/metrics-agent/tests/fixtures/barnes_app/config.ru b/buildpacks/metrics-agent/tests/fixtures/barnes_app/config.ru new file mode 100644 index 00000000..041a6deb --- /dev/null +++ b/buildpacks/metrics-agent/tests/fixtures/barnes_app/config.ru @@ -0,0 +1 @@ +run Proc.new { |env| ['200', {'Content-Type' => 'text/html'}, ['Hello World']] } diff --git a/buildpacks/metrics-agent/tests/fixtures/barnes_app/config/puma.rb b/buildpacks/metrics-agent/tests/fixtures/barnes_app/config/puma.rb new file mode 100644 index 00000000..e9e8332e --- /dev/null +++ b/buildpacks/metrics-agent/tests/fixtures/barnes_app/config/puma.rb @@ -0,0 +1,9 @@ +require 'barnes' + +before_fork do + # worker specific setup + + Barnes.start # Must have enabled worker mode for this to block to be called +end + +workers 2 diff --git a/buildpacks/metrics-agent/tests/integration_test.rs b/buildpacks/metrics-agent/tests/integration_test.rs new file mode 100644 index 00000000..927eb388 --- /dev/null +++ b/buildpacks/metrics-agent/tests/integration_test.rs @@ -0,0 +1,41 @@ +#![warn(clippy::pedantic)] + +use indoc::formatdoc; +use libcnb_test::{ + assert_contains, assert_empty, BuildConfig, BuildpackReference, ContainerConfig, + ContainerContext, TestRunner, +}; +use std::path::PathBuf; +use std::thread; +use std::time::Duration; +use thiserror::__private::DisplayAsDisplay; +use ureq::Response; + +#[test] +#[ignore = "integration test"] +fn test_barnes_app() { + TestRunner::default().build( + BuildConfig::new("heroku/builder:22", "tests/fixtures/barnes_app").buildpacks(vec![ + BuildpackReference::Crate, + BuildpackReference::Local(PathBuf::from("../ruby")), + ]), + |context| { + assert_contains!(context.pack_stdout, "# Heroku StatsD Metrics Agent"); + assert_contains!(context.pack_stdout, "# Heroku Ruby Buildpack"); + + context.start_container( + ContainerConfig::new() + .entrypoint(["launcher"]) + .envs(vec![ + ("HEROKU_METRICS_URL", "example.com"), + ("DYNO", "web.1"), + ]) + .command(["ps x"]), + |container| { + let log_output = container.logs_wait(); + assert_contains!(log_output.stdout, "agentmon_loop --path"); + }, + ); + }, + ); +} From 54605d4318d6b5e2d809d70b43115f59dfec0b2d Mon Sep 17 00:00:00 2001 From: Richard Schneeman Date: Wed, 9 Aug 2023 12:24:20 -0500 Subject: [PATCH 08/40] Enable agentmon logging --- .../src/layers/install_agentmon.rs | 30 +++++++++++++++---- 1 file changed, 24 insertions(+), 6 deletions(-) diff --git a/buildpacks/metrics-agent/src/layers/install_agentmon.rs b/buildpacks/metrics-agent/src/layers/install_agentmon.rs index 74aa0682..69f46b83 100644 --- a/buildpacks/metrics-agent/src/layers/install_agentmon.rs +++ b/buildpacks/metrics-agent/src/layers/install_agentmon.rs @@ -160,7 +160,7 @@ fn write_execd(agentmon_path: &Path, layer_path: &Path) -> Result Result> {log_file} + + start-stop-daemon --start --background \ + --exec "{agentmon_loop}" \ + -- --path {agentmon_path} + fi + "#), ) .map_err(DownloadAgentmonError::CouldNotWriteDestinationFile)?; - script + execd_script }; Ok(execd_script) From cc1684e89582386de81b2b77570d671717beecae Mon Sep 17 00:00:00 2001 From: Richard Schneeman Date: Wed, 9 Aug 2023 12:29:58 -0500 Subject: [PATCH 09/40] Clippy --- .../metrics-agent/src/layers/install_agentmon.rs | 6 ++++-- buildpacks/metrics-agent/tests/integration_test.rs | 10 +--------- 2 files changed, 5 insertions(+), 11 deletions(-) diff --git a/buildpacks/metrics-agent/src/layers/install_agentmon.rs b/buildpacks/metrics-agent/src/layers/install_agentmon.rs index 69f46b83..93f612a1 100644 --- a/buildpacks/metrics-agent/src/layers/install_agentmon.rs +++ b/buildpacks/metrics-agent/src/layers/install_agentmon.rs @@ -184,7 +184,8 @@ fn write_execd(agentmon_path: &Path, layer_path: &Path) -> Result Result Date: Thu, 10 Aug 2023 09:10:35 -0500 Subject: [PATCH 10/40] Fix tool key --- buildpacks/metrics-agent/buildpack.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/buildpacks/metrics-agent/buildpack.toml b/buildpacks/metrics-agent/buildpack.toml index c5fcc2b7..3559fc5b 100644 --- a/buildpacks/metrics-agent/buildpack.toml +++ b/buildpacks/metrics-agent/buildpack.toml @@ -19,5 +19,5 @@ type = "BSD-3-Clause" [metadata] [metadata.release] -[metadata.release.docker] -# repository = "docker.io/heroku/buildpack-ruby" +[metadata.release.image] +repository = "docker.io/heroku/statsd-metrics" From a50891a1440e20aaeaee7e0c7e3de35cb58df2c7 Mon Sep 17 00:00:00 2001 From: Richard Schneeman Date: Thu, 10 Aug 2023 13:32:53 -0500 Subject: [PATCH 11/40] Prefix image with `buildpack-` Not everything on docker is a buildpack, we use prefixes to make it clear at a glance what is. --- buildpacks/metrics-agent/buildpack.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/buildpacks/metrics-agent/buildpack.toml b/buildpacks/metrics-agent/buildpack.toml index 3559fc5b..5d4005d9 100644 --- a/buildpacks/metrics-agent/buildpack.toml +++ b/buildpacks/metrics-agent/buildpack.toml @@ -20,4 +20,4 @@ type = "BSD-3-Clause" [metadata] [metadata.release] [metadata.release.image] -repository = "docker.io/heroku/statsd-metrics" +repository = "docker.io/heroku/buildpack-statsd-metrics" From 8528b0b5d91d1100f72cd82283e19c75e08d644f Mon Sep 17 00:00:00 2001 From: Richard Schneeman Date: Thu, 10 Aug 2023 16:31:21 -0500 Subject: [PATCH 12/40] Update agentmon_loop - Fix bad docstrings - Move sleep_for to a constant - Refactor logic into smaller, more consistent functions - Added tests for argument generation --- .../metrics-agent/src/bin/agentmon_loop.rs | 182 +++++++++++++----- 1 file changed, 138 insertions(+), 44 deletions(-) diff --git a/buildpacks/metrics-agent/src/bin/agentmon_loop.rs b/buildpacks/metrics-agent/src/bin/agentmon_loop.rs index 98067832..c00bcf5a 100644 --- a/buildpacks/metrics-agent/src/bin/agentmon_loop.rs +++ b/buildpacks/metrics-agent/src/bin/agentmon_loop.rs @@ -3,77 +3,171 @@ #![warn(clippy::pedantic)] use clap::Parser; -use commons::fun_run::CmdMapExt; +use std::ffi::OsStr; +use std::process::ExitStatus; use std::{ - ffi::OsString, + collections::HashMap, path::{Path, PathBuf}, process::{exit, Command}, + thread::sleep, time::Duration, }; +const SLEEP_FOR: Duration = Duration::from_secs(1); -/// Simple program to greet a person +/// Agentmon Loop +/// +/// Boots agentmon (a statsd server) in a loop +/// +/// Example: +/// +/// $ cargo run --bin agentmon_loop -- --path + +/// Turn CLI arguments into a Rust struct #[derive(Parser, Debug)] struct Args { - /// Name of the person to greet + /// Path to the agentmon executable e.g. --path #[arg(short, long)] path: PathBuf, } fn main() { - let sleep_for = std::time::Duration::from_secs(1); - let heroku_metrics_url = if let Some(url) = std::env::var_os("HEROKU_METRICS_URL") { - url - } else { - eprintln!("Metrics agent exiting: 0 (HEROKU_METRICS_URL is not set)"); - exit(0) - }; - - if let Some(true) = - std::env::var_os("DYNO").map(|value| value.to_string_lossy().starts_with("run.")) - { - eprintln!("Metrics agent exiting: 0 (one off dyno detected i.e. DYNO=\"run.*\")"); - exit(0) - } - let agentmon = Args::parse().path; if !agentmon.exists() { eprintln!("Path does not exist {}", agentmon.display()); exit(1); } + let agentmon_args = match build_args(std::env::vars().collect::>()) { + Ok(args) => args, + Err(e) => { + eprintln!("Cannot start agentmon: {e}"); + exit(1) + } + }; + loop { - run_agentmon(&agentmon, &heroku_metrics_url, &sleep_for); + match run(&agentmon, &agentmon_args) { + Ok(status) => { + eprintln!("process completed with status=${status}, sleeping {SLEEP_FOR:?}"); + } + Err(error) => { + eprintln!("process could not be run due to error: {error}, sleeping {SLEEP_FOR:?}"); + } + }; + sleep(SLEEP_FOR); } } -fn run_agentmon(agentmon: &Path, heroku_metrics_url: &OsString, sleep_for: &Duration) { - if let Some(port) = std::env::var_os("PORT") { - let statsd_addr = { - let mut string = OsString::from("statsd-addr=:"); - string.push(&port); - string - }; +/// Print and run executable +/// +/// Runs an executable at the given path with args and streams the results. +fn run(path: &Path, args: I) -> Result +where + I: IntoIterator, + S: AsRef, +{ + let mut cmd = Command::new(path); + cmd.args(args); - let result = Command::new(agentmon).cmd_map(|cmd| { - cmd.arg(&statsd_addr); + eprintln!("Running: {}", commons::fun_run::display(&mut cmd)); - if let Some(true) = std::env::var_os("AGENTMON_DEBUG").map(|value| value == *"true") { - cmd.arg("-debug"); - }; - cmd.arg(heroku_metrics_url); + cmd.spawn().and_then(|mut child| child.wait()) +} - cmd.spawn().and_then(|mut child| child.wait()) - }); +#[derive(Debug, thiserror::Error, PartialEq)] +enum Error { + #[error("PORT environment variable is not set")] + MissingPort, - match result { - Ok(status) => eprintln!("agentmon completed with status=${status}. Restarting"), - Err(error) => { - eprintln!("agentmon could not be run due to error: {error}"); - eprintln!("Retrying"); - } - }; + #[error("HEROKU_METRICS_URL environment variable is not set")] + MissingMetricsUrl, + + #[error("One off dyno detected i.e. DYNO=\"run.*\"")] + RunDynoDetected, +} + +/// Constructs the arguments for agentmon based on environment variables +/// +/// # Errors +/// +/// - PORT is not set +/// - HEROKU_METRICS_URL is not set +/// - DYNO starts with `run.` +fn build_args(env: HashMap) -> Result, Error> { + let mut args = Vec::new(); + if let Some(true) = env.get("DYNO").map(|value| value.starts_with("run.")) { + return Err(Error::RunDynoDetected); + } + + if let Some(port) = env.get("PORT") { + args.push(format!("statsd-addr=:{port}")); + } else { + return Err(Error::MissingPort); + }; + + if let Some(true) = env.get("AGENTMON_DEBUG").map(|value| value == "true") { + args.push("-debug".to_string()); + }; + + if let Some(url) = env.get("HEROKU_METRICS_URL") { + args.push(url.clone()); } else { - eprintln!("PORT is not set, sleeping {sleep_for:?}"); + return Err(Error::MissingMetricsUrl); + }; + + Ok(args) +} + +#[cfg(test)] +mod test { + use super::*; + + #[test] + fn missing_port() { + let result = build_args(HashMap::new()); + + assert_eq!(result, Err(Error::MissingPort)); + } + + #[test] + fn agentmon_args() { + let mut env = HashMap::new(); + env.insert("PORT".to_string(), "90210".to_string()); + env.insert( + "HEROKU_METRICS_URL".to_string(), + "https://example.com".to_string(), + ); + + let result = build_args(env); + + assert_eq!( + result, + Ok(vec![ + "statsd-addr=:90210".to_string(), + "https://example.com".to_string() + ]) + ); + } + + #[test] + fn agentmon_debug_args() { + let mut env = HashMap::new(); + env.insert("PORT".to_string(), "90210".to_string()); + env.insert( + "HEROKU_METRICS_URL".to_string(), + "https://example.com".to_string(), + ); + env.insert("AGENTMON_DEBUG".to_string(), "true".to_string()); + + let result = build_args(env); + + assert_eq!( + result, + Ok(vec![ + "statsd-addr=:90210".to_string(), + "-debug".to_string(), + "https://example.com".to_string() + ]) + ); } - std::thread::sleep(*sleep_for); } From 7bae99248418933d39cc659dd294424f99472320 Mon Sep 17 00:00:00 2001 From: Richard Schneeman Date: Thu, 10 Aug 2023 16:37:41 -0500 Subject: [PATCH 13/40] Remove unneeded enum contortions --- buildpacks/metrics-agent/src/main.rs | 19 +++---------------- 1 file changed, 3 insertions(+), 16 deletions(-) diff --git a/buildpacks/metrics-agent/src/main.rs b/buildpacks/metrics-agent/src/main.rs index 69b4e776..ce63a595 100644 --- a/buildpacks/metrics-agent/src/main.rs +++ b/buildpacks/metrics-agent/src/main.rs @@ -65,23 +65,10 @@ impl Buildpack for MetricsAgentBuildpack { } } -#[derive(Debug)] -enum Cause { - OurError(MetricsAgentError), - FrameworkError(libcnb::Error), -} - -fn cause(err: libcnb::Error) -> Cause { - match err { - libcnb::Error::BuildpackError(err) => Cause::OurError(err), - err => Cause::FrameworkError(err), - } -} - pub(crate) fn on_error(err: libcnb::Error) { - match cause(err) { - Cause::OurError(error) => log_our_error(error), - Cause::FrameworkError(error) => ErrorInfo::header_body_details( + match err { + libcnb::Error::BuildpackError(error) => log_our_error(error), + error => ErrorInfo::header_body_details( "heroku/buildpack-ruby internal buildpack error", formatdoc! {" An unexpected internal error was reported by the framework used From c5510f34c151473c816a8e729ae31505f36e6753 Mon Sep 17 00:00:00 2001 From: Richard Schneeman Date: Fri, 11 Aug 2023 09:15:26 -0500 Subject: [PATCH 14/40] Inline binary logic and consolidate test logic --- .../src/layers/install_agentmon.rs | 37 +++++++------------ 1 file changed, 14 insertions(+), 23 deletions(-) diff --git a/buildpacks/metrics-agent/src/layers/install_agentmon.rs b/buildpacks/metrics-agent/src/layers/install_agentmon.rs index 93f612a1..c2b0e4bc 100644 --- a/buildpacks/metrics-agent/src/layers/install_agentmon.rs +++ b/buildpacks/metrics-agent/src/layers/install_agentmon.rs @@ -252,10 +252,14 @@ pub(crate) fn untar( } /// Sets file permissions on the given path to 7xx (similar to `chmod +x `) +/// +/// i.e. chmod +x will ensure that the first digit +/// of the file permission is 7 on unix so if you pass +/// in 0o455 it would be mutated to 0o755 pub fn chmod_plus_x(path: &Path) -> Result<(), std::io::Error> { let mut perms = fs_err::metadata(path)?.permissions(); let mut mode = perms.mode(); - octal_executable_permission(&mut mode); + mode |= 0o700; perms.set_mode(mode); fs_err::set_permissions(path, perms) @@ -270,16 +274,6 @@ fn write_bash_script(path: &Path, script: impl AsRef) -> std::io::Result<() Ok(()) } -/// Ensures the provided octal number's executable -/// bit is enabled. -/// -/// i.e. chmod +x will ensure that the first digit -/// of the file permission is 7 on unix so if you pass -/// in 0o455 it would be mutated to 0o755 -fn octal_executable_permission(mode: &mut u32) { - *mode |= 0o700; -} - pub(crate) fn download( uri: impl AsRef, destination: impl AsRef, @@ -309,21 +303,18 @@ mod tests { std::fs::write(&file, "lol").unwrap(); let before = file.metadata().unwrap().permissions().mode(); + + let foo = before | 0o777; + + dbg!(before); + dbg!(foo); + chmod_plus_x(&file).unwrap(); + let after = file.metadata().unwrap().permissions().mode(); assert!(before != after); - } - #[test] - fn test_executable_logic() { - // Sets executable bit - let mut mode = 0o455; - octal_executable_permission(&mut mode); - assert_eq!(0o755, mode); - - // Does not affect already executable - let mut mode = 0o745; - octal_executable_permission(&mut mode); - assert_eq!(0o745, mode); + // Assert executable + assert_eq!(after, after | 0o700); } } From 802b9a6edb35dbcbddc1447d19d1c8fe3cdac44a Mon Sep 17 00:00:00 2001 From: Richard Schneeman Date: Fri, 11 Aug 2023 13:19:08 -0500 Subject: [PATCH 15/40] Hardcode agentmon url, improve caching Agentmon hasn't seen a release in 6 years. We can hardcode the url for now. When the URL doesn't change we can keep the last download around. I always want to update the scripts on disk so then any changes to the files will be picked up, so we'll always call update or create, and never keep. We could get fancy and use MetadataDigest to cache those too, but writing them is pretty much instantaneous so I don't think it buys us anything. This change allows us to remove the `cached` dependency. --- Cargo.lock | 213 +---------------- buildpacks/metrics-agent/Cargo.toml | 1 - .../src/layers/install_agentmon.rs | 218 ++++++++---------- 3 files changed, 109 insertions(+), 323 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 9d09fe1f..5aa836ba 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -89,17 +89,6 @@ dependencies = [ "windows-sys", ] -[[package]] -name = "async-trait" -version = "0.1.73" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc00ceb34980c03614e35a3a4e218276a0a824e911d07651cd0d858a51e8c0f0" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.29", -] - [[package]] name = "autocfg" version = "1.1.0" @@ -230,42 +219,6 @@ version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "89b2fd2a0dcf38d7971e2194b6b6eebab45ae01067456a7fd93d5547a61b70be" -[[package]] -name = "cached" -version = "0.44.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b195e4fbc4b6862bbd065b991a34750399c119797efff72492f28a5864de8700" -dependencies = [ - "async-trait", - "cached_proc_macro", - "cached_proc_macro_types", - "futures", - "hashbrown 0.13.2", - "instant", - "once_cell", - "thiserror", - "tokio", -] - -[[package]] -name = "cached_proc_macro" -version = "0.17.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b48814962d2fd604c50d2b9433c2a41a0ab567779ee2c02f7fba6eca1221f082" -dependencies = [ - "cached_proc_macro_types", - "darling", - "proc-macro2", - "quote", - "syn 1.0.109", -] - -[[package]] -name = "cached_proc_macro_types" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3a4f925191b4367301851c6d99b09890311d74b0d43f274c0b34c86d308a3663" - [[package]] name = "camino" version = "1.1.6" @@ -372,7 +325,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.29", + "syn", ] [[package]] @@ -488,41 +441,6 @@ dependencies = [ "typenum", ] -[[package]] -name = "darling" -version = "0.14.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b750cb3417fd1b327431a470f388520309479ab0bf5e323505daf0290cd3850" -dependencies = [ - "darling_core", - "darling_macro", -] - -[[package]] -name = "darling_core" -version = "0.14.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "109c1ca6e6b7f82cc233a97004ea8ed7ca123a9af07a8230878fcfda9b158bf0" -dependencies = [ - "fnv", - "ident_case", - "proc-macro2", - "quote", - "strsim", - "syn 1.0.109", -] - -[[package]] -name = "darling_macro" -version = "0.14.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4aab4dbc9f7611d8b55048a3a16d2d010c2c8334e46304b40ac1cc14bf3b48e" -dependencies = [ - "darling_core", - "quote", - "syn 1.0.109", -] - [[package]] name = "deranged" version = "0.3.8" @@ -647,20 +565,6 @@ version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" -[[package]] -name = "futures" -version = "0.3.28" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23342abe12aba583913b2e62f22225ff9c950774065e4bfb61a19cd9770fec40" -dependencies = [ - "futures-channel", - "futures-core", - "futures-io", - "futures-sink", - "futures-task", - "futures-util", -] - [[package]] name = "futures-channel" version = "0.3.28" @@ -668,7 +572,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "955518d47e09b25bbebc7a18df10b81f0c766eaf4c4f1cccef2fca5f2a4fb5f2" dependencies = [ "futures-core", - "futures-sink", ] [[package]] @@ -677,12 +580,6 @@ version = "0.3.28" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4bca583b7e26f571124fe5b7561d49cb2868d79116cfa0eefce955557c6fee8c" -[[package]] -name = "futures-io" -version = "0.3.28" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4fff74096e71ed47f8e023204cfd0aa1289cd54ae5430a9523be060cdb849964" - [[package]] name = "futures-macro" version = "0.3.28" @@ -691,7 +588,7 @@ checksum = "89ca545a94061b6365f2c7355b4b32bd20df3ff95f02da9329b34ccc3bd6ee72" dependencies = [ "proc-macro2", "quote", - "syn 2.0.29", + "syn", ] [[package]] @@ -714,7 +611,6 @@ checksum = "26b01e40b772d54cf6c6d721c1d1abd0647a0106a12ecaa1c186273392a69533" dependencies = [ "futures-core", "futures-macro", - "futures-sink", "futures-task", "pin-project-lite", "pin-utils", @@ -779,12 +675,6 @@ version = "0.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" -[[package]] -name = "hashbrown" -version = "0.13.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43a3c133739dddd0d2990f9a4bdf8eb4b21ef50e4851ca85ab661199821d510e" - [[package]] name = "hashbrown" version = "0.14.0" @@ -829,7 +719,6 @@ dependencies = [ name = "heroku-statsd-metrics" version = "0.0.0" dependencies = [ - "cached", "clap", "commons", "flate2", @@ -949,12 +838,6 @@ dependencies = [ "cc", ] -[[package]] -name = "ident_case" -version = "1.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" - [[package]] name = "idna" version = "0.4.0" @@ -992,15 +875,6 @@ version = "2.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2c785eefb63ebd0e33416dfcb8d6da0bf27ce752843a45632a67bf10d4d4b5c4" -[[package]] -name = "instant" -version = "0.1.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a5bbe824c507c5da5956355e86a746d82e0e1464f65d862cc5e71da70e94b2c" -dependencies = [ - "cfg-if", -] - [[package]] name = "is_executable" version = "1.0.1" @@ -1173,7 +1047,7 @@ dependencies = [ "cargo_metadata 0.15.4", "fancy-regex", "quote", - "syn 2.0.29", + "syn", ] [[package]] @@ -1185,7 +1059,7 @@ dependencies = [ "cargo_metadata 0.15.4", "fancy-regex", "quote", - "syn 2.0.29", + "syn", ] [[package]] @@ -1196,7 +1070,7 @@ dependencies = [ "cargo_metadata 0.15.4", "fancy-regex", "quote", - "syn 2.0.29", + "syn", ] [[package]] @@ -1208,7 +1082,7 @@ dependencies = [ "cargo_metadata 0.17.0", "fancy-regex", "quote", - "syn 2.0.29", + "syn", ] [[package]] @@ -1294,16 +1168,6 @@ version = "0.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "57bcfdad1b858c2db7c38303a6d2ad4dfaf5eb53dfeb0910128b2c26d6158503" -[[package]] -name = "lock_api" -version = "0.4.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c1cc9717a20b1bb222f333e6a92fd32f7d8a18ddc5a3191a11af45dcbf4dcd16" -dependencies = [ - "autocfg", - "scopeguard", -] - [[package]] name = "log" version = "0.4.20" @@ -1388,29 +1252,6 @@ dependencies = [ "num-traits", ] -[[package]] -name = "parking_lot" -version = "0.12.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3742b2c103b9f06bc9fff0a37ff4912935851bee6d36f3c02bcc755bcfec228f" -dependencies = [ - "lock_api", - "parking_lot_core", -] - -[[package]] -name = "parking_lot_core" -version = "0.9.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "93f00c865fe7cabf650081affecd3871070f26767e7b2070a3ffae14c654b447" -dependencies = [ - "cfg-if", - "libc", - "redox_syscall", - "smallvec", - "windows-targets", -] - [[package]] name = "pathdiff" version = "0.2.1" @@ -1450,7 +1291,7 @@ checksum = "4359fd9c9171ec6e8c62926d6faaf553a8dc3f64e1507e76da7911b4f6a04405" dependencies = [ "proc-macro2", "quote", - "syn 2.0.29", + "syn", ] [[package]] @@ -1708,7 +1549,7 @@ checksum = "4eca7ac642d82aa35b60049a6eccb4be6be75e599bd2e9adb5f875a737654af2" dependencies = [ "proc-macro2", "quote", - "syn 2.0.29", + "syn", ] [[package]] @@ -1730,7 +1571,7 @@ checksum = "8725e1dfadb3a50f7e5ce0b1a540466f6ed3fe7a0fca2ac2b8b831d31316bd00" dependencies = [ "proc-macro2", "quote", - "syn 2.0.29", + "syn", ] [[package]] @@ -1789,12 +1630,6 @@ dependencies = [ "autocfg", ] -[[package]] -name = "smallvec" -version = "1.11.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62bb4feee49fdd9f707ef802e22365a35de4b7b299de4763d44bfea899442ff9" - [[package]] name = "socket2" version = "0.4.9" @@ -1827,17 +1662,6 @@ version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" -[[package]] -name = "syn" -version = "1.0.109" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" -dependencies = [ - "proc-macro2", - "quote", - "unicode-ident", -] - [[package]] name = "syn" version = "2.0.29" @@ -1899,7 +1723,7 @@ checksum = "6bb623b56e39ab7dcd4b1b98bb6c8f8d907ed255b18de254088016b27a8ee19b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.29", + "syn", ] [[package]] @@ -1956,24 +1780,11 @@ dependencies = [ "libc", "mio", "num_cpus", - "parking_lot", "pin-project-lite", "socket2 0.5.3", - "tokio-macros", "windows-sys", ] -[[package]] -name = "tokio-macros" -version = "2.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "630bdcf245f78637c13ec01ffae6187cca34625e8c63150d424b59e55af2675e" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.29", -] - [[package]] name = "tokio-stream" version = "0.1.14" @@ -2206,7 +2017,7 @@ dependencies = [ "once_cell", "proc-macro2", "quote", - "syn 2.0.29", + "syn", "wasm-bindgen-shared", ] @@ -2228,7 +2039,7 @@ checksum = "54681b18a46765f095758388f2d0cf16eb8d4169b639ab575a8f5693af210c7b" dependencies = [ "proc-macro2", "quote", - "syn 2.0.29", + "syn", "wasm-bindgen-backend", "wasm-bindgen-shared", ] diff --git a/buildpacks/metrics-agent/Cargo.toml b/buildpacks/metrics-agent/Cargo.toml index 7e47b0e9..82ae6c09 100644 --- a/buildpacks/metrics-agent/Cargo.toml +++ b/buildpacks/metrics-agent/Cargo.toml @@ -21,7 +21,6 @@ thiserror = "1" ureq = "2" url = { version = "2", features = ["serde"] } glob = "0.3" -cached = "0.44.0" clap = { version = "4.3.17", features = ["derive"] } libherokubuildpack.workspace = true diff --git a/buildpacks/metrics-agent/src/layers/install_agentmon.rs b/buildpacks/metrics-agent/src/layers/install_agentmon.rs index c2b0e4bc..abe79350 100644 --- a/buildpacks/metrics-agent/src/layers/install_agentmon.rs +++ b/buildpacks/metrics-agent/src/layers/install_agentmon.rs @@ -1,6 +1,5 @@ use crate::build_output; use crate::{MetricsAgentBuildpack, MetricsAgentError}; -use cached::proc_macro::cached; use flate2::read::GzDecoder; use libcnb::data::layer_content_metadata::LayerTypes; use libcnb::layer::ExistingLayerStrategy; @@ -11,11 +10,22 @@ use libcnb::{ }; use serde::{Deserialize, Serialize}; use std::os::unix::fs::PermissionsExt; -use std::path::Path; -use std::path::PathBuf; +use std::path::{Path, PathBuf}; use tar::Archive; use tempfile::NamedTempFile; -use url::Url; + +/// Agentmon URL +/// +/// - Repo: https://github.com/heroku/agentmon +/// - Releases: https://github.com/heroku/agentmon/releases +/// +/// To get the latest s3 url: +/// +/// ```shell +/// $ curl https://agentmon-releases.s3.amazonaws.com/latest +/// ``` +const DOWNLOAD_URL: &str = + "https://agentmon-releases.s3.amazonaws.com/agentmon-0.3.1-linux-amd64.tar.gz"; #[derive(Debug)] pub(crate) struct InstallAgentmon { @@ -24,28 +34,11 @@ pub(crate) struct InstallAgentmon { #[derive(Deserialize, Serialize, Debug, Clone)] pub(crate) struct Metadata { - download_url: Option, -} - -// All cloneable subtypes to make cachable -#[derive(thiserror::Error, Debug, Clone)] -pub(crate) enum GetUrlError { - #[error("Response successful, but body not in the form of a URL: {0}")] - CannotConvertResponseToString(String), - - #[error("Cannot parse url: {0}")] - UrlParseError(url::ParseError), - - // Boxed to prevent `large_enum_variant` errors since `ureq::Error` is massive. - #[error("Network error while retrieving the url: {0}")] - RequestError(String), + download_url: Option, } #[derive(thiserror::Error, Debug)] -pub(crate) enum DownloadAgentmonError { - #[error("Could not determine the url of the latest agentmont release.\n{0}")] - CannotGetLatestUrl(GetUrlError), - +pub(crate) enum InstallAgentmonError { #[error("Could not read file permissions {0}")] PermissionError(std::io::Error), @@ -86,25 +79,40 @@ impl Layer for InstallAgentmon { libcnb::layer::LayerResult, ::Error, > { - let destination_dir = layer_path.join("bin"); - let executable = destination_dir.join("agentmon"); + let bin_dir = layer_path.join("bin"); let mut timer = self.section.say_with_inline_timer("Downloading"); + let agentmon = + agentmon_download(&bin_dir).map_err(MetricsAgentError::InstallAgentmonError)?; + timer.done(); - let url = get_latest_url() - .map_err(DownloadAgentmonError::CannotGetLatestUrl) - .map_err(MetricsAgentError::DownloadAgentmonError)?; + self.section.say("Writing scripts"); + let execd = write_execd_script(&agentmon, layer_path) + .map_err(MetricsAgentError::InstallAgentmonError)?; - download_to_dir(&destination_dir, &url) - .map_err(MetricsAgentError::DownloadAgentmonError)?; - timer.done(); + LayerResultBuilder::new(Metadata { + download_url: Some(DOWNLOAD_URL.to_string()), + }) + .exec_d_program("spawn-agentmon", execd) + .build() + } + + fn update( + &self, + _context: &libcnb::build::BuildContext, + layer_data: &libcnb::layer::LayerData, + ) -> Result< + libcnb::layer::LayerResult, + ::Error, + > { + let layer_path = &layer_data.path; self.section.say("Writing scripts"); - let execd = write_execd(&executable, layer_path) - .map_err(MetricsAgentError::DownloadAgentmonError)?; + let execd = write_execd_script(&layer_path.join("bin").join("agentmon"), layer_path) + .map_err(MetricsAgentError::InstallAgentmonError)?; LayerResultBuilder::new(Metadata { - download_url: Some(url), + download_url: Some(DOWNLOAD_URL.to_string()), }) .exec_d_program("spawn agentmon", execd) .build() @@ -116,25 +124,19 @@ impl Layer for InstallAgentmon { layer_data: &libcnb::layer::LayerData, ) -> Result::Error> { - if let Some(old_url) = &layer_data.content_metadata.metadata.download_url { - let url = get_latest_url() - .map_err(DownloadAgentmonError::CannotGetLatestUrl) - .map_err(MetricsAgentError::DownloadAgentmonError)?; - - if old_url == &url { - self.section.say("Using cache"); - Ok(ExistingLayerStrategy::Keep) - } else { - let url = build_output::fmt::value(url); - self.section - .say_with_details("Clearing cache", format!("Updated url {url}")); + match &layer_data.content_metadata.metadata.download_url { + Some(url) if url == DOWNLOAD_URL => { + self.section.say("Using cached metrics agent"); + Ok(ExistingLayerStrategy::Update) + } + Some(url) => { + self.section.say_with_details( + "Updating metrics agent", + format!("{} to {}", url, DOWNLOAD_URL), + ); Ok(ExistingLayerStrategy::Recreate) } - } else { - self.section - .say_with_details("Clearing cache", "No url found in metadata"); - - Ok(ExistingLayerStrategy::Recreate) + None => Ok(ExistingLayerStrategy::Recreate), } } @@ -153,84 +155,58 @@ impl Layer for InstallAgentmon { } } -fn write_execd(agentmon_path: &Path, layer_path: &Path) -> Result { - let agentmon_path = agentmon_path - .canonicalize() - .map_err(DownloadAgentmonError::CouldNotOpenFile)?; - let agentmon_path = agentmon_path.display(); - - // This script boots and runs agentmon in a loop - let agentmon_loop = { - let script = layer_path.join("agentmon_loop"); - - // Copy compiled binary from `bin/agentmon_loop.rs` to layer - fs_err::copy(additional_buildpack_binary_path!("agentmon_loop"), &script) - .map_err(DownloadAgentmonError::CouldNotWriteDestinationFile)?; - - script - .canonicalize() - .map_err(DownloadAgentmonError::CouldNotOpenFile)? - }; - - // We use the exec.d to boot a process. This script MUST exit though as otherwise - // The container would never boot. To handle this we intentionally leak a process - let execd_script = { - let execd_script = layer_path.join("agentmon_exec.d"); - let log_file = layer_path.join("background.log"); - fs_err::write(&log_file, "") - .map_err(DownloadAgentmonError::CouldNotCreateDestinationFile)?; - - let log_file = log_file.display(); - let agentmon_loop = agentmon_loop.display(); - write_bash_script( - &execd_script, - format!( - r#" - if [ -z "$AGENTMON_DEBUG" ] - then - start-stop-daemon --start --background \ - --exec "{agentmon_loop}" \ - --output {log_file} \ - -- --path {agentmon_path} - else - echo "To enable logging run with AGENTMON_DEBUG=1" >> {log_file} - - start-stop-daemon --start --background \ - --exec "{agentmon_loop}" \ - -- --path {agentmon_path} - fi - "# - ), - ) - .map_err(DownloadAgentmonError::CouldNotWriteDestinationFile)?; - - execd_script - }; - - Ok(execd_script) +fn write_execd_script(agentmon: &Path, layer_path: &Path) -> Result { + let log = layer_path.join("output.log"); + let execd = layer_path.join("execd"); + let daemon = layer_path.join("launch_daemon"); + let run_loop = layer_path.join("agentmon_loop"); + + // Ensure log file exists + fs_err::write(&log, "").map_err(InstallAgentmonError::CouldNotWriteDestinationFile)?; + + // agentmon_loop boots agentmon continuously + fs_err::copy( + additional_buildpack_binary_path!("agentmon_loop"), + &run_loop, + ) + .map_err(InstallAgentmonError::CouldNotWriteDestinationFile)?; + + // The `launch_daemon` schedules `agentmon_loop` to run in the background + fs_err::copy(additional_buildpack_binary_path!("launch_daemon"), &daemon) + .map_err(InstallAgentmonError::CouldNotWriteDestinationFile)?; + + // The execd bash script will be run by CNB lifecycle, it runs the `launch_daemon` + fs_err::write( + &execd, + format!( + r#"#!/usr/bin/env bash + + {daemon} --log {log} --loop-path {run_loop} --agentmon {agentmon} + "#, + log = log.display(), + daemon = daemon.display(), + run_loop = run_loop.display(), + agentmon = agentmon.display(), + ), + ) + .map_err(InstallAgentmonError::CouldNotCreateDestinationFile)?; + + chmod_plus_x(&execd).map_err(InstallAgentmonError::PermissionError)?; + + Ok(execd) } -#[cached] -fn get_latest_url() -> Result { - // This file on S3 stores a raw string that holds the URL to the latest agentmon release - // It's not a redirect to the latest file, it's a string body that contains a URL. - let base = Url::parse("https://agentmon-releases.s3.amazonaws.com/latest") - .expect("Internal error: Bad url"); - - let body = ureq::get(base.as_ref()) - .call() - .map_err(|err| GetUrlError::RequestError(err.to_string()))? - .into_string() - .map_err(|error| GetUrlError::CannotConvertResponseToString(error.to_string()))?; +fn agentmon_download(dir: &Path) -> Result { + download_to_dir(DOWNLOAD_URL, dir)?; - Url::parse(body.as_str().trim()).map_err(GetUrlError::UrlParseError) + Ok(dir.join("agentmon")) } -fn download_to_dir(destination: &Path, url: &Url) -> Result<(), DownloadAgentmonError> { +fn download_to_dir(url: impl AsRef, destination: &Path) -> Result<(), InstallAgentmonError> { let agentmon_tgz = NamedTempFile::new().map_err(DownloadAgentmonError::CouldNotCreateDestinationFile)?; - download(url.as_ref(), agentmon_tgz.path())?; + download(url, agentmon_tgz.path())?; untar(agentmon_tgz.path(), destination)?; From 61adc80766d20ea2329f66faa589fd0cf1c877c2 Mon Sep 17 00:00:00 2001 From: Richard Schneeman Date: Mon, 28 Aug 2023 16:03:21 -0500 Subject: [PATCH 16/40] Move metrics logic inside of heroku/ruby --- Cargo.lock | 778 +----------------- Cargo.toml | 7 +- buildpacks/metrics-agent/CHANGELOG.md | 4 - buildpacks/metrics-agent/Cargo.toml | 29 - buildpacks/metrics-agent/buildpack.toml | 23 - buildpacks/metrics-agent/src/layers/mod.rs | 1 - buildpacks/metrics-agent/src/main.rs | 101 --- .../metrics-agent/tests/integration_test.rs | 33 - buildpacks/ruby/Cargo.toml | 1 + .../src/bin/agentmon_loop.rs | 27 +- buildpacks/ruby/src/bin/launch_daemon.rs | 77 ++ buildpacks/ruby/src/layers.rs | 1 + .../src/layers/metrics_agent_install.rs} | 90 +- buildpacks/ruby/src/main.rs | 13 + buildpacks/ruby/src/user_errors.rs | 11 + .../tests/fixtures/barnes_app/Gemfile | 0 .../tests/fixtures/barnes_app/Gemfile.lock | 0 .../tests/fixtures/barnes_app/config.ru | 0 .../tests/fixtures/barnes_app/config/puma.rb | 0 buildpacks/ruby/tests/integration_test.rs | 32 + 20 files changed, 210 insertions(+), 1018 deletions(-) delete mode 100644 buildpacks/metrics-agent/CHANGELOG.md delete mode 100644 buildpacks/metrics-agent/Cargo.toml delete mode 100644 buildpacks/metrics-agent/buildpack.toml delete mode 100644 buildpacks/metrics-agent/src/layers/mod.rs delete mode 100644 buildpacks/metrics-agent/src/main.rs delete mode 100644 buildpacks/metrics-agent/tests/integration_test.rs rename buildpacks/{metrics-agent => ruby}/src/bin/agentmon_loop.rs (83%) create mode 100644 buildpacks/ruby/src/bin/launch_daemon.rs rename buildpacks/{metrics-agent/src/layers/install_agentmon.rs => ruby/src/layers/metrics_agent_install.rs} (72%) rename buildpacks/{metrics-agent => ruby}/tests/fixtures/barnes_app/Gemfile (100%) rename buildpacks/{metrics-agent => ruby}/tests/fixtures/barnes_app/Gemfile.lock (100%) rename buildpacks/{metrics-agent => ruby}/tests/fixtures/barnes_app/config.ru (100%) rename buildpacks/{metrics-agent => ruby}/tests/fixtures/barnes_app/config/puma.rb (100%) diff --git a/Cargo.lock b/Cargo.lock index 5aa836ba..ccb656b8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,15 +2,6 @@ # It is not intended for manual editing. version = 3 -[[package]] -name = "addr2line" -version = "0.21.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a30b2e23b9e17a9f90641c7ab1549cd9b44f296d3ccbf309d2863cfe398a0cb" -dependencies = [ - "gimli", -] - [[package]] name = "adler" version = "1.0.2" @@ -26,21 +17,6 @@ dependencies = [ "memchr", ] -[[package]] -name = "android-tzdata" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e999941b234f3131b00bc13c22d06e8c5ff726d1b6318ac7eb276997bbb4fef0" - -[[package]] -name = "android_system_properties" -version = "0.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" -dependencies = [ - "libc", -] - [[package]] name = "anstream" version = "0.5.0" @@ -95,27 +71,6 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" -[[package]] -name = "backtrace" -version = "0.3.69" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2089b7e3f35b9dd2d0ed921ead4f6d318c27680d4a5bd167b3ee120edb105837" -dependencies = [ - "addr2line", - "cc", - "cfg-if", - "libc", - "miniz_oxide", - "object", - "rustc-demangle", -] - -[[package]] -name = "base64" -version = "0.13.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8" - [[package]] name = "base64" version = "0.21.3" @@ -158,45 +113,6 @@ dependencies = [ "generic-array", ] -[[package]] -name = "bollard" -version = "0.14.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af254ed2da4936ef73309e9597180558821cb16ae9bba4cb24ce6b612d8d80ed" -dependencies = [ - "base64 0.21.3", - "bollard-stubs", - "bytes", - "futures-core", - "futures-util", - "hex", - "http", - "hyper", - "hyperlocal", - "log", - "pin-project-lite", - "serde", - "serde_derive", - "serde_json", - "serde_repr", - "serde_urlencoded", - "thiserror", - "tokio", - "tokio-util", - "url", - "winapi", -] - -[[package]] -name = "bollard-stubs" -version = "1.42.0-rc.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "602bda35f33aeb571cef387dcd4042c643a8bf689d8aaac2cc47ea24cb7bc7e0" -dependencies = [ - "serde", - "serde_with", -] - [[package]] name = "bumpalo" version = "3.13.0" @@ -213,12 +129,6 @@ dependencies = [ "utf8-width", ] -[[package]] -name = "bytes" -version = "1.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89b2fd2a0dcf38d7971e2194b6b6eebab45ae01067456a7fd93d5547a61b70be" - [[package]] name = "camino" version = "1.1.6" @@ -237,20 +147,6 @@ dependencies = [ "serde", ] -[[package]] -name = "cargo_metadata" -version = "0.15.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eee4243f1f26fc7a42710e7439c149e2b10b05472f88090acce52632f231a73a" -dependencies = [ - "camino", - "cargo-platform", - "semver", - "serde", - "serde_json", - "thiserror", -] - [[package]] name = "cargo_metadata" version = "0.17.0" @@ -280,19 +176,6 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" -[[package]] -name = "chrono" -version = "0.4.26" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec837a71355b28f6556dbd569b37b3f363091c0bd4b2e735674521b4c5fd9bc5" -dependencies = [ - "android-tzdata", - "iana-time-zone", - "num-traits", - "serde", - "winapi", -] - [[package]] name = "clap" version = "4.4.1" @@ -352,8 +235,8 @@ dependencies = [ "glob", "indoc", "lazy_static", - "libcnb 0.14.0", - "libherokubuildpack 0.14.0", + "libcnb", + "libherokubuildpack", "regex", "serde", "sha2", @@ -364,12 +247,6 @@ dependencies = [ "which_problem", ] -[[package]] -name = "core-foundation-sys" -version = "0.8.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e496a50fda8aacccc86d7529e2c1e0892dbd0f898a6b5645b5561b89c3210efa" - [[package]] name = "cpufeatures" version = "0.2.9" @@ -441,15 +318,6 @@ dependencies = [ "typenum", ] -[[package]] -name = "deranged" -version = "0.3.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2696e8a945f658fd14dc3b87242e6b80cd0f36ff04ea560fa39082368847946" -dependencies = [ - "serde", -] - [[package]] name = "digest" version = "0.10.7" @@ -565,58 +433,6 @@ version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" -[[package]] -name = "futures-channel" -version = "0.3.28" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "955518d47e09b25bbebc7a18df10b81f0c766eaf4c4f1cccef2fca5f2a4fb5f2" -dependencies = [ - "futures-core", -] - -[[package]] -name = "futures-core" -version = "0.3.28" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4bca583b7e26f571124fe5b7561d49cb2868d79116cfa0eefce955557c6fee8c" - -[[package]] -name = "futures-macro" -version = "0.3.28" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89ca545a94061b6365f2c7355b4b32bd20df3ff95f02da9329b34ccc3bd6ee72" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "futures-sink" -version = "0.3.28" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f43be4fe21a13b9781a69afa4985b0f6ee0e1afab2c6f454a8cf30e2b2237b6e" - -[[package]] -name = "futures-task" -version = "0.3.28" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "76d3d132be6c0e6aa1534069c705a74a5997a356c0dc2f86a47765e5617c5b65" - -[[package]] -name = "futures-util" -version = "0.3.28" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26b01e40b772d54cf6c6d721c1d1abd0647a0106a12ecaa1c186273392a69533" -dependencies = [ - "futures-core", - "futures-macro", - "futures-task", - "pin-project-lite", - "pin-utils", - "slab", -] - [[package]] name = "generic-array" version = "0.14.7" @@ -638,43 +454,12 @@ dependencies = [ "wasi", ] -[[package]] -name = "gimli" -version = "0.28.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6fb8d784f27acf97159b40fc4db5ecd8aa23b9ad5ef69cdd136d3bc80665f0c0" - [[package]] name = "glob" version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b" -[[package]] -name = "h2" -version = "0.3.21" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91fc23aa11be92976ef4729127f1a74adf36d8436f7816b185d18df956790833" -dependencies = [ - "bytes", - "fnv", - "futures-core", - "futures-sink", - "futures-util", - "http", - "indexmap 1.9.3", - "slab", - "tokio", - "tokio-util", - "tracing", -] - -[[package]] -name = "hashbrown" -version = "0.12.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" - [[package]] name = "hashbrown" version = "0.14.0" @@ -696,28 +481,6 @@ checksum = "443144c8cdadd93ebf52ddb4056d257f5b52c04d3c804e657d19eb73fc33668b" [[package]] name = "heroku-ruby-buildpack" version = "0.0.0" -dependencies = [ - "commons", - "flate2", - "fs-err", - "glob", - "indoc", - "libcnb 0.14.0", - "libcnb-test 0.14.0", - "rand", - "regex", - "serde", - "tar", - "tempfile", - "thiserror", - "toml", - "ureq", - "url", -] - -[[package]] -name = "heroku-statsd-metrics" -version = "0.0.0" dependencies = [ "clap", "commons", @@ -725,119 +488,19 @@ dependencies = [ "fs-err", "glob", "indoc", - "libcnb 0.13.0", - "libcnb-test 0.13.0", - "libherokubuildpack 0.12.0", + "libcnb", + "libcnb-test", "rand", "regex", "serde", "tar", "tempfile", "thiserror", + "toml", "ureq", "url", ] -[[package]] -name = "hex" -version = "0.4.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" - -[[package]] -name = "http" -version = "0.2.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bd6effc99afb63425aff9b05836f029929e345a6148a14b7ecd5ab67af944482" -dependencies = [ - "bytes", - "fnv", - "itoa", -] - -[[package]] -name = "http-body" -version = "0.4.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d5f38f16d184e36f2408a55281cd658ecbd3ca05cce6d6510a176eca393e26d1" -dependencies = [ - "bytes", - "http", - "pin-project-lite", -] - -[[package]] -name = "httparse" -version = "1.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d897f394bad6a705d5f4104762e116a75639e470d80901eed05a860a95cb1904" - -[[package]] -name = "httpdate" -version = "1.0.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9" - -[[package]] -name = "hyper" -version = "0.14.27" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ffb1cfd654a8219eaef89881fdb3bb3b1cdc5fa75ded05d6933b2b382e395468" -dependencies = [ - "bytes", - "futures-channel", - "futures-core", - "futures-util", - "h2", - "http", - "http-body", - "httparse", - "httpdate", - "itoa", - "pin-project-lite", - "socket2 0.4.9", - "tokio", - "tower-service", - "tracing", - "want", -] - -[[package]] -name = "hyperlocal" -version = "0.8.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fafdf7b2b2de7c9784f76e02c0935e65a8117ec3b768644379983ab333ac98c" -dependencies = [ - "futures-util", - "hex", - "hyper", - "pin-project", - "tokio", -] - -[[package]] -name = "iana-time-zone" -version = "0.1.57" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2fad5b825842d2b38bd206f3e81d6957625fd7f0a361e345c30e01a0ae2dd613" -dependencies = [ - "android_system_properties", - "core-foundation-sys", - "iana-time-zone-haiku", - "js-sys", - "wasm-bindgen", - "windows", -] - -[[package]] -name = "iana-time-zone-haiku" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" -dependencies = [ - "cc", -] - [[package]] name = "idna" version = "0.4.0" @@ -848,17 +511,6 @@ dependencies = [ "unicode-normalization", ] -[[package]] -name = "indexmap" -version = "1.9.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99" -dependencies = [ - "autocfg", - "hashbrown 0.12.3", - "serde", -] - [[package]] name = "indexmap" version = "2.0.0" @@ -866,7 +518,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d5477fe2230a79769d8dc68e0eabf5437907c0457a5614a9e8dddb67f65eb65d" dependencies = [ "equivalent", - "hashbrown 0.14.0", + "hashbrown", ] [[package]] @@ -920,83 +572,17 @@ version = "0.2.147" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b4668fb0ea861c1df094127ac5f1da3409a82116a4ba74fca2e58ef927159bb3" -[[package]] -name = "libcnb" -version = "0.12.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "027cd4a736600564c4e7aebf124eabb9d7dc622bcfeefb414cc7c4c7d7ac6595" -dependencies = [ - "libcnb-data 0.12.0", - "libcnb-proc-macros 0.12.0", - "serde", - "thiserror", - "toml", -] - -[[package]] -name = "libcnb" -version = "0.13.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "39d5e0e5f0ea6fe91d867457289d88c4f56631e37fac072d11676ff970715012" -dependencies = [ - "libcnb-data 0.13.0 (registry+https://github.com/rust-lang/crates.io-index)", - "libcnb-proc-macros 0.13.0 (registry+https://github.com/rust-lang/crates.io-index)", - "serde", - "thiserror", - "toml", -] - [[package]] name = "libcnb" version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5132851c82d808e6b42edd1cc9e7cb9b16b0274c325b25fdb42660fae9b2e88b" dependencies = [ - "libcnb-data 0.14.0", - "libcnb-proc-macros 0.14.0", - "serde", - "thiserror", - "toml", -] - -[[package]] -name = "libcnb-data" -version = "0.12.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e8840246c7aced3307fa193edc5d26482f92f992986a33860b6b3b523a67975" -dependencies = [ - "fancy-regex", - "libcnb-proc-macros 0.12.0", - "serde", - "thiserror", - "toml", -] - -[[package]] -name = "libcnb-data" -version = "0.13.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "631bda3e80115baf38894609cde58b796d3b3fc0f47cca369321c230df53d563" -dependencies = [ - "fancy-regex", - "libcnb-proc-macros 0.13.0 (registry+https://github.com/rust-lang/crates.io-index)", + "libcnb-data", + "libcnb-proc-macros", "serde", "thiserror", "toml", - "uriparse", -] - -[[package]] -name = "libcnb-data" -version = "0.13.0" -source = "git+https://github.com/heroku/libcnb.rs?branch=libcnb_test_local_and_meta_buildpack_support#2042df4578e9a908617b2aca656cba0557ee525b" -dependencies = [ - "fancy-regex", - "libcnb-proc-macros 0.13.0 (git+https://github.com/heroku/libcnb.rs?branch=libcnb_test_local_and_meta_buildpack_support)", - "serde", - "thiserror", - "toml", - "uriparse", ] [[package]] @@ -1006,133 +592,52 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4bed8b0f2676aebeb216a7f7872151fbf74ff3706c7027449573c03c9d7f3393" dependencies = [ "fancy-regex", - "libcnb-proc-macros 0.14.0", + "libcnb-proc-macros", "serde", "thiserror", "toml", "uriparse", ] -[[package]] -name = "libcnb-package" -version = "0.13.0" -source = "git+https://github.com/heroku/libcnb.rs?branch=libcnb_test_local_and_meta_buildpack_support#2042df4578e9a908617b2aca656cba0557ee525b" -dependencies = [ - "cargo_metadata 0.15.4", - "libcnb-data 0.13.0 (git+https://github.com/heroku/libcnb.rs?branch=libcnb_test_local_and_meta_buildpack_support)", - "petgraph", - "toml", - "which", -] - [[package]] name = "libcnb-package" version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "412f8c3ee7e2fcaff1acd1926a238cac271e86d7258c51038053fac17d85d144" dependencies = [ - "cargo_metadata 0.17.0", - "libcnb-data 0.14.0", + "cargo_metadata", + "libcnb-data", "petgraph", "toml", "which", ] -[[package]] -name = "libcnb-proc-macros" -version = "0.12.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c9e31cc93a20d00f1d54ecb55dcff681669871e6d8a8b63ac0320e77fca1987c" -dependencies = [ - "cargo_metadata 0.15.4", - "fancy-regex", - "quote", - "syn", -] - -[[package]] -name = "libcnb-proc-macros" -version = "0.13.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ab33c1d63ffd280516abc7ada744fc1b653a888c439f5e2962d5371d0aecaf7" -dependencies = [ - "cargo_metadata 0.15.4", - "fancy-regex", - "quote", - "syn", -] - -[[package]] -name = "libcnb-proc-macros" -version = "0.13.0" -source = "git+https://github.com/heroku/libcnb.rs?branch=libcnb_test_local_and_meta_buildpack_support#2042df4578e9a908617b2aca656cba0557ee525b" -dependencies = [ - "cargo_metadata 0.15.4", - "fancy-regex", - "quote", - "syn", -] - [[package]] name = "libcnb-proc-macros" version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c715cec438b3a02c3564e9b9c20a78c54b9c71874249bec1e3d45fcd2537cfcf" dependencies = [ - "cargo_metadata 0.17.0", + "cargo_metadata", "fancy-regex", "quote", "syn", ] -[[package]] -name = "libcnb-test" -version = "0.13.0" -source = "git+https://github.com/heroku/libcnb.rs?branch=libcnb_test_local_and_meta_buildpack_support#2042df4578e9a908617b2aca656cba0557ee525b" -dependencies = [ - "bollard", - "fastrand", - "fs_extra", - "libcnb-data 0.13.0 (git+https://github.com/heroku/libcnb.rs?branch=libcnb_test_local_and_meta_buildpack_support)", - "libcnb-package 0.13.0", - "serde", - "tempfile", - "tokio", - "tokio-stream", -] - [[package]] name = "libcnb-test" version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "92ee152780ab4ad6e4aca8ec12767c090677294b7aa4ffc1a90633d38b429c9e" dependencies = [ - "cargo_metadata 0.17.0", + "cargo_metadata", "fastrand", "fs_extra", - "libcnb-data 0.14.0", - "libcnb-package 0.14.0", + "libcnb-data", + "libcnb-package", "tempfile", ] -[[package]] -name = "libherokubuildpack" -version = "0.12.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4ee2764ebf688454c4fcbcd7c52ff277b5cb2e196c502ff4ce92de563cb10ea2" -dependencies = [ - "crossbeam-utils", - "flate2", - "libcnb 0.12.0", - "pathdiff", - "sha2", - "tar", - "termcolor", - "thiserror", - "toml", - "ureq", -] - [[package]] name = "libherokubuildpack" version = "0.14.0" @@ -1141,7 +646,7 @@ checksum = "999689d1a9f8cbea478ae7c4ce6136601e7abe9512ccfc0409f5525949a41457" dependencies = [ "crossbeam-utils", "flate2", - "libcnb 0.14.0", + "libcnb", "pathdiff", "sha2", "tar", @@ -1198,17 +703,6 @@ dependencies = [ "adler", ] -[[package]] -name = "mio" -version = "0.8.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "927a765cd3fc26206e66b296465fa9d3e5ab003e651c1b3c060e7956d96b19d2" -dependencies = [ - "libc", - "wasi", - "windows-sys", -] - [[package]] name = "num-traits" version = "0.2.16" @@ -1228,15 +722,6 @@ dependencies = [ "libc", ] -[[package]] -name = "object" -version = "0.32.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77ac5bbd07aea88c60a577a1ce218075ffd59208b2d7ca97adf9bfc5aeb21ebe" -dependencies = [ - "memchr", -] - [[package]] name = "once_cell" version = "1.18.0" @@ -1271,41 +756,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e1d3afd2628e69da2be385eb6f2fd57c8ac7977ceeff6dc166ff1657b0e386a9" dependencies = [ "fixedbitset", - "indexmap 2.0.0", -] - -[[package]] -name = "pin-project" -version = "1.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fda4ed1c6c173e3fc7a83629421152e01d7b1f9b7f65fb301e490e8cfc656422" -dependencies = [ - "pin-project-internal", -] - -[[package]] -name = "pin-project-internal" -version = "1.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4359fd9c9171ec6e8c62926d6faaf553a8dc3f64e1507e76da7911b4f6a04405" -dependencies = [ - "proc-macro2", - "quote", - "syn", + "indexmap", ] -[[package]] -name = "pin-project-lite" -version = "0.2.13" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8afb450f006bf6385ca15ef45d71d2288452bc3683ce2e2cacc0d18e4be60b58" - -[[package]] -name = "pin-utils" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" - [[package]] name = "pkg-config" version = "0.3.27" @@ -1441,12 +894,6 @@ dependencies = [ "winapi", ] -[[package]] -name = "rustc-demangle" -version = "0.1.23" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d626bb9dae77e28219937af045c257c28bfd3f69333c512553507f5f9798cb76" - [[package]] name = "rustix" version = "0.38.9" @@ -1563,17 +1010,6 @@ dependencies = [ "serde", ] -[[package]] -name = "serde_repr" -version = "0.1.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8725e1dfadb3a50f7e5ce0b1a540466f6ed3fe7a0fca2ac2b8b831d31316bd00" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - [[package]] name = "serde_spanned" version = "0.6.3" @@ -1583,33 +1019,6 @@ dependencies = [ "serde", ] -[[package]] -name = "serde_urlencoded" -version = "0.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd" -dependencies = [ - "form_urlencoded", - "itoa", - "ryu", - "serde", -] - -[[package]] -name = "serde_with" -version = "2.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07ff71d2c147a7b57362cead5e22f772cd52f6ab31cfcd9edcd7f6aeb2a0afbe" -dependencies = [ - "base64 0.13.1", - "chrono", - "hex", - "indexmap 1.9.3", - "serde", - "serde_json", - "time", -] - [[package]] name = "sha2" version = "0.10.7" @@ -1621,35 +1030,6 @@ dependencies = [ "digest", ] -[[package]] -name = "slab" -version = "0.4.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f92a496fb766b417c996b9c5e57daf2f7ad3b0bebe1ccfca4856390e3d3bb67" -dependencies = [ - "autocfg", -] - -[[package]] -name = "socket2" -version = "0.4.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "64a4a911eed85daf18834cfaa86a79b7d266ff93ff5ba14005426219480ed662" -dependencies = [ - "libc", - "winapi", -] - -[[package]] -name = "socket2" -version = "0.5.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2538b18701741680e0322a2302176d3253a35388e2e62f172f64f4f16605f877" -dependencies = [ - "libc", - "windows-sys", -] - [[package]] name = "spin" version = "0.5.2" @@ -1726,34 +1106,6 @@ dependencies = [ "syn", ] -[[package]] -name = "time" -version = "0.3.28" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "17f6bb557fd245c28e6411aa56b6403c689ad95061f50e4be16c274e70a17e48" -dependencies = [ - "deranged", - "itoa", - "serde", - "time-core", - "time-macros", -] - -[[package]] -name = "time-core" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7300fbefb4dadc1af235a9cef3737cea692a9d97e1b9cbcd4ebdae6f8868e6fb" - -[[package]] -name = "time-macros" -version = "0.2.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a942f44339478ef67935ab2bbaec2fb0322496cf3cbe84b261e06ac3814c572" -dependencies = [ - "time-core", -] - [[package]] name = "tinyvec" version = "1.6.0" @@ -1769,47 +1121,6 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" -[[package]] -name = "tokio" -version = "1.32.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "17ed6077ed6cd6c74735e21f37eb16dc3935f96878b1fe961074089cc80893f9" -dependencies = [ - "backtrace", - "bytes", - "libc", - "mio", - "num_cpus", - "pin-project-lite", - "socket2 0.5.3", - "windows-sys", -] - -[[package]] -name = "tokio-stream" -version = "0.1.14" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "397c988d37662c7dda6d2208364a706264bf3d6138b11d436cbac0ad38832842" -dependencies = [ - "futures-core", - "pin-project-lite", - "tokio", -] - -[[package]] -name = "tokio-util" -version = "0.7.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "806fe8c2c87eccc8b3267cbae29ed3ab2d0bd37fca70ab622e46aaa9375ddb7d" -dependencies = [ - "bytes", - "futures-core", - "futures-sink", - "pin-project-lite", - "tokio", - "tracing", -] - [[package]] name = "toml" version = "0.7.6" @@ -1837,45 +1148,13 @@ version = "0.19.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8123f27e969974a3dfba720fdb560be359f57b44302d280ba72e76a74480e8a" dependencies = [ - "indexmap 2.0.0", + "indexmap", "serde", "serde_spanned", "toml_datetime", "winnow", ] -[[package]] -name = "tower-service" -version = "0.3.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6bc1c9ce2b5135ac7f93c72918fc37feb872bdc6a5533a8b85eb4b86bfdae52" - -[[package]] -name = "tracing" -version = "0.1.37" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ce8c33a8d48bd45d624a6e523445fd21ec13d3653cd51f681abf67418f54eb8" -dependencies = [ - "cfg-if", - "pin-project-lite", - "tracing-core", -] - -[[package]] -name = "tracing-core" -version = "0.1.31" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0955b8137a1df6f1a2e9a37d8a6656291ff0297c1a97c24e0d8425fe2312f79a" -dependencies = [ - "once_cell", -] - -[[package]] -name = "try-lock" -version = "0.2.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3528ecfd12c466c6f163363caf2d02a71161dd5e1cc6ae7b34207ea2d42d81ed" - [[package]] name = "typenum" version = "1.16.0" @@ -1915,7 +1194,7 @@ version = "2.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0b11c96ac7ee530603dcdf68ed1557050f374ce55a5a07193ebf8cbc9f8927e9" dependencies = [ - "base64 0.21.3", + "base64", "flate2", "log", "once_cell", @@ -1944,7 +1223,6 @@ dependencies = [ "form_urlencoded", "idna", "percent-encoding", - "serde", ] [[package]] @@ -1981,15 +1259,6 @@ dependencies = [ "winapi-util", ] -[[package]] -name = "want" -version = "0.3.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e" -dependencies = [ - "try-lock", -] - [[package]] name = "wasi" version = "0.11.0+wasi-snapshot-preview1" @@ -2124,15 +1393,6 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" -[[package]] -name = "windows" -version = "0.48.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e686886bc078bc1b0b600cac0147aadb815089b6e4da64016cbd754b6342700f" -dependencies = [ - "windows-targets", -] - [[package]] name = "windows-sys" version = "0.48.0" diff --git a/Cargo.toml b/Cargo.toml index 0a609cf1..65fbbbee 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,10 +1,5 @@ [workspace] -resolver = "2" -members = [ - "buildpacks/metrics-agent", - "buildpacks/ruby", - "commons" -] +members = ["buildpacks/ruby", "commons"] [workspace.package] rust-version = "1.64" diff --git a/buildpacks/metrics-agent/CHANGELOG.md b/buildpacks/metrics-agent/CHANGELOG.md deleted file mode 100644 index 1410f16b..00000000 --- a/buildpacks/metrics-agent/CHANGELOG.md +++ /dev/null @@ -1,4 +0,0 @@ -# Changelog - -## [Unreleased] - diff --git a/buildpacks/metrics-agent/Cargo.toml b/buildpacks/metrics-agent/Cargo.toml deleted file mode 100644 index 82ae6c09..00000000 --- a/buildpacks/metrics-agent/Cargo.toml +++ /dev/null @@ -1,29 +0,0 @@ -[package] -name = "heroku-statsd-metrics" -# This crate is not published, so the only version that is used is the one in buildpack.toml. -version = "0.0.0" -publish = false -edition.workspace = true -rust-version.workspace = true - -[dependencies] -commons = { path = "../../commons" } -flate2 = "1" -fs-err = "2" -indoc = "2" -libcnb = "0.13" -rand = "0.8" -regex = "1" -serde = "1" -tar = "0.4" -tempfile = "3" -thiserror = "1" -ureq = "2" -url = { version = "2", features = ["serde"] } -glob = "0.3" -clap = { version = "4.3.17", features = ["derive"] } -libherokubuildpack.workspace = true - - -[dev-dependencies] -libcnb-test = { git = "https://github.com/heroku/libcnb.rs", branch = "libcnb_test_local_and_meta_buildpack_support" } diff --git a/buildpacks/metrics-agent/buildpack.toml b/buildpacks/metrics-agent/buildpack.toml deleted file mode 100644 index 5d4005d9..00000000 --- a/buildpacks/metrics-agent/buildpack.toml +++ /dev/null @@ -1,23 +0,0 @@ -api = "0.9" - -[buildpack] -id = "heroku/statsd-metrics" -version = "2.0.0" -name = "Statsd Metrics" -homepage = "https://github.com/heroku/buildpacks-ruby" -description = "Installs an agent to send language specific metrics back to heroku via statsd" -keywords = ["statsd", "metrics"] - -[[stacks]] -id = "heroku-20" - -[[stacks]] -id = "heroku-22" - -[[buildpack.licenses]] -type = "BSD-3-Clause" - -[metadata] -[metadata.release] -[metadata.release.image] -repository = "docker.io/heroku/buildpack-statsd-metrics" diff --git a/buildpacks/metrics-agent/src/layers/mod.rs b/buildpacks/metrics-agent/src/layers/mod.rs deleted file mode 100644 index 96155e8c..00000000 --- a/buildpacks/metrics-agent/src/layers/mod.rs +++ /dev/null @@ -1 +0,0 @@ -pub(crate) mod install_agentmon; diff --git a/buildpacks/metrics-agent/src/main.rs b/buildpacks/metrics-agent/src/main.rs deleted file mode 100644 index ce63a595..00000000 --- a/buildpacks/metrics-agent/src/main.rs +++ /dev/null @@ -1,101 +0,0 @@ -mod layers; - -use crate::layers::install_agentmon::{DownloadAgentmonError, InstallAgentmon}; -use commons::build_output::{self, fmt::ErrorInfo}; -use indoc::formatdoc; -use libcnb::{ - build::{BuildContext, BuildResult, BuildResultBuilder}, - buildpack_main, - data::{build_plan::BuildPlanBuilder, layer_name}, - detect::{DetectContext, DetectResult, DetectResultBuilder}, - Buildpack, -}; - -#[derive(Debug)] -enum MetricsAgentError { - DownloadAgentmonError(DownloadAgentmonError), -} - -impl From for libcnb::Error { - fn from(error: MetricsAgentError) -> Self { - libcnb::Error::BuildpackError(error) - } -} - -buildpack_main!(MetricsAgentBuildpack); - -pub(crate) struct MetricsAgentBuildpack; - -impl Buildpack for MetricsAgentBuildpack { - type Platform = libcnb::generic::GenericPlatform; - type Metadata = libcnb::generic::GenericMetadata; - type Error = MetricsAgentError; - - fn detect(&self, context: DetectContext) -> libcnb::Result { - let plan_builder = BuildPlanBuilder::new().provides("heroku-statsd-metrics-agent"); - - if let Ok(true) = fs_err::read_to_string(context.app_dir.join("Gemfile.lock")) - .map(|lockfile| lockfile.contains("barnes")) - { - DetectResultBuilder::pass() - .build_plan(plan_builder.requires("heroku-statsd-metrics-agent").build()) - .build() - } else { - DetectResultBuilder::pass() - .build_plan(plan_builder.build()) - .build() - } - } - - fn build(&self, context: BuildContext) -> libcnb::Result { - let build_duration = build_output::buildpack_name("Heroku StatsD Metrics Agent"); - - let section = build_output::section("Metrics agent"); - context.handle_layer( - layer_name!("statsd-metrics-agent"), - InstallAgentmon { section }, - )?; - - build_duration.done_timed(); - BuildResultBuilder::new().build() - } - - fn on_error(&self, err: libcnb::Error) { - on_error(err); - } -} - -pub(crate) fn on_error(err: libcnb::Error) { - match err { - libcnb::Error::BuildpackError(error) => log_our_error(error), - error => ErrorInfo::header_body_details( - "heroku/buildpack-ruby internal buildpack error", - formatdoc! {" - An unexpected internal error was reported by the framework used - by this buildpack. - - If the issue persists, consider opening an issue on the GitHub - repository. If you are unable to deploy to Heroku as a result - of this issue, consider opening a ticket for additional support. - "}, - error, - ) - .print(), - }; -} - -fn log_our_error(error: MetricsAgentError) { - match error { - MetricsAgentError::DownloadAgentmonError(error) => ErrorInfo::header_body_details( - formatdoc! { - "Could not install Statsd agent" - }, - formatdoc! { - "An error occured while downloading and installing the metrics agent - the buildpack cannot continue" - }, - error, - ) - .print(), - } -} diff --git a/buildpacks/metrics-agent/tests/integration_test.rs b/buildpacks/metrics-agent/tests/integration_test.rs deleted file mode 100644 index 118cefda..00000000 --- a/buildpacks/metrics-agent/tests/integration_test.rs +++ /dev/null @@ -1,33 +0,0 @@ -#![warn(clippy::pedantic)] - -use libcnb_test::{assert_contains, BuildConfig, BuildpackReference, ContainerConfig, TestRunner}; -use std::path::PathBuf; - -#[test] -#[ignore = "integration test"] -fn test_barnes_app() { - TestRunner::default().build( - BuildConfig::new("heroku/builder:22", "tests/fixtures/barnes_app").buildpacks(vec![ - BuildpackReference::Crate, - BuildpackReference::Local(PathBuf::from("../ruby")), - ]), - |context| { - assert_contains!(context.pack_stdout, "# Heroku StatsD Metrics Agent"); - assert_contains!(context.pack_stdout, "# Heroku Ruby Buildpack"); - - context.start_container( - ContainerConfig::new() - .entrypoint(["launcher"]) - .envs(vec![ - ("HEROKU_METRICS_URL", "example.com"), - ("DYNO", "web.1"), - ]) - .command(["ps x"]), - |container| { - let log_output = container.logs_wait(); - assert_contains!(log_output.stdout, "agentmon_loop --path"); - }, - ); - }, - ); -} diff --git a/buildpacks/ruby/Cargo.toml b/buildpacks/ruby/Cargo.toml index 4afc6df6..f30514e6 100644 --- a/buildpacks/ruby/Cargo.toml +++ b/buildpacks/ruby/Cargo.toml @@ -23,6 +23,7 @@ tempfile = "3" thiserror = "1" ureq = "2" url = "2" +clap = { version = "4.4.1", features = ["derive"] } [dev-dependencies] libcnb-test = "=0.14.0" diff --git a/buildpacks/metrics-agent/src/bin/agentmon_loop.rs b/buildpacks/ruby/src/bin/agentmon_loop.rs similarity index 83% rename from buildpacks/metrics-agent/src/bin/agentmon_loop.rs rename to buildpacks/ruby/src/bin/agentmon_loop.rs index c00bcf5a..2d3dac1f 100644 --- a/buildpacks/metrics-agent/src/bin/agentmon_loop.rs +++ b/buildpacks/ruby/src/bin/agentmon_loop.rs @@ -20,7 +20,9 @@ const SLEEP_FOR: Duration = Duration::from_secs(1); /// /// Example: /// +/// ```shell /// $ cargo run --bin agentmon_loop -- --path +/// ``` /// Turn CLI arguments into a Rust struct #[derive(Parser, Debug)] @@ -37,7 +39,7 @@ fn main() { exit(1); } - let agentmon_args = match build_args(std::env::vars().collect::>()) { + let agentmon_args = match build_args(&std::env::vars().collect::>()) { Ok(args) => args, Err(e) => { eprintln!("Cannot start agentmon: {e}"); @@ -90,22 +92,21 @@ enum Error { /// /// # Errors /// -/// - PORT is not set -/// - HEROKU_METRICS_URL is not set -/// - DYNO starts with `run.` -fn build_args(env: HashMap) -> Result, Error> { +/// - Environment variables: PORT or `HEROKU_METRICS_URL` are not set +/// - Environment variable DYNO starts with `run.` +fn build_args(env: &HashMap) -> Result, Error> { let mut args = Vec::new(); - if let Some(true) = env.get("DYNO").map(|value| value.starts_with("run.")) { + if env.get("DYNO").is_some_and(|value| value.starts_with("run.")) { return Err(Error::RunDynoDetected); } if let Some(port) = env.get("PORT") { - args.push(format!("statsd-addr=:{port}")); + args.push(format!("-statsd-addr=:{port}")); } else { return Err(Error::MissingPort); }; - if let Some(true) = env.get("AGENTMON_DEBUG").map(|value| value == "true") { + if env.get("AGENTMON_DEBUG").is_some_and(|value| value == "true") { args.push("-debug".to_string()); }; @@ -124,7 +125,7 @@ mod test { #[test] fn missing_port() { - let result = build_args(HashMap::new()); + let result = build_args(&HashMap::new()); assert_eq!(result, Err(Error::MissingPort)); } @@ -138,12 +139,12 @@ mod test { "https://example.com".to_string(), ); - let result = build_args(env); + let result = build_args(&env); assert_eq!( result, Ok(vec![ - "statsd-addr=:90210".to_string(), + "-statsd-addr=:90210".to_string(), "https://example.com".to_string() ]) ); @@ -159,12 +160,12 @@ mod test { ); env.insert("AGENTMON_DEBUG".to_string(), "true".to_string()); - let result = build_args(env); + let result = build_args(&env); assert_eq!( result, Ok(vec![ - "statsd-addr=:90210".to_string(), + "-statsd-addr=:90210".to_string(), "-debug".to_string(), "https://example.com".to_string() ]) diff --git a/buildpacks/ruby/src/bin/launch_daemon.rs b/buildpacks/ruby/src/bin/launch_daemon.rs new file mode 100644 index 00000000..ce689645 --- /dev/null +++ b/buildpacks/ruby/src/bin/launch_daemon.rs @@ -0,0 +1,77 @@ +use clap::Parser; +use std::path::PathBuf; +use std::process::exit; +use std::process::Command; + +/// Schedules agentmon to run as a background daemon + +/// CLI argument parser +/// +/// ```shell +/// $ cargo run --bin launch_daemon \ +/// --log \ +/// --agentmon +/// --loop-path +/// ``` +#[derive(Parser, Debug)] +struct Args { + #[arg(long, value_parser = absolute_path_exists)] + log: PathBuf, + + #[arg(long, value_parser = absolute_path_exists)] + agentmon: PathBuf, + + #[arg(long, value_parser = absolute_path_exists)] + loop_path: PathBuf, +} + +fn absolute_path_exists(s: &str) -> Result { + fs_err::canonicalize(PathBuf::from(s)) + .map_err(|e| format!("{s} is not a valid path. Details: {e}")) + .and_then(|path| match path.try_exists() { + Ok(true) => Ok(path), + Ok(false) => Err(format!("path {} does not exist", path.display())), + Err(e) => Err(format!("problem verifying {} exists {e}", path.display())), + }) +} + +fn main() { + let Args { + log, + loop_path, + agentmon, + } = Args::parse(); + + let mut command = Command::new("start-stop-daemon"); + if std::env::var_os("AGENTMON_DEBUG").is_some() { + command.args(["--output", &log.to_string_lossy()]); + } else { + match fs_err::write(&log, "To enable logging run with AGENTMON_DEBUG=1") { + Ok(_) => {} + Err(error) => eprintln!( + "Could not write to log file {}. Reason: {error}", + log.display() + ), + }; + } + command.args([ + "--start", + "--background", + "--exec", + &loop_path.to_string_lossy(), + "--", + "--path", + &agentmon.to_string_lossy(), + ]); + + match command.spawn().map(|mut child| child.wait()) { + Ok(_) => {} + Err(error) => { + eprintln!( + "Command failed {}. Details: {error}", + commons::fun_run::display(&mut command) + ); + exit(1) + } + }; +} diff --git a/buildpacks/ruby/src/layers.rs b/buildpacks/ruby/src/layers.rs index 822c6800..62c00630 100644 --- a/buildpacks/ruby/src/layers.rs +++ b/buildpacks/ruby/src/layers.rs @@ -1,5 +1,6 @@ mod bundle_download_layer; mod bundle_install_layer; +pub(crate) mod metrics_agent_install; mod ruby_install_layer; pub(crate) use self::bundle_download_layer::BundleDownloadLayer; diff --git a/buildpacks/metrics-agent/src/layers/install_agentmon.rs b/buildpacks/ruby/src/layers/metrics_agent_install.rs similarity index 72% rename from buildpacks/metrics-agent/src/layers/install_agentmon.rs rename to buildpacks/ruby/src/layers/metrics_agent_install.rs index abe79350..6e20a46c 100644 --- a/buildpacks/metrics-agent/src/layers/install_agentmon.rs +++ b/buildpacks/ruby/src/layers/metrics_agent_install.rs @@ -1,5 +1,5 @@ use crate::build_output; -use crate::{MetricsAgentBuildpack, MetricsAgentError}; +use crate::{RubyBuildpack, RubyBuildpackError}; use flate2::read::GzDecoder; use libcnb::data::layer_content_metadata::LayerTypes; use libcnb::layer::ExistingLayerStrategy; @@ -16,8 +16,8 @@ use tempfile::NamedTempFile; /// Agentmon URL /// -/// - Repo: https://github.com/heroku/agentmon -/// - Releases: https://github.com/heroku/agentmon/releases +/// - Repo: +/// - Releases: /// /// To get the latest s3 url: /// @@ -28,7 +28,7 @@ const DOWNLOAD_URL: &str = "https://agentmon-releases.s3.amazonaws.com/agentmon-0.3.1-linux-amd64.tar.gz"; #[derive(Debug)] -pub(crate) struct InstallAgentmon { +pub(crate) struct MetricsAgentInstall { pub(crate) section: build_output::Section, } @@ -38,7 +38,7 @@ pub(crate) struct Metadata { } #[derive(thiserror::Error, Debug)] -pub(crate) enum InstallAgentmonError { +pub(crate) enum MetricsAgentInstallError { #[error("Could not read file permissions {0}")] PermissionError(std::io::Error), @@ -59,8 +59,8 @@ pub(crate) enum InstallAgentmonError { CouldNotWriteDestinationFile(std::io::Error), } -impl Layer for InstallAgentmon { - type Buildpack = MetricsAgentBuildpack; +impl Layer for MetricsAgentInstall { + type Buildpack = RubyBuildpack; type Metadata = Metadata; fn types(&self) -> libcnb::data::layer_content_metadata::LayerTypes { @@ -83,17 +83,17 @@ impl Layer for InstallAgentmon { let mut timer = self.section.say_with_inline_timer("Downloading"); let agentmon = - agentmon_download(&bin_dir).map_err(MetricsAgentError::InstallAgentmonError)?; + agentmon_download(&bin_dir).map_err(RubyBuildpackError::MetricsAgentError)?; timer.done(); self.section.say("Writing scripts"); let execd = write_execd_script(&agentmon, layer_path) - .map_err(MetricsAgentError::InstallAgentmonError)?; + .map_err(RubyBuildpackError::MetricsAgentError)?; LayerResultBuilder::new(Metadata { download_url: Some(DOWNLOAD_URL.to_string()), }) - .exec_d_program("spawn-agentmon", execd) + .exec_d_program("spawn_metrics_agent", execd) .build() } @@ -109,7 +109,7 @@ impl Layer for InstallAgentmon { self.section.say("Writing scripts"); let execd = write_execd_script(&layer_path.join("bin").join("agentmon"), layer_path) - .map_err(MetricsAgentError::InstallAgentmonError)?; + .map_err(RubyBuildpackError::MetricsAgentError)?; LayerResultBuilder::new(Metadata { download_url: Some(DOWNLOAD_URL.to_string()), @@ -132,7 +132,7 @@ impl Layer for InstallAgentmon { Some(url) => { self.section.say_with_details( "Updating metrics agent", - format!("{} to {}", url, DOWNLOAD_URL), + format!("{url} to {DOWNLOAD_URL}"), ); Ok(ExistingLayerStrategy::Recreate) } @@ -155,25 +155,28 @@ impl Layer for InstallAgentmon { } } -fn write_execd_script(agentmon: &Path, layer_path: &Path) -> Result { +fn write_execd_script( + agentmon: &Path, + layer_path: &Path, +) -> Result { let log = layer_path.join("output.log"); let execd = layer_path.join("execd"); let daemon = layer_path.join("launch_daemon"); let run_loop = layer_path.join("agentmon_loop"); // Ensure log file exists - fs_err::write(&log, "").map_err(InstallAgentmonError::CouldNotWriteDestinationFile)?; + fs_err::write(&log, "").map_err(MetricsAgentInstallError::CouldNotWriteDestinationFile)?; // agentmon_loop boots agentmon continuously fs_err::copy( additional_buildpack_binary_path!("agentmon_loop"), &run_loop, ) - .map_err(InstallAgentmonError::CouldNotWriteDestinationFile)?; + .map_err(MetricsAgentInstallError::CouldNotWriteDestinationFile)?; // The `launch_daemon` schedules `agentmon_loop` to run in the background fs_err::copy(additional_buildpack_binary_path!("launch_daemon"), &daemon) - .map_err(InstallAgentmonError::CouldNotWriteDestinationFile)?; + .map_err(MetricsAgentInstallError::CouldNotWriteDestinationFile)?; // The execd bash script will be run by CNB lifecycle, it runs the `launch_daemon` fs_err::write( @@ -181,50 +184,53 @@ fn write_execd_script(agentmon: &Path, layer_path: &Path) -> Result Result { +fn agentmon_download(dir: &Path) -> Result { download_to_dir(DOWNLOAD_URL, dir)?; Ok(dir.join("agentmon")) } -fn download_to_dir(url: impl AsRef, destination: &Path) -> Result<(), InstallAgentmonError> { +fn download_to_dir( + url: impl AsRef, + destination: &Path, +) -> Result<(), MetricsAgentInstallError> { let agentmon_tgz = - NamedTempFile::new().map_err(DownloadAgentmonError::CouldNotCreateDestinationFile)?; + NamedTempFile::new().map_err(MetricsAgentInstallError::CouldNotCreateDestinationFile)?; download(url, agentmon_tgz.path())?; untar(agentmon_tgz.path(), destination)?; - chmod_plus_x(&destination.join("agentmon")).map_err(DownloadAgentmonError::PermissionError)?; + chmod_plus_x(&destination.join("agentmon")) + .map_err(MetricsAgentInstallError::PermissionError)?; Ok(()) } -pub(crate) fn untar( +fn untar( path: impl AsRef, destination: impl AsRef, -) -> Result<(), DownloadAgentmonError> { +) -> Result<(), MetricsAgentInstallError> { let file = - fs_err::File::open(path.as_ref()).map_err(DownloadAgentmonError::CouldNotOpenFile)?; + fs_err::File::open(path.as_ref()).map_err(MetricsAgentInstallError::CouldNotOpenFile)?; Archive::new(GzDecoder::new(file)) .unpack(destination.as_ref()) - .map_err(DownloadAgentmonError::CouldNotUnpack) + .map_err(MetricsAgentInstallError::CouldNotUnpack) } /// Sets file permissions on the given path to 7xx (similar to `chmod +x `) @@ -232,7 +238,7 @@ pub(crate) fn untar( /// i.e. chmod +x will ensure that the first digit /// of the file permission is 7 on unix so if you pass /// in 0o455 it would be mutated to 0o755 -pub fn chmod_plus_x(path: &Path) -> Result<(), std::io::Error> { +fn chmod_plus_x(path: &Path) -> Result<(), std::io::Error> { let mut perms = fs_err::metadata(path)?.permissions(); let mut mode = perms.mode(); mode |= 0o700; @@ -241,29 +247,20 @@ pub fn chmod_plus_x(path: &Path) -> Result<(), std::io::Error> { fs_err::set_permissions(path, perms) } -/// Write a script to the target path while adding a bash shebang line and setting execution permissions -fn write_bash_script(path: &Path, script: impl AsRef) -> std::io::Result<()> { - let script = script.as_ref(); - fs_err::write(path, format!("#!/usr/bin/env bash\n\n{script}"))?; - chmod_plus_x(path)?; - - Ok(()) -} - -pub(crate) fn download( +fn download( uri: impl AsRef, destination: impl AsRef, -) -> Result<(), DownloadAgentmonError> { +) -> Result<(), MetricsAgentInstallError> { let mut response_reader = ureq::get(uri.as_ref()) .call() - .map_err(|err| DownloadAgentmonError::RequestError(Box::new(err)))? + .map_err(|err| MetricsAgentInstallError::RequestError(Box::new(err)))? .into_reader(); let mut destination_file = fs_err::File::create(destination.as_ref()) - .map_err(DownloadAgentmonError::CouldNotCreateDestinationFile)?; + .map_err(MetricsAgentInstallError::CouldNotCreateDestinationFile)?; std::io::copy(&mut response_reader, &mut destination_file) - .map_err(DownloadAgentmonError::CouldNotWriteDestinationFile)?; + .map_err(MetricsAgentInstallError::CouldNotWriteDestinationFile)?; Ok(()) } @@ -280,11 +277,6 @@ mod tests { let before = file.metadata().unwrap().permissions().mode(); - let foo = before | 0o777; - - dbg!(before); - dbg!(foo); - chmod_plus_x(&file).unwrap(); let after = file.metadata().unwrap().permissions().mode(); diff --git a/buildpacks/ruby/src/main.rs b/buildpacks/ruby/src/main.rs index 084197e8..911c490c 100644 --- a/buildpacks/ruby/src/main.rs +++ b/buildpacks/ruby/src/main.rs @@ -1,6 +1,7 @@ #![warn(unused_crate_dependencies)] #![warn(clippy::pedantic)] #![allow(clippy::module_name_repetitions)] +use crate::layers::metrics_agent_install::{MetricsAgentInstall, MetricsAgentInstallError}; use crate::layers::{RubyInstallError, RubyInstallLayer}; use crate::rake_task_detect::RakeError; use commons::build_output; @@ -30,6 +31,8 @@ mod user_errors; #[cfg(test)] use libcnb_test as _; +use clap as _; + pub(crate) struct RubyBuildpack; impl Buildpack for RubyBuildpack { @@ -77,6 +80,15 @@ impl Buildpack for RubyBuildpack { let bundler_version = gemfile_lock.resolve_bundler("2.4.5"); let ruby_version = gemfile_lock.resolve_ruby("3.1.3"); + // ## Install metrics agent + if lockfile_contents.contains("barnes") { + let section = build_output::section("Metrics agent"); + context.handle_layer( + layer_name!("metrics_agent"), + MetricsAgentInstall { section }, + )?; + } + // ## Install executable ruby version env = { @@ -179,6 +191,7 @@ pub(crate) enum RubyBuildpackError { RakeDetectError(RakeError), GemListGetError(gem_list::ListError), RubyInstallError(RubyInstallError), + MetricsAgentError(MetricsAgentInstallError), MissingGemfileLock(std::io::Error), InAppDirCacheError(CacheError), BundleInstallDigestError(commons::metadata_digest::DigestError), diff --git a/buildpacks/ruby/src/user_errors.rs b/buildpacks/ruby/src/user_errors.rs index 81a39dc9..236bbdac 100644 --- a/buildpacks/ruby/src/user_errors.rs +++ b/buildpacks/ruby/src/user_errors.rs @@ -70,6 +70,17 @@ fn log_our_error(error: RubyBuildpackError) { error, ) .print(), + RubyBuildpackError::MetricsAgentError(error) => ErrorInfo::header_body_details( + formatdoc! { + "Could not install Statsd agent" + }, + formatdoc! { + "An error occured while downloading and installing the metrics agent + the buildpack cannot continue" + }, + error, + ) + .print(), RubyBuildpackError::BundleInstallDigestError(error) => ErrorInfo::header_body_details( "Could not generate digest", formatdoc! {" diff --git a/buildpacks/metrics-agent/tests/fixtures/barnes_app/Gemfile b/buildpacks/ruby/tests/fixtures/barnes_app/Gemfile similarity index 100% rename from buildpacks/metrics-agent/tests/fixtures/barnes_app/Gemfile rename to buildpacks/ruby/tests/fixtures/barnes_app/Gemfile diff --git a/buildpacks/metrics-agent/tests/fixtures/barnes_app/Gemfile.lock b/buildpacks/ruby/tests/fixtures/barnes_app/Gemfile.lock similarity index 100% rename from buildpacks/metrics-agent/tests/fixtures/barnes_app/Gemfile.lock rename to buildpacks/ruby/tests/fixtures/barnes_app/Gemfile.lock diff --git a/buildpacks/metrics-agent/tests/fixtures/barnes_app/config.ru b/buildpacks/ruby/tests/fixtures/barnes_app/config.ru similarity index 100% rename from buildpacks/metrics-agent/tests/fixtures/barnes_app/config.ru rename to buildpacks/ruby/tests/fixtures/barnes_app/config.ru diff --git a/buildpacks/metrics-agent/tests/fixtures/barnes_app/config/puma.rb b/buildpacks/ruby/tests/fixtures/barnes_app/config/puma.rb similarity index 100% rename from buildpacks/metrics-agent/tests/fixtures/barnes_app/config/puma.rb rename to buildpacks/ruby/tests/fixtures/barnes_app/config/puma.rb diff --git a/buildpacks/ruby/tests/integration_test.rs b/buildpacks/ruby/tests/integration_test.rs index ab25139d..ffe2123e 100644 --- a/buildpacks/ruby/tests/integration_test.rs +++ b/buildpacks/ruby/tests/integration_test.rs @@ -112,6 +112,38 @@ fn test_ruby_app_with_yarn_app() { ); } +#[test] +#[ignore = "integration test"] +fn test_barnes_app() { + TestRunner::default().build( + BuildConfig::new("heroku/builder:22", "tests/fixtures/barnes_app").buildpacks(vec![ + BuildpackReference::Crate, + ]), + |context| { + assert_contains!(context.pack_stdout, "# Heroku StatsD Metrics Agent"); + assert_contains!(context.pack_stdout, "# Heroku Ruby Buildpack"); + + context.start_container( + ContainerConfig::new() + .entrypoint("launcher") + .envs(vec![ + ("HEROKU_METRICS_URL", "example.com"), + ("DYNO", "web.1"), + ("PORT", "1234"), + ]) + .command(["ps x"]), + |container| { + let log_output = container.logs_wait(); + println!("{}", log_output.stdout); + println!("{}", log_output.stderr); + + assert_contains!(log_output.stdout, "agentmon_loop --path"); + }, + ); + }, + ); +} + fn request_container( container: &ContainerContext, port: u16, From e988a9e24c9770bc145f664d1db3218fc97e8056 Mon Sep 17 00:00:00 2001 From: Richard Schneeman Date: Mon, 28 Aug 2023 16:47:46 -0500 Subject: [PATCH 17/40] Revert "Update build output" This reverts commit 5f449e6a6d00d29edbefe8e4745e27f7705f8ba6. --- buildpacks/ruby/src/bin/agentmon_loop.rs | 10 +++++++-- .../ruby/src/layers/metrics_agent_install.rs | 6 ++--- buildpacks/ruby/tests/integration_test.rs | 5 ++--- commons/src/build_output.rs | 22 ++++++------------- 4 files changed, 19 insertions(+), 24 deletions(-) diff --git a/buildpacks/ruby/src/bin/agentmon_loop.rs b/buildpacks/ruby/src/bin/agentmon_loop.rs index 2d3dac1f..16fe1b44 100644 --- a/buildpacks/ruby/src/bin/agentmon_loop.rs +++ b/buildpacks/ruby/src/bin/agentmon_loop.rs @@ -96,7 +96,10 @@ enum Error { /// - Environment variable DYNO starts with `run.` fn build_args(env: &HashMap) -> Result, Error> { let mut args = Vec::new(); - if env.get("DYNO").is_some_and(|value| value.starts_with("run.")) { + if env + .get("DYNO") + .is_some_and(|value| value.starts_with("run.")) + { return Err(Error::RunDynoDetected); } @@ -106,7 +109,10 @@ fn build_args(env: &HashMap) -> Result, Error> { return Err(Error::MissingPort); }; - if env.get("AGENTMON_DEBUG").is_some_and(|value| value == "true") { + if env + .get("AGENTMON_DEBUG") + .is_some_and(|value| value == "true") + { args.push("-debug".to_string()); }; diff --git a/buildpacks/ruby/src/layers/metrics_agent_install.rs b/buildpacks/ruby/src/layers/metrics_agent_install.rs index 6e20a46c..7aad6b40 100644 --- a/buildpacks/ruby/src/layers/metrics_agent_install.rs +++ b/buildpacks/ruby/src/layers/metrics_agent_install.rs @@ -130,10 +130,8 @@ impl Layer for MetricsAgentInstall { Ok(ExistingLayerStrategy::Update) } Some(url) => { - self.section.say_with_details( - "Updating metrics agent", - format!("{url} to {DOWNLOAD_URL}"), - ); + self.section + .say_with_details("Updating metrics agent", format!("{url} to {DOWNLOAD_URL}")); Ok(ExistingLayerStrategy::Recreate) } None => Ok(ExistingLayerStrategy::Recreate), diff --git a/buildpacks/ruby/tests/integration_test.rs b/buildpacks/ruby/tests/integration_test.rs index ffe2123e..6453b4c6 100644 --- a/buildpacks/ruby/tests/integration_test.rs +++ b/buildpacks/ruby/tests/integration_test.rs @@ -116,9 +116,8 @@ fn test_ruby_app_with_yarn_app() { #[ignore = "integration test"] fn test_barnes_app() { TestRunner::default().build( - BuildConfig::new("heroku/builder:22", "tests/fixtures/barnes_app").buildpacks(vec![ - BuildpackReference::Crate, - ]), + BuildConfig::new("heroku/builder:22", "tests/fixtures/barnes_app") + .buildpacks(vec![BuildpackReference::Crate]), |context| { assert_contains!(context.pack_stdout, "# Heroku StatsD Metrics Agent"); assert_contains!(context.pack_stdout, "# Heroku Ruby Buildpack"); diff --git a/commons/src/build_output.rs b/commons/src/build_output.rs index 10cb8596..1bc6cb91 100644 --- a/commons/src/build_output.rs +++ b/commons/src/build_output.rs @@ -533,15 +533,14 @@ pub mod fmt { let noun = noun.as_ref(); let header = header.as_ref(); let body = help_url(body, url); - - let contents = colorize( + colorize( color, bangify(formatdoc! {" {noun} {header} - {body}"}), - ); - format!("\n{contents}") + {body} + "}), + ) } #[must_use] @@ -601,18 +600,11 @@ pub mod fmt { debug_details, } = info; + let body = look_at_me(ERROR_COLOR, "ERROR:", header, body, url); if let Some(details) = debug_details { - let message = look_at_me( - ERROR_COLOR, - "ERROR:", - header, - format!("{body}\n\nDebug information: "), - url, - ); - - format!("{message}\n{details}\n") + format!("{body}\n\nDebug information: {details}") } else { - look_at_me(ERROR_COLOR, "ERROR:", header, body, url) + body } } From be5d0719ed2ea162255f158153e9613f39b8cd45 Mon Sep 17 00:00:00 2001 From: Richard Schneeman Date: Mon, 28 Aug 2023 17:49:57 -0500 Subject: [PATCH 18/40] Changelog entry and fix tests --- buildpacks/ruby/CHANGELOG.md | 4 +++- buildpacks/ruby/tests/integration_test.rs | 1 - 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/buildpacks/ruby/CHANGELOG.md b/buildpacks/ruby/CHANGELOG.md index 0ff092b8..a418ef3a 100644 --- a/buildpacks/ruby/CHANGELOG.md +++ b/buildpacks/ruby/CHANGELOG.md @@ -7,6 +7,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +- buildpacks/ruby: Introduce heroku build metrics support (https://github.com/heroku/buildpacks-ruby/pull/172) + ## [2.0.1] - 2023-07-25 - Commons: Introduce `build_output` module (https://github.com/heroku/buildpacks-ruby/pull/155) @@ -19,4 +21,4 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Version 2.0.0 for the first release is not a typo. There was an issue in pack where a builder with the same name and version number would reuse artifacts left on image from [prior runs which caused issues](https://github.com/buildpacks/pack/issues/1322#issuecomment-1038241038). There were prior releases of `heroku/ruby` CNB from different sources that triggered this problem. To ensure no one would encounter that issue we developed and released using a version we know has not been used before. Version 2.0 was the first major version without a prior release of `heroku/ruby` CNB from any source. [unreleased]: https://github.com/heroku/buildpacks-ruby/compare/v2.0.1...HEAD -[2.0.1]: https://github.com/heroku/buildpacks-ruby/releases/tag/v2.0.1 \ No newline at end of file +[2.0.1]: https://github.com/heroku/buildpacks-ruby/releases/tag/v2.0.1 diff --git a/buildpacks/ruby/tests/integration_test.rs b/buildpacks/ruby/tests/integration_test.rs index 6453b4c6..cbc7e560 100644 --- a/buildpacks/ruby/tests/integration_test.rs +++ b/buildpacks/ruby/tests/integration_test.rs @@ -119,7 +119,6 @@ fn test_barnes_app() { BuildConfig::new("heroku/builder:22", "tests/fixtures/barnes_app") .buildpacks(vec![BuildpackReference::Crate]), |context| { - assert_contains!(context.pack_stdout, "# Heroku StatsD Metrics Agent"); assert_contains!(context.pack_stdout, "# Heroku Ruby Buildpack"); context.start_container( From ab2e6da8014115af02c67de5b5e3cc3cc9db6710 Mon Sep 17 00:00:00 2001 From: Richard Schneeman Date: Mon, 11 Sep 2023 10:51:14 -0500 Subject: [PATCH 19/40] Update buildpacks/ruby/Cargo.toml Co-authored-by: Ed Morley <501702+edmorley@users.noreply.github.com> --- buildpacks/ruby/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/buildpacks/ruby/Cargo.toml b/buildpacks/ruby/Cargo.toml index f30514e6..7f7e70c8 100644 --- a/buildpacks/ruby/Cargo.toml +++ b/buildpacks/ruby/Cargo.toml @@ -23,7 +23,7 @@ tempfile = "3" thiserror = "1" ureq = "2" url = "2" -clap = { version = "4.4.1", features = ["derive"] } +clap = { version = "4", features = ["derive"] } [dev-dependencies] libcnb-test = "=0.14.0" From ce540e47ed2a644842bb22fb551f60a87ba2176c Mon Sep 17 00:00:00 2001 From: Richard Schneeman Date: Mon, 11 Sep 2023 10:50:26 -0500 Subject: [PATCH 20/40] Prefer unwrap_or_else over match with Ok(_) --- buildpacks/ruby/src/bin/launch_daemon.rs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/buildpacks/ruby/src/bin/launch_daemon.rs b/buildpacks/ruby/src/bin/launch_daemon.rs index ce689645..9e0336a9 100644 --- a/buildpacks/ruby/src/bin/launch_daemon.rs +++ b/buildpacks/ruby/src/bin/launch_daemon.rs @@ -64,14 +64,14 @@ fn main() { &agentmon.to_string_lossy(), ]); - match command.spawn().map(|mut child| child.wait()) { - Ok(_) => {} - Err(error) => { + command + .spawn() + .and_then(|mut child| child.wait()) + .unwrap_or_else(|error| { eprintln!( "Command failed {}. Details: {error}", commons::fun_run::display(&mut command) ); exit(1) - } - }; + }); } From 2897d75f7af624b4788657e42cb97037d429bfbb Mon Sep 17 00:00:00 2001 From: Richard Schneeman Date: Mon, 11 Sep 2023 10:52:43 -0500 Subject: [PATCH 21/40] Remove changelog prefix --- buildpacks/ruby/CHANGELOG.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/buildpacks/ruby/CHANGELOG.md b/buildpacks/ruby/CHANGELOG.md index a418ef3a..26d8182b 100644 --- a/buildpacks/ruby/CHANGELOG.md +++ b/buildpacks/ruby/CHANGELOG.md @@ -7,7 +7,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] -- buildpacks/ruby: Introduce heroku build metrics support (https://github.com/heroku/buildpacks-ruby/pull/172) +- Introduce heroku build metrics support (https://github.com/heroku/buildpacks-ruby/pull/172) +- Changelog moved to be per-crate rather than for the whole project (https://github.com/heroku/buildpacks-ruby/pull/154) ## [2.0.1] - 2023-07-25 From e41ae1f76e85f103e9f4880906148d65d1a334ff Mon Sep 17 00:00:00 2001 From: Richard Schneeman Date: Mon, 11 Sep 2023 10:56:35 -0500 Subject: [PATCH 22/40] Prefer unwrap_or_else over match with Ok(_) --- buildpacks/ruby/src/bin/launch_daemon.rs | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/buildpacks/ruby/src/bin/launch_daemon.rs b/buildpacks/ruby/src/bin/launch_daemon.rs index 9e0336a9..edaa1c7d 100644 --- a/buildpacks/ruby/src/bin/launch_daemon.rs +++ b/buildpacks/ruby/src/bin/launch_daemon.rs @@ -46,13 +46,12 @@ fn main() { if std::env::var_os("AGENTMON_DEBUG").is_some() { command.args(["--output", &log.to_string_lossy()]); } else { - match fs_err::write(&log, "To enable logging run with AGENTMON_DEBUG=1") { - Ok(_) => {} - Err(error) => eprintln!( + fs_err::write(&log, "To enable logging run with AGENTMON_DEBUG=1").unwrap_or_else(|error| { + eprintln!( "Could not write to log file {}. Reason: {error}", log.display() - ), - }; + ) + }) } command.args([ "--start", From aade4c0593fdd2797e7b6ea8c754646b30dece31 Mon Sep 17 00:00:00 2001 From: Richard Schneeman Date: Mon, 11 Sep 2023 11:00:16 -0500 Subject: [PATCH 23/40] Prefer HasMap::from() over mut HashMap::new() --- buildpacks/ruby/src/bin/agentmon_loop.rs | 28 +++++++++++++----------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/buildpacks/ruby/src/bin/agentmon_loop.rs b/buildpacks/ruby/src/bin/agentmon_loop.rs index 16fe1b44..596447b1 100644 --- a/buildpacks/ruby/src/bin/agentmon_loop.rs +++ b/buildpacks/ruby/src/bin/agentmon_loop.rs @@ -138,12 +138,13 @@ mod test { #[test] fn agentmon_args() { - let mut env = HashMap::new(); - env.insert("PORT".to_string(), "90210".to_string()); - env.insert( - "HEROKU_METRICS_URL".to_string(), - "https://example.com".to_string(), - ); + let env = HashMap::from([ + ("PORT".to_string(), "90210".to_string()), + ( + "HEROKU_METRICS_URL".to_string(), + "https://example.com".to_string(), + ), + ]); let result = build_args(&env); @@ -158,13 +159,14 @@ mod test { #[test] fn agentmon_debug_args() { - let mut env = HashMap::new(); - env.insert("PORT".to_string(), "90210".to_string()); - env.insert( - "HEROKU_METRICS_URL".to_string(), - "https://example.com".to_string(), - ); - env.insert("AGENTMON_DEBUG".to_string(), "true".to_string()); + let env = HashMap::from([ + ("PORT".to_string(), "90210".to_string()), + ( + "HEROKU_METRICS_URL".to_string(), + "https://example.com".to_string(), + ), + ("AGENTMON_DEBUG".to_string(), "true".to_string()), + ]); let result = build_args(&env); From dcea7647d1213a774139ccfde732c0531744b1bd Mon Sep 17 00:00:00 2001 From: Richard Schneeman Date: Mon, 11 Sep 2023 11:01:13 -0500 Subject: [PATCH 24/40] Fix docs --- buildpacks/ruby/src/bin/agentmon_loop.rs | 2 +- buildpacks/ruby/src/bin/launch_daemon.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/buildpacks/ruby/src/bin/agentmon_loop.rs b/buildpacks/ruby/src/bin/agentmon_loop.rs index 596447b1..ec7b791e 100644 --- a/buildpacks/ruby/src/bin/agentmon_loop.rs +++ b/buildpacks/ruby/src/bin/agentmon_loop.rs @@ -27,7 +27,7 @@ const SLEEP_FOR: Duration = Duration::from_secs(1); /// Turn CLI arguments into a Rust struct #[derive(Parser, Debug)] struct Args { - /// Path to the agentmon executable e.g. --path + /// Path to the agentmon executable e.g. --path #[arg(short, long)] path: PathBuf, } diff --git a/buildpacks/ruby/src/bin/launch_daemon.rs b/buildpacks/ruby/src/bin/launch_daemon.rs index edaa1c7d..4027b41e 100644 --- a/buildpacks/ruby/src/bin/launch_daemon.rs +++ b/buildpacks/ruby/src/bin/launch_daemon.rs @@ -10,7 +10,7 @@ use std::process::Command; /// ```shell /// $ cargo run --bin launch_daemon \ /// --log \ -/// --agentmon +/// --agentmon \ /// --loop-path /// ``` #[derive(Parser, Debug)] From a7d3d4c401095671dfb1ff520ac4e373b47f90dc Mon Sep 17 00:00:00 2001 From: Richard Schneeman Date: Mon, 11 Sep 2023 13:56:27 -0500 Subject: [PATCH 25/40] Use `Path::try_exists()` instead of `exists()` Differentiate between "no such file" versus "there's a file here, but something stops us from accessing it. --- buildpacks/ruby/src/bin/agentmon_loop.rs | 50 ++++++++++++++---------- 1 file changed, 29 insertions(+), 21 deletions(-) diff --git a/buildpacks/ruby/src/bin/agentmon_loop.rs b/buildpacks/ruby/src/bin/agentmon_loop.rs index ec7b791e..cc80a6d4 100644 --- a/buildpacks/ruby/src/bin/agentmon_loop.rs +++ b/buildpacks/ruby/src/bin/agentmon_loop.rs @@ -34,29 +34,37 @@ struct Args { fn main() { let agentmon = Args::parse().path; - if !agentmon.exists() { - eprintln!("Path does not exist {}", agentmon.display()); - exit(1); - } - - let agentmon_args = match build_args(&std::env::vars().collect::>()) { - Ok(args) => args, - Err(e) => { - eprintln!("Cannot start agentmon: {e}"); + let agentmon_args = build_args(&std::env::vars().collect::>()) + .unwrap_or_else(|error| { + eprintln!("Cannot start agentmon. {error}"); exit(1) + }); + + match agentmon.try_exists() { + Ok(true) => loop { + match run(&agentmon, &agentmon_args) { + Ok(status) => { + eprintln!("Process completed with status={status}, sleeping {SLEEP_FOR:?}"); + } + Err(error) => { + eprintln!( + "Process could not run due to error. {error}, sleeping {SLEEP_FOR:?}" + ); + } + }; + sleep(SLEEP_FOR); + }, + Ok(false) => { + eprintln!("Path does not exist {path}", path = agentmon.display()); + exit(1); + } + Err(error) => { + eprintln!( + "Could not access {path}. {error}", + path = agentmon.display() + ); + exit(1); } - }; - - loop { - match run(&agentmon, &agentmon_args) { - Ok(status) => { - eprintln!("process completed with status=${status}, sleeping {SLEEP_FOR:?}"); - } - Err(error) => { - eprintln!("process could not be run due to error: {error}, sleeping {SLEEP_FOR:?}"); - } - }; - sleep(SLEEP_FOR); } } From b68c1778123890852feb551657c35ace18b46233 Mon Sep 17 00:00:00 2001 From: Richard Schneeman Date: Mon, 11 Sep 2023 14:00:51 -0500 Subject: [PATCH 26/40] Fix stringly typed errors The error result of `absolute_path_exists` was `String` to normalize the error type and allow inserting the path name. This refactors that logic into a proper error enum. --- buildpacks/ruby/src/bin/launch_daemon.rs | 35 ++++++++++++++++++------ 1 file changed, 27 insertions(+), 8 deletions(-) diff --git a/buildpacks/ruby/src/bin/launch_daemon.rs b/buildpacks/ruby/src/bin/launch_daemon.rs index 4027b41e..22f6092a 100644 --- a/buildpacks/ruby/src/bin/launch_daemon.rs +++ b/buildpacks/ruby/src/bin/launch_daemon.rs @@ -25,14 +25,33 @@ struct Args { loop_path: PathBuf, } -fn absolute_path_exists(s: &str) -> Result { - fs_err::canonicalize(PathBuf::from(s)) - .map_err(|e| format!("{s} is not a valid path. Details: {e}")) - .and_then(|path| match path.try_exists() { - Ok(true) => Ok(path), - Ok(false) => Err(format!("path {} does not exist", path.display())), - Err(e) => Err(format!("problem verifying {} exists {e}", path.display())), - }) +#[derive(Debug, thiserror::Error)] +enum ParseAbsoluteError { + #[error("Cannot determine cannonical path for {0}. {1}")] + CannotCanonicalize(PathBuf, std::io::Error), + + #[error("Path does not exist {0}")] + DoesNotExist(PathBuf), + + #[error("Cannot read {0}. {1}")] + CannotRead(PathBuf, std::io::Error), +} + +/// Used to validate a path pased to the CLI exists and is accessible +fn absolute_path_exists(input: &str) -> Result { + let input = PathBuf::from(input); + let path = input + .canonicalize() + .map_err(|error| ParseAbsoluteError::CannotCanonicalize(input, error))?; + + if path + .try_exists() + .map_err(|error| ParseAbsoluteError::CannotRead(path.clone(), error))? + { + Ok(path) + } else { + Err(ParseAbsoluteError::DoesNotExist(path)) + } } fn main() { From 283e600a79aee2a56a83733203ad34ca93f1f154 Mon Sep 17 00:00:00 2001 From: Richard Schneeman Date: Mon, 11 Sep 2023 14:42:57 -0500 Subject: [PATCH 27/40] Env var key is static & add disable instructions To ensure we're not misspelling the key name somewhere in the output I made it a static. I also added notes on how to disable the feature consistent with how it currently works (and also how all env var based enable/disable flags work in the existing buildpack I.e. they check for presence, not for content. This is also consistent with the Ruby language as `0`, `"0"`, and `"false"` (string false) are all "truthy". The only "falsey" values in Ruby are `false` (boolean false ) and `nil` --- buildpacks/ruby/src/bin/launch_daemon.rs | 26 +++++++++++++++++++++--- 1 file changed, 23 insertions(+), 3 deletions(-) diff --git a/buildpacks/ruby/src/bin/launch_daemon.rs b/buildpacks/ruby/src/bin/launch_daemon.rs index 22f6092a..1e86b716 100644 --- a/buildpacks/ruby/src/bin/launch_daemon.rs +++ b/buildpacks/ruby/src/bin/launch_daemon.rs @@ -3,6 +3,8 @@ use std::path::PathBuf; use std::process::exit; use std::process::Command; +static AGENTMON_DEBUG: &str = "AGENTMON_DEBUG"; + /// Schedules agentmon to run as a background daemon /// CLI argument parser @@ -62,16 +64,34 @@ fn main() { } = Args::parse(); let mut command = Command::new("start-stop-daemon"); - if std::env::var_os("AGENTMON_DEBUG").is_some() { + if let Some(value) = std::env::var_os(AGENTMON_DEBUG) { + fs_err::write( + &log, + format!( + "Logging enabled via `{AGENTMON_DEBUG}={value:?}`. To disable `unset {AGENTMON_DEBUG}`" + ), + ) + .unwrap_or_else(|error| { + eprintln!( + "Could not write to log file {}. Reason: {error}", + log.display() + ) + }); + command.args(["--output", &log.to_string_lossy()]); } else { - fs_err::write(&log, "To enable logging run with AGENTMON_DEBUG=1").unwrap_or_else(|error| { + fs_err::write( + &log, + format!("To enable logging run with {AGENTMON_DEBUG}=1"), + ) + .unwrap_or_else(|error| { eprintln!( "Could not write to log file {}. Reason: {error}", log.display() ) - }) + }); } + command.args([ "--start", "--background", From 991adb4fd9d8e2e97570ce6e560087b36915ea32 Mon Sep 17 00:00:00 2001 From: Richard Schneeman Date: Mon, 11 Sep 2023 14:51:30 -0500 Subject: [PATCH 28/40] Static env var keys and tests Ed mentioned "There are three error variants, but only one is tested? Should there be tests for the other cases?" So I added test cases for them. Moved env var key name to statics for consistency. --- buildpacks/ruby/src/bin/agentmon_loop.rs | 42 ++++++++++++++++-------- 1 file changed, 28 insertions(+), 14 deletions(-) diff --git a/buildpacks/ruby/src/bin/agentmon_loop.rs b/buildpacks/ruby/src/bin/agentmon_loop.rs index cc80a6d4..5b8ae67d 100644 --- a/buildpacks/ruby/src/bin/agentmon_loop.rs +++ b/buildpacks/ruby/src/bin/agentmon_loop.rs @@ -12,6 +12,12 @@ use std::{ thread::sleep, time::Duration, }; + +static PORT: &str = "PORT"; +static DYNO: &str = "DYNO"; +static AGENTMON_DEBUG: &str = "AGENTMON_DEBUG"; +static HEROKU_METRICS_URL: &str = "HEROKU_METRICS_URL"; + const SLEEP_FOR: Duration = Duration::from_secs(1); /// Agentmon Loop @@ -86,13 +92,13 @@ where #[derive(Debug, thiserror::Error, PartialEq)] enum Error { - #[error("PORT environment variable is not set")] + #[error("{PORT} environment variable is not set")] MissingPort, - #[error("HEROKU_METRICS_URL environment variable is not set")] + #[error("{HEROKU_METRICS_URL} environment variable is not set")] MissingMetricsUrl, - #[error("One off dyno detected i.e. DYNO=\"run.*\"")] + #[error("One off dyno detected i.e. {DYNO}=\"run.*\"")] RunDynoDetected, } @@ -104,27 +110,21 @@ enum Error { /// - Environment variable DYNO starts with `run.` fn build_args(env: &HashMap) -> Result, Error> { let mut args = Vec::new(); - if env - .get("DYNO") - .is_some_and(|value| value.starts_with("run.")) - { + if env.get(DYNO).is_some_and(|value| value.starts_with("run.")) { return Err(Error::RunDynoDetected); } - if let Some(port) = env.get("PORT") { + if let Some(port) = env.get(PORT) { args.push(format!("-statsd-addr=:{port}")); } else { return Err(Error::MissingPort); }; - if env - .get("AGENTMON_DEBUG") - .is_some_and(|value| value == "true") - { + if env.get(AGENTMON_DEBUG).is_some_and(|value| value == "true") { args.push("-debug".to_string()); }; - if let Some(url) = env.get("HEROKU_METRICS_URL") { + if let Some(url) = env.get(HEROKU_METRICS_URL) { args.push(url.clone()); } else { return Err(Error::MissingMetricsUrl); @@ -137,6 +137,20 @@ fn build_args(env: &HashMap) -> Result, Error> { mod test { use super::*; + #[test] + fn missing_run_dyno() { + let result = build_args(&HashMap::from([("DYNO".to_string(), "run.1".to_string())])); + + assert_eq!(result, Err(Error::RunDynoDetected)); + } + + #[test] + fn missing_metrics_url() { + let result = build_args(&HashMap::from([("PORT".to_string(), "123".to_string())])); + + assert_eq!(result, Err(Error::MissingMetricsUrl)); + } + #[test] fn missing_port() { let result = build_args(&HashMap::new()); @@ -145,7 +159,7 @@ mod test { } #[test] - fn agentmon_args() { + fn agentmon_statsd_addr() { let env = HashMap::from([ ("PORT".to_string(), "90210".to_string()), ( From abcf16deba12e3bca4dbebb334252b7af4b2963f Mon Sep 17 00:00:00 2001 From: Richard Schneeman Date: Mon, 11 Sep 2023 14:54:22 -0500 Subject: [PATCH 29/40] Make error more specific > Ed: This struct is only returned by one function, rather than being a general error struct for the whole binary. Should it have a more specific name than Error? eg BuildArgsError or similar? Here's what that looks like. I'm okay with either. I think this is better if someone needs to add a different error in the future as this will guide them to make a new enum where appropriate. --- buildpacks/ruby/src/bin/agentmon_loop.rs | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/buildpacks/ruby/src/bin/agentmon_loop.rs b/buildpacks/ruby/src/bin/agentmon_loop.rs index 5b8ae67d..c7f3ec63 100644 --- a/buildpacks/ruby/src/bin/agentmon_loop.rs +++ b/buildpacks/ruby/src/bin/agentmon_loop.rs @@ -91,7 +91,7 @@ where } #[derive(Debug, thiserror::Error, PartialEq)] -enum Error { +enum BuildArgsError { #[error("{PORT} environment variable is not set")] MissingPort, @@ -108,16 +108,16 @@ enum Error { /// /// - Environment variables: PORT or `HEROKU_METRICS_URL` are not set /// - Environment variable DYNO starts with `run.` -fn build_args(env: &HashMap) -> Result, Error> { +fn build_args(env: &HashMap) -> Result, BuildArgsError> { let mut args = Vec::new(); if env.get(DYNO).is_some_and(|value| value.starts_with("run.")) { - return Err(Error::RunDynoDetected); + return Err(BuildArgsError::RunDynoDetected); } if let Some(port) = env.get(PORT) { args.push(format!("-statsd-addr=:{port}")); } else { - return Err(Error::MissingPort); + return Err(BuildArgsError::MissingPort); }; if env.get(AGENTMON_DEBUG).is_some_and(|value| value == "true") { @@ -127,7 +127,7 @@ fn build_args(env: &HashMap) -> Result, Error> { if let Some(url) = env.get(HEROKU_METRICS_URL) { args.push(url.clone()); } else { - return Err(Error::MissingMetricsUrl); + return Err(BuildArgsError::MissingMetricsUrl); }; Ok(args) @@ -141,21 +141,21 @@ mod test { fn missing_run_dyno() { let result = build_args(&HashMap::from([("DYNO".to_string(), "run.1".to_string())])); - assert_eq!(result, Err(Error::RunDynoDetected)); + assert_eq!(result, Err(BuildArgsError::RunDynoDetected)); } #[test] fn missing_metrics_url() { let result = build_args(&HashMap::from([("PORT".to_string(), "123".to_string())])); - assert_eq!(result, Err(Error::MissingMetricsUrl)); + assert_eq!(result, Err(BuildArgsError::MissingMetricsUrl)); } #[test] fn missing_port() { let result = build_args(&HashMap::new()); - assert_eq!(result, Err(Error::MissingPort)); + assert_eq!(result, Err(BuildArgsError::MissingPort)); } #[test] From daa1074165ca014a613a617e7bfb39db3cbeb2b5 Mon Sep 17 00:00:00 2001 From: Richard Schneeman Date: Mon, 11 Sep 2023 14:55:43 -0500 Subject: [PATCH 30/40] Simplify spawning daemon --- buildpacks/ruby/src/bin/agentmon_loop.rs | 2 +- buildpacks/ruby/src/bin/launch_daemon.rs | 17 +++++++---------- 2 files changed, 8 insertions(+), 11 deletions(-) diff --git a/buildpacks/ruby/src/bin/agentmon_loop.rs b/buildpacks/ruby/src/bin/agentmon_loop.rs index c7f3ec63..161afd52 100644 --- a/buildpacks/ruby/src/bin/agentmon_loop.rs +++ b/buildpacks/ruby/src/bin/agentmon_loop.rs @@ -87,7 +87,7 @@ where eprintln!("Running: {}", commons::fun_run::display(&mut cmd)); - cmd.spawn().and_then(|mut child| child.wait()) + cmd.status() } #[derive(Debug, thiserror::Error, PartialEq)] diff --git a/buildpacks/ruby/src/bin/launch_daemon.rs b/buildpacks/ruby/src/bin/launch_daemon.rs index 1e86b716..18c4dec2 100644 --- a/buildpacks/ruby/src/bin/launch_daemon.rs +++ b/buildpacks/ruby/src/bin/launch_daemon.rs @@ -102,14 +102,11 @@ fn main() { &agentmon.to_string_lossy(), ]); - command - .spawn() - .and_then(|mut child| child.wait()) - .unwrap_or_else(|error| { - eprintln!( - "Command failed {}. Details: {error}", - commons::fun_run::display(&mut command) - ); - exit(1) - }); + command.status().unwrap_or_else(|error| { + eprintln!( + "Command failed {}. Details: {error}", + commons::fun_run::display(&mut command) + ); + exit(1) + }); } From b5540ad8a54806bad811cc14e832f1783a87e32d Mon Sep 17 00:00:00 2001 From: Richard Schneeman Date: Mon, 11 Sep 2023 15:20:58 -0500 Subject: [PATCH 31/40] Match functionality to function name The `download_to_dir` did more than download, it also untar-d and set executable permissions. The `agentmon_download` was a very loose wrapper around `download_to_dir`. I made removed the chmod functionality from `download_to_dir` and renamed it `download_untar`. I moved the chmod functionality into `agentmond_download` and renamed it `agentmon_install`. It's a minor change but I think it's an improvement. --- .../ruby/src/layers/metrics_agent_install.rs | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/buildpacks/ruby/src/layers/metrics_agent_install.rs b/buildpacks/ruby/src/layers/metrics_agent_install.rs index 7aad6b40..d6594290 100644 --- a/buildpacks/ruby/src/layers/metrics_agent_install.rs +++ b/buildpacks/ruby/src/layers/metrics_agent_install.rs @@ -82,8 +82,8 @@ impl Layer for MetricsAgentInstall { let bin_dir = layer_path.join("bin"); let mut timer = self.section.say_with_inline_timer("Downloading"); - let agentmon = - agentmon_download(&bin_dir).map_err(RubyBuildpackError::MetricsAgentError)?; + let agentmon = install_agentmon(&bin_dir).map_err(RubyBuildpackError::MetricsAgentError)?; + timer.done(); self.section.say("Writing scripts"); @@ -196,13 +196,14 @@ fn write_execd_script( Ok(execd) } -fn agentmon_download(dir: &Path) -> Result { - download_to_dir(DOWNLOAD_URL, dir)?; +fn install_agentmon(dir: &Path) -> Result { + let agentmon = download_untar(DOWNLOAD_URL, &dir).map(|_| dir.join("agentmon"))?; - Ok(dir.join("agentmon")) + chmod_plus_x(&agentmon).map_err(MetricsAgentInstallError::PermissionError)?; + Ok(agentmon) } -fn download_to_dir( +fn download_untar( url: impl AsRef, destination: &Path, ) -> Result<(), MetricsAgentInstallError> { @@ -213,9 +214,6 @@ fn download_to_dir( untar(agentmon_tgz.path(), destination)?; - chmod_plus_x(&destination.join("agentmon")) - .map_err(MetricsAgentInstallError::PermissionError)?; - Ok(()) } From 8e40054cd132144a2a07633fab20179acfc7bbd4 Mon Sep 17 00:00:00 2001 From: Richard Schneeman Date: Mon, 11 Sep 2023 15:27:30 -0500 Subject: [PATCH 32/40] Log when barnes is installed, or isn't This presents a tip to the user to tell them how to enable metrics collection. --- buildpacks/ruby/src/layers/metrics_agent_install.rs | 2 +- buildpacks/ruby/src/main.rs | 9 +++++++-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/buildpacks/ruby/src/layers/metrics_agent_install.rs b/buildpacks/ruby/src/layers/metrics_agent_install.rs index d6594290..52e9848a 100644 --- a/buildpacks/ruby/src/layers/metrics_agent_install.rs +++ b/buildpacks/ruby/src/layers/metrics_agent_install.rs @@ -197,7 +197,7 @@ fn write_execd_script( } fn install_agentmon(dir: &Path) -> Result { - let agentmon = download_untar(DOWNLOAD_URL, &dir).map(|_| dir.join("agentmon"))?; + let agentmon = download_untar(DOWNLOAD_URL, dir).map(|_| dir.join("agentmon"))?; chmod_plus_x(&agentmon).map_err(MetricsAgentInstallError::PermissionError)?; Ok(agentmon) diff --git a/buildpacks/ruby/src/main.rs b/buildpacks/ruby/src/main.rs index 911c490c..ef09f267 100644 --- a/buildpacks/ruby/src/main.rs +++ b/buildpacks/ruby/src/main.rs @@ -81,13 +81,18 @@ impl Buildpack for RubyBuildpack { let ruby_version = gemfile_lock.resolve_ruby("3.1.3"); // ## Install metrics agent + let section = build_output::section("Metrics agent"); if lockfile_contents.contains("barnes") { - let section = build_output::section("Metrics agent"); context.handle_layer( layer_name!("metrics_agent"), MetricsAgentInstall { section }, )?; - } + } else { + section.say_with_details( + "Skipping install", + "`gem 'barnes'` not found in Gemfile.lock", + ); + }; // ## Install executable ruby version From 81d2fa4a06bb53c3d7a3303d3045f958c60ce4be Mon Sep 17 00:00:00 2001 From: Richard Schneeman Date: Tue, 12 Sep 2023 14:51:41 -0500 Subject: [PATCH 33/40] Address process booting determinism The method we're using to launch a daemon process does not give us guarantees that `agentmon_loop --path` will be executed by the time the process boots. To address this we can loop and re-check for some period of time. This method utilizes the output log from agentmon loop as directed by the spawn daemon command. --- buildpacks/ruby/src/bin/agentmon_loop.rs | 29 +++++++++++++---------- buildpacks/ruby/tests/integration_test.rs | 26 ++++++++++++++++---- 2 files changed, 37 insertions(+), 18 deletions(-) diff --git a/buildpacks/ruby/src/bin/agentmon_loop.rs b/buildpacks/ruby/src/bin/agentmon_loop.rs index 161afd52..b5f31b87 100644 --- a/buildpacks/ruby/src/bin/agentmon_loop.rs +++ b/buildpacks/ruby/src/bin/agentmon_loop.rs @@ -47,19 +47,22 @@ fn main() { }); match agentmon.try_exists() { - Ok(true) => loop { - match run(&agentmon, &agentmon_args) { - Ok(status) => { - eprintln!("Process completed with status={status}, sleeping {SLEEP_FOR:?}"); - } - Err(error) => { - eprintln!( - "Process could not run due to error. {error}, sleeping {SLEEP_FOR:?}" - ); - } - }; - sleep(SLEEP_FOR); - }, + Ok(true) => { + eprintln!("Booting agentmon_loop"); + loop { + match run(&agentmon, &agentmon_args) { + Ok(status) => { + eprintln!("Process completed with status={status}, sleeping {SLEEP_FOR:?}"); + } + Err(error) => { + eprintln!( + "Process could not run due to error. {error}, sleeping {SLEEP_FOR:?}" + ); + } + }; + sleep(SLEEP_FOR); + } + } Ok(false) => { eprintln!("Path does not exist {path}", path = agentmon.display()); exit(1); diff --git a/buildpacks/ruby/tests/integration_test.rs b/buildpacks/ruby/tests/integration_test.rs index cbc7e560..448d255f 100644 --- a/buildpacks/ruby/tests/integration_test.rs +++ b/buildpacks/ruby/tests/integration_test.rs @@ -5,7 +5,7 @@ use libcnb_test::{ ContainerContext, TestRunner, }; use std::thread; -use std::time::Duration; +use std::time::{Duration, Instant}; use thiserror::__private::DisplayAsDisplay; use ureq::Response; @@ -125,17 +125,33 @@ fn test_barnes_app() { ContainerConfig::new() .entrypoint("launcher") .envs(vec![ - ("HEROKU_METRICS_URL", "example.com"), ("DYNO", "web.1"), ("PORT", "1234"), + ("AGENTMON_DEBUG", "1"), + ("HEROKU_METRICS_URL", "example.com"), ]) - .command(["ps x"]), + .command(["while true; do sleep 1; done"]), |container| { - let log_output = container.logs_wait(); + let boot_message = "Booting agentmon_loop"; + let mut agentmon_log = String::new(); + + let started = Instant::now(); + while started.elapsed() < Duration::from_secs(20) { + if agentmon_log.contains(boot_message) { + break; + } + + std::thread::sleep(frac_seconds(0.1)); + agentmon_log = container + .shell_exec("cat /layers/heroku_ruby/metrics_agent/output.log") + .stdout; + } + + let log_output = container.logs_now(); println!("{}", log_output.stdout); println!("{}", log_output.stderr); - assert_contains!(log_output.stdout, "agentmon_loop --path"); + assert_contains!(agentmon_log, boot_message); }, ); }, From 98ab37cbc2bb54a64a40e6410e755f6f6e10e388 Mon Sep 17 00:00:00 2001 From: Richard Schneeman Date: Tue, 12 Sep 2023 15:11:54 -0500 Subject: [PATCH 34/40] Update buildpacks/ruby/src/layers/metrics_agent_install.rs Co-authored-by: Ed Morley <501702+edmorley@users.noreply.github.com> --- buildpacks/ruby/src/layers/metrics_agent_install.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/buildpacks/ruby/src/layers/metrics_agent_install.rs b/buildpacks/ruby/src/layers/metrics_agent_install.rs index 52e9848a..5246c0d1 100644 --- a/buildpacks/ruby/src/layers/metrics_agent_install.rs +++ b/buildpacks/ruby/src/layers/metrics_agent_install.rs @@ -22,10 +22,10 @@ use tempfile::NamedTempFile; /// To get the latest s3 url: /// /// ```shell -/// $ curl https://agentmon-releases.s3.amazonaws.com/latest +/// $ curl https://agentmon-releases.s3.us-east-1.amazonaws.com/latest /// ``` const DOWNLOAD_URL: &str = - "https://agentmon-releases.s3.amazonaws.com/agentmon-0.3.1-linux-amd64.tar.gz"; + "https://agentmon-releases.s3.us-east-1.amazonaws.com/agentmon-0.3.1-linux-amd64.tar.gz"; #[derive(Debug)] pub(crate) struct MetricsAgentInstall { From 66273453b0ad87c16c2f01feae3c381a920f75e4 Mon Sep 17 00:00:00 2001 From: Richard Schneeman Date: Tue, 12 Sep 2023 15:14:23 -0500 Subject: [PATCH 35/40] Remove unneeded BuildpackReference::Crate This is the default. It is only needed when multiple build packs are specified --- buildpacks/ruby/tests/integration_test.rs | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/buildpacks/ruby/tests/integration_test.rs b/buildpacks/ruby/tests/integration_test.rs index 448d255f..d6d9dc85 100644 --- a/buildpacks/ruby/tests/integration_test.rs +++ b/buildpacks/ruby/tests/integration_test.rs @@ -13,8 +13,7 @@ use ureq::Response; #[ignore = "integration test"] fn test_default_app() { TestRunner::default().build( - BuildConfig::new("heroku/builder:22", "tests/fixtures/default_ruby") - .buildpacks(vec![BuildpackReference::Crate]), + BuildConfig::new("heroku/builder:22", "tests/fixtures/default_ruby"), |context| { assert_contains!(context.pack_stdout, "# Heroku Ruby Buildpack"); assert_contains!( @@ -116,8 +115,7 @@ fn test_ruby_app_with_yarn_app() { #[ignore = "integration test"] fn test_barnes_app() { TestRunner::default().build( - BuildConfig::new("heroku/builder:22", "tests/fixtures/barnes_app") - .buildpacks(vec![BuildpackReference::Crate]), + BuildConfig::new("heroku/builder:22", "tests/fixtures/barnes_app"), |context| { assert_contains!(context.pack_stdout, "# Heroku Ruby Buildpack"); From 40a5b814e15031f25bf3795e320bc9c9a210ebce Mon Sep 17 00:00:00 2001 From: Richard Schneeman Date: Tue, 12 Sep 2023 15:16:47 -0500 Subject: [PATCH 36/40] Fix accidentally removed line --- Cargo.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/Cargo.toml b/Cargo.toml index 65fbbbee..f33686ac 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,4 +1,5 @@ [workspace] +resolver = "2" members = ["buildpacks/ruby", "commons"] [workspace.package] From 41c12fe7ff2946655cea688cacce94d516809836 Mon Sep 17 00:00:00 2001 From: Richard Schneeman Date: Tue, 12 Sep 2023 15:18:55 -0500 Subject: [PATCH 37/40] Remove unused workspace declarations --- Cargo.toml | 8 -------- 1 file changed, 8 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index f33686ac..a1f41d09 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,11 +1,3 @@ [workspace] resolver = "2" members = ["buildpacks/ruby", "commons"] - -[workspace.package] -rust-version = "1.64" -edition = "2021" -license = "BSD-3-Clause" - -[workspace.dependencies] -libherokubuildpack = "0.12" From 4ef43eee2d2bbc984aee8a883525d05d0ecff7b4 Mon Sep 17 00:00:00 2001 From: Richard Schneeman Date: Mon, 18 Sep 2023 10:30:16 -0500 Subject: [PATCH 38/40] Add download checksum for agentmon TGZ --- Cargo.lock | 1 + buildpacks/ruby/Cargo.toml | 1 + .../ruby/src/layers/metrics_agent_install.rs | 24 +++++++++++++++++++ 3 files changed, 26 insertions(+) diff --git a/Cargo.lock b/Cargo.lock index ccb656b8..1518ad6a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -493,6 +493,7 @@ dependencies = [ "rand", "regex", "serde", + "sha2", "tar", "tempfile", "thiserror", diff --git a/buildpacks/ruby/Cargo.toml b/buildpacks/ruby/Cargo.toml index 7f7e70c8..c9b9aa5f 100644 --- a/buildpacks/ruby/Cargo.toml +++ b/buildpacks/ruby/Cargo.toml @@ -24,6 +24,7 @@ thiserror = "1" ureq = "2" url = "2" clap = { version = "4", features = ["derive"] } +sha2 = "0.10.7" [dev-dependencies] libcnb-test = "=0.14.0" diff --git a/buildpacks/ruby/src/layers/metrics_agent_install.rs b/buildpacks/ruby/src/layers/metrics_agent_install.rs index 5246c0d1..81528cf4 100644 --- a/buildpacks/ruby/src/layers/metrics_agent_install.rs +++ b/buildpacks/ruby/src/layers/metrics_agent_install.rs @@ -9,6 +9,7 @@ use libcnb::{ layer::{Layer, LayerResultBuilder}, }; use serde::{Deserialize, Serialize}; +use sha2::Digest; use std::os::unix::fs::PermissionsExt; use std::path::{Path, PathBuf}; use tar::Archive; @@ -26,6 +27,7 @@ use tempfile::NamedTempFile; /// ``` const DOWNLOAD_URL: &str = "https://agentmon-releases.s3.us-east-1.amazonaws.com/agentmon-0.3.1-linux-amd64.tar.gz"; +const DOWNLOAD_SHA: &str = "f9bf9f33c949e15ffed77046ca38f8dae9307b6a0181c6af29a25dec46eb2dac"; #[derive(Debug)] pub(crate) struct MetricsAgentInstall { @@ -57,6 +59,9 @@ pub(crate) enum MetricsAgentInstallError { #[error("Could not write file: {0}")] CouldNotWriteDestinationFile(std::io::Error), + + #[error("Checksum of download failed. Expected {DOWNLOAD_SHA} got {0}")] + ChecksumFailed(String), } impl Layer for MetricsAgentInstall { @@ -153,6 +158,15 @@ impl Layer for MetricsAgentInstall { } } +// Check integrity of download, `sha_some` is awesome. +fn sha_some(path: &Path) -> Result { + let mut hasher = sha2::Sha256::new(); + let contents = fs_err::read(path)?; + hasher.update(&contents); + + Ok(format!("{:x}", hasher.finalize())) +} + fn write_execd_script( agentmon: &Path, layer_path: &Path, @@ -212,6 +226,16 @@ fn download_untar( download(url, agentmon_tgz.path())?; + sha_some(agentmon_tgz.path()) + .map_err(MetricsAgentInstallError::CouldNotOpenFile) + .and_then(|checksum| { + if DOWNLOAD_SHA == checksum { + Ok(()) + } else { + Err(MetricsAgentInstallError::ChecksumFailed(checksum)) + } + })?; + untar(agentmon_tgz.path(), destination)?; Ok(()) From 43cbaa61536c5a24c13e3a6ca600f8fcb592a13e Mon Sep 17 00:00:00 2001 From: Richard Schneeman Date: Mon, 18 Sep 2023 10:31:56 -0500 Subject: [PATCH 39/40] Update changelog to use Keep a Changelog format --- buildpacks/ruby/CHANGELOG.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/buildpacks/ruby/CHANGELOG.md b/buildpacks/ruby/CHANGELOG.md index 26d8182b..8dcadb46 100644 --- a/buildpacks/ruby/CHANGELOG.md +++ b/buildpacks/ruby/CHANGELOG.md @@ -2,11 +2,13 @@ All notable changes to this project will be documented in this file. -The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), +The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [Unreleased] +### Added + - Introduce heroku build metrics support (https://github.com/heroku/buildpacks-ruby/pull/172) - Changelog moved to be per-crate rather than for the whole project (https://github.com/heroku/buildpacks-ruby/pull/154) From f4abf4d77da70a11e40c5570efddbefdd01ed062 Mon Sep 17 00:00:00 2001 From: Richard Schneeman Date: Mon, 18 Sep 2023 12:35:37 -0500 Subject: [PATCH 40/40] Prefer shared tooling --- Cargo.lock | 2 +- buildpacks/ruby/Cargo.toml | 2 +- buildpacks/ruby/src/layers/metrics_agent_install.rs | 13 ++----------- 3 files changed, 4 insertions(+), 13 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 1518ad6a..c7b25bab 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -490,10 +490,10 @@ dependencies = [ "indoc", "libcnb", "libcnb-test", + "libherokubuildpack", "rand", "regex", "serde", - "sha2", "tar", "tempfile", "thiserror", diff --git a/buildpacks/ruby/Cargo.toml b/buildpacks/ruby/Cargo.toml index c9b9aa5f..32c24100 100644 --- a/buildpacks/ruby/Cargo.toml +++ b/buildpacks/ruby/Cargo.toml @@ -15,6 +15,7 @@ indoc = "2" # libcnb has a much bigger impact on buildpack behaviour than any other dependencies, # so it's pinned to an exact version to isolate it from lockfile refreshes. libcnb = "=0.14.0" +libherokubuildpack = "=0.14.0" rand = "0.8" regex = "1" serde = "1" @@ -24,7 +25,6 @@ thiserror = "1" ureq = "2" url = "2" clap = { version = "4", features = ["derive"] } -sha2 = "0.10.7" [dev-dependencies] libcnb-test = "=0.14.0" diff --git a/buildpacks/ruby/src/layers/metrics_agent_install.rs b/buildpacks/ruby/src/layers/metrics_agent_install.rs index 81528cf4..af0361c7 100644 --- a/buildpacks/ruby/src/layers/metrics_agent_install.rs +++ b/buildpacks/ruby/src/layers/metrics_agent_install.rs @@ -8,8 +8,8 @@ use libcnb::{ generic::GenericMetadata, layer::{Layer, LayerResultBuilder}, }; +use libherokubuildpack::digest::sha256; use serde::{Deserialize, Serialize}; -use sha2::Digest; use std::os::unix::fs::PermissionsExt; use std::path::{Path, PathBuf}; use tar::Archive; @@ -158,15 +158,6 @@ impl Layer for MetricsAgentInstall { } } -// Check integrity of download, `sha_some` is awesome. -fn sha_some(path: &Path) -> Result { - let mut hasher = sha2::Sha256::new(); - let contents = fs_err::read(path)?; - hasher.update(&contents); - - Ok(format!("{:x}", hasher.finalize())) -} - fn write_execd_script( agentmon: &Path, layer_path: &Path, @@ -226,7 +217,7 @@ fn download_untar( download(url, agentmon_tgz.path())?; - sha_some(agentmon_tgz.path()) + sha256(agentmon_tgz.path()) .map_err(MetricsAgentInstallError::CouldNotOpenFile) .and_then(|checksum| { if DOWNLOAD_SHA == checksum {