From d72c474b20f216cee989b9b4c47cc7ab555f2e96 Mon Sep 17 00:00:00 2001 From: Colin Walters Date: Tue, 28 Sep 2021 12:44:59 -0400 Subject: [PATCH] Add new refescape module Prep for work on the new container module, where we want to store container image references (e.g. `docker://quay.io/coreos/fedora`) as ostree refs. Several bits of that are not valid in ostree refs, such as the `:` or the double `//` (which would be an empty filesystem path). This escaping scheme uses `_` in a similar way as a `\` character is used in other syntax. For example, `:` is `_3A_` (hexadecimal). `//` is escaped as `/_2F_` (i.e. the second `/` is escaped). --- lib/Cargo.toml | 1 + lib/src/lib.rs | 1 + lib/src/refescape.rs | 193 +++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 195 insertions(+) create mode 100644 lib/src/refescape.rs diff --git a/lib/Cargo.toml b/lib/Cargo.toml index 5ec67dd9..bdefb234 100644 --- a/lib/Cargo.toml +++ b/lib/Cargo.toml @@ -45,6 +45,7 @@ tracing = "0.1" [dev-dependencies] clap = "2.33.3" indoc = "1.0.3" +quickcheck = "1" sh-inline = "0.1.0" structopt = "0.3.21" diff --git a/lib/src/lib.rs b/lib/src/lib.rs index 9a25b1df..3f7ce54c 100644 --- a/lib/src/lib.rs +++ b/lib/src/lib.rs @@ -26,6 +26,7 @@ pub mod cli; pub mod container; pub mod diff; pub mod ima; +pub mod refescape; pub mod tar; pub mod tokio_util; diff --git a/lib/src/refescape.rs b/lib/src/refescape.rs new file mode 100644 index 00000000..e15cc8b3 --- /dev/null +++ b/lib/src/refescape.rs @@ -0,0 +1,193 @@ +//! Escape strings for use in ostree refs. +//! +//! It can be desirable to map arbitrary identifiers, such as RPM/dpkg +//! package names or container image references (e.g. `docker://quay.io/examplecorp/os:latest`) +//! into ostree refs (branch names) which have a quite restricted set +//! of valid characters; basically alphanumeric, plus `/`, `-`, `_`. +//! +//! This escaping scheme uses `_` in a similar way as a `\` character is +//! used in Rust unicode escaped values. For example, `:` is `_3A_` (hexadecimal). +//! Because the empty path is not valid, `//` is escaped as `/_2F_` (i.e. the second `/` is escaped). + +use anyhow::Result; +use std::convert::TryInto; +use std::fmt::Write; + +/// Escape a string in a suitable fashion for an ostree ref. +/// +/// There are just a few restrictions: +/// - The empty string is not supported +/// - There may not be embedded `NUL` (`\\0`) characters. +fn escape_for_ref(s: &str) -> Result { + if s.is_empty() { + return Err(anyhow::anyhow!("Invalid empty string for ref")); + } + fn escape_c(r: &mut String, c: char) { + write!(r, "_{:02X}_", c as u32).unwrap() + } + let mut r = String::new(); + let mut it = s.chars().peekable(); + + // We require a leading alphanumeric + match it.peek() { + Some(&c) if !c.is_ascii_alphanumeric() => { + escape_c(&mut r, c); + let _ = it.next(); + } + _ => {} + } + + while let Some(c) = it.next() { + match (c, it.peek().copied()) { + ('\0', _) => { + return Err(anyhow::anyhow!( + "Invalid embedded NUL in string for ostree ref" + )); + } + (c, _) if c.is_ascii_alphanumeric() => { + r.push(c); + } + // The `/` must be followed by alphanumeric + ('/', Some(next)) if !next.is_ascii_alphanumeric() => { + let _ = it.next(); + r.push('/'); + escape_c(&mut r, next); + } + // Also can't end on a `/` per above. + ('/', None) => { + escape_c(&mut r, '/'); + } + // Pass through - and `/` as valid + ('-' | '/', _) => r.push(c), + // `_` becomes `__`. + ('_', _) => r.push_str("__"), + // Otherwise, escape. + (o, _) => escape_c(&mut r, o), + } + } + Ok(r) +} + +/// Compute a string suitable for use as an OSTree ref, where `s` can be a (nearly) +/// arbitrary UTF-8 string. This requires a non-empty prefix. +/// +/// The intention behind requiring a prefix is that a common need is to use e.g. +/// [`ostree::Repo::list_refs`] to find refs of a certain "type". +/// +/// ```rust +/// # fn test() -> anyhow::Result<()> { +/// use ostree_ext::refescape; +/// assert_eq!(refescape::prefix_escape_for_ref("container", "registry:quay.io/coreos/fedora:latest")?, "container/registry_3A_quay_2E_io/coreos/fedora_3A_latest"); +/// # Ok(()) +/// } +/// ``` +pub fn prefix_escape_for_ref(prefix: &str, s: &str) -> Result { + Ok(format!("{}/{}", prefix, escape_for_ref(s)?)) +} + +/// Reverse the effect of [`escape_for_ref()`]. +fn unescape_for_ref(s: &str) -> Result { + let mut r = String::new(); + let mut it = s.chars(); + let mut buf = String::new(); + while let Some(c) = it.next() { + match c { + c if c.is_ascii_alphanumeric() => { + r.push(c); + } + '-' | '/' => r.push(c), + '_' => { + let next = it.next(); + if let Some('_') = next { + r.push('_') + } else if let Some(c) = next { + buf.clear(); + buf.push(c); + while let Some(c) = it.next() { + if c == '_' { + break; + } + buf.push(c); + } + let v = u32::from_str_radix(&buf, 16)?; + let c: char = v.try_into()?; + r.push(c); + } + } + o => anyhow::bail!("Invalid character {}", o), + } + } + Ok(r) +} + +/// Remove a prefix from an ostree ref, and return the unescaped remainder. +/// +/// ```rust +/// # fn test() -> anyhow::Result<()> { +/// use ostree_ext::refescape; +/// assert_eq!(refescape::unprefix_unescape_ref("container", "container/registry_3A_quay_2E_io/coreos/fedora_3A_latest")?, "registry:quay.io/coreos/fedora:latest"); +/// # Ok(()) +/// } +/// ``` +pub fn unprefix_unescape_ref(prefix: &str, ostree_ref: &str) -> Result { + let rest = ostree_ref.strip_prefix(prefix).ok_or_else(|| { + anyhow::anyhow!( + "ref does not match expected prefix {}: {}", + ostree_ref, + prefix + ) + })?; + Ok(unescape_for_ref(rest)?) +} + +#[cfg(test)] +mod test { + use super::*; + use quickcheck::{quickcheck, TestResult}; + + const UNCHANGED: &[&str] = &["foo", "foo/bar/baz-blah/foo"]; + const ROUNDTRIP: &[&str] = &[ + "localhost:5000/foo:latest", + "fedora/x86_64/coreos", + "/foo/bar/foo.oci-archive", + "docker://quay.io/exampleos/blah:latest", + "oci-archive:/path/to/foo.ociarchive", + ]; + + #[test] + fn escape() { + // These strings shouldn't change + for &v in UNCHANGED { + let escaped = &escape_for_ref(v).unwrap(); + ostree::validate_rev(escaped).unwrap(); + assert_eq!(escaped.as_str(), v); + } + // Roundtrip cases, plus unchanged cases + for &v in UNCHANGED.iter().chain(ROUNDTRIP) { + let escaped = &escape_for_ref(v).unwrap(); + ostree::validate_rev(escaped).unwrap(); + let unescaped = unescape_for_ref(&escaped).unwrap(); + assert_eq!(v, unescaped); + } + // Explicit test + assert_eq!( + escape_for_ref(ROUNDTRIP[0]).unwrap(), + "localhost_3A_5000/foo_3A_latest" + ); + } + + fn roundtrip(s: String) -> TestResult { + // Ensure we only try strings which match the predicates. + let escaped = match escape_for_ref(&s) { + Ok(v) => v, + Err(_) => return TestResult::discard(), + }; + let unescaped = unescape_for_ref(&escaped).unwrap(); + TestResult::from_bool(unescaped == s) + } + + #[test] + fn qcheck() { + quickcheck(roundtrip as fn(String) -> TestResult); + } +}