Skip to content

Commit

Permalink
Embed the PuzzleFS metadata files into the PuzzleFS manifest
Browse files Browse the repository at this point in the history 
This simplifies the PuzzleFS layout by storing all the metadata
information into a single metadata file. The previous layout had one
manifest file which contained references to a list of metadata files,
each stored separately.

Relevant discussions: project-machine#55

Signed-off-by: Ariel Miculas-Trif <[email protected]>
  • Loading branch information
@ariel-miculas
ariel-miculas committed Sep 6, 2024
1 parent 0413317 commit 970362a
Show file tree
Hide file tree
Showing 6 changed files with 146 additions and 177 deletions.
85 changes: 19 additions & 66 deletions puzzlefs-lib/src/builder.rs
View file
Original file line number Diff line number Diff line change
@@ -1,8 +1,7 @@
use crate::common::{AVG_CHUNK_SIZE, MAX_CHUNK_SIZE, MIN_CHUNK_SIZE};
use crate::compression::{Compression, Noop, Zstd};
use crate::fsverity_helpers::{
check_fs_verity, fsverity_enable, get_fs_verity_digest, InnerHashAlgorithm,
FS_VERITY_BLOCK_SIZE_DEFAULT,
check_fs_verity, fsverity_enable, InnerHashAlgorithm, FS_VERITY_BLOCK_SIZE_DEFAULT,
};
use crate::oci::Digest;
use std::any::Any;
Expand Down Expand Up @@ -77,7 +76,7 @@ struct Other {
additional: Option<InodeAdditional>,
}

fn serialize_manifest(rootfs: Rootfs) -> Result<Vec<u8>> {
fn serialize_metadata(rootfs: Rootfs) -> Result<Vec<u8>> {
let mut message = ::capnp::message::Builder::new_default();
let mut capnp_rootfs = message.init_root::<metadata_capnp::rootfs::Builder<'_>>();

Expand All @@ -88,24 +87,6 @@ fn serialize_manifest(rootfs: Rootfs) -> Result<Vec<u8>> {
Ok(buf)
}

fn serialize_metadata(inodes: Vec<Inode>) -> Result<Vec<u8>> {
let mut message = ::capnp::message::Builder::new_default();
let capnp_inode_vector = message.init_root::<metadata_capnp::inode_vector::Builder<'_>>();
let inodes_len = inodes.len().try_into()?;

let mut capnp_inodes = capnp_inode_vector.init_inodes(inodes_len);

for (i, inode) in inodes.iter().enumerate() {
// we already checked that the length of pfs_inodes fits inside a u32
let mut capnp_inode = capnp_inodes.reborrow().get(i as u32);
inode.fill_capnp(&mut capnp_inode)?;
}

let mut buf = Vec::new();
::capnp::serialize::write_message(&mut buf, &message)?;
Ok(buf)
}

fn process_chunks<C: Compression + Any>(
oci: &Image,
mut chunker: StreamCDC,
Expand Down Expand Up @@ -183,7 +164,7 @@ fn build_delta<C: Compression + Any>(
oci: &Image,
mut existing: Option<PuzzleFS>,
verity_data: &mut VerityData,
) -> Result<Descriptor> {
) -> Result<Vec<Inode>> {
let mut dirs = HashMap::<u64, Dir>::new();
let mut files = Vec::<File>::new();
let mut others = Vec::<Other>::new();
Expand Down Expand Up @@ -405,30 +386,18 @@ fn build_delta<C: Compression + Any>(

pfs_inodes.sort_by(|a, b| a.ino.cmp(&b.ino));

let md_buf = serialize_metadata(pfs_inodes)?;

let (desc, ..) = oci.put_blob::<Noop, media_types::Inodes>(md_buf.as_slice())?;
let verity_hash = get_fs_verity_digest(md_buf.as_slice())?;
verity_data.insert(desc.digest.underlying(), verity_hash);

Ok(desc)
Ok(pfs_inodes)
}

pub fn build_initial_rootfs<C: Compression + Any>(
rootfs: &Path,
oci: &Image,
) -> Result<Descriptor> {
let mut verity_data: VerityData = BTreeMap::new();
let desc = build_delta::<C>(rootfs, oci, None, &mut verity_data)?;
let metadatas = [BlobRef {
offset: 0,
digest: desc.digest.underlying(),
compressed: false,
}]
.to_vec();

let rootfs_buf = serialize_manifest(Rootfs {
metadatas,
let inodes = build_delta::<C>(rootfs, oci, None, &mut verity_data)?;

let rootfs_buf = serialize_metadata(Rootfs {
metadatas: vec![inodes],
fs_verity_data: verity_data,
manifest_version: PUZZLEFS_IMAGE_MANIFEST_VERSION,
})?;
Expand All @@ -448,21 +417,16 @@ pub fn add_rootfs_delta<C: Compression + Any>(
let mut verity_data: VerityData = BTreeMap::new();
let pfs = PuzzleFS::open(oci, tag, None)?;
let oci = Arc::clone(&pfs.oci);
let mut rootfs = oci.open_rootfs_blob::<Noop>(tag, None)?;
let mut rootfs = Rootfs::try_from(oci.open_rootfs_blob(tag, None)?)?;

let desc = build_delta::<C>(rootfs_path, &oci, Some(pfs), &mut verity_data)?;
let br = BlobRef {
digest: desc.digest.underlying(),
offset: 0,
compressed: false,
};
let inodes = build_delta::<C>(rootfs_path, &oci, Some(pfs), &mut verity_data)?;

if !rootfs.metadatas.iter().any(|&x| x == br) {
rootfs.metadatas.insert(0, br);
if !rootfs.metadatas.iter().any(|x| *x == inodes) {
rootfs.metadatas.insert(0, inodes);
}

rootfs.fs_verity_data.extend(verity_data);
let rootfs_buf = serialize_manifest(rootfs)?;
let rootfs_buf = serialize_metadata(rootfs)?;
Ok((
oci.put_blob::<Noop, media_types::Rootfs>(rootfs_buf.as_slice())?
.0,
Expand All @@ -488,9 +452,9 @@ pub fn enable_fs_verity(oci: Image, tag: &str, manifest_root_hash: &str) -> Resu

let pfs = PuzzleFS::open(oci, tag, None)?;
let oci = Arc::clone(&pfs.oci);
let rootfs = oci.open_rootfs_blob::<Noop>(tag, None)?;
let rootfs = oci.open_rootfs_blob(tag, None)?;

for (content_addressed_file, verity_hash) in rootfs.fs_verity_data {
for (content_addressed_file, verity_hash) in rootfs.get_verity_data()? {
let file_path = oci
.blob_path()
.join(Digest::new(&content_addressed_file).to_string());
Expand Down Expand Up @@ -521,8 +485,6 @@ pub fn build_test_fs(path: &Path, image: &Image) -> Result<Descriptor> {
pub mod tests {
use super::*;

use std::backtrace::Backtrace;

use tempfile::tempdir;

use crate::reader::WalkPuzzleFS;
Expand All @@ -541,12 +503,8 @@ pub mod tests {
let dir = tempdir().unwrap();
let image = Image::new(dir.path()).unwrap();
let rootfs_desc = build_test_fs(Path::new("src/builder/test/test-1"), &image).unwrap();
let rootfs = Rootfs::open(
image
.open_compressed_blob::<Noop>(&rootfs_desc.digest, None)
.unwrap(),
)
.unwrap();
image.add_tag("test-tag", rootfs_desc)?;
let rootfs = image.open_rootfs_blob("test-tag", None).unwrap();

// there should be a blob that matches the hash of the test data, since it all gets input
// as one chunk and there's only one file
Expand All @@ -563,16 +521,11 @@ pub mod tests {
)
.unwrap();

let metadata_digest = rootfs.metadatas[0].try_into().unwrap();
let blob = image.open_metadata_blob(&metadata_digest, None).unwrap();
let mut inodes = Vec::new();

// we can at least deserialize inodes and they look sane
for i in 0..2 {
inodes.push(Inode::from_capnp(
blob.find_inode((i + 1).try_into()?)?
.ok_or(WireFormatError::InvalidSerializedData(Backtrace::capture()))?,
)?);
inodes.push(rootfs.find_inode(i + 1)?);
}

assert_eq!(inodes[0].ino, 1);
Expand Down Expand Up @@ -620,7 +573,7 @@ pub mod tests {
let (desc, image) = add_rootfs_delta::<DefaultCompression>(&delta_dir, image, tag).unwrap();
let new_tag = "test2";
image.add_tag(new_tag, desc).unwrap();
let delta = image.open_rootfs_blob::<Noop>(new_tag, None).unwrap();
let delta = Rootfs::try_from(image.open_rootfs_blob(new_tag, None).unwrap()).unwrap();
assert_eq!(delta.metadatas.len(), 2);

let image = Image::new(dir.path()).unwrap();
Expand Down
2 changes: 1 addition & 1 deletion puzzlefs-lib/src/format/metadata.capnp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -70,7 +70,7 @@ struct VerityData {
}

struct Rootfs {
metadatas@0: List(BlobRef);
metadatas@0: List(InodeVector);
fsVerityData@1: List(VerityData);
manifestVersion@2: UInt64;
}
141 changes: 108 additions & 33 deletions puzzlefs-lib/src/format/types.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,6 @@ use std::ffi::OsString;
use std::fmt;
use std::fs;
use std::io;
use std::io::Read;
use std::os::unix::ffi::OsStrExt;
use std::os::unix::ffi::OsStringExt;
use std::os::unix::fs::{FileTypeExt, MetadataExt, PermissionsExt};
Expand All @@ -30,25 +29,25 @@ pub type VerityData = BTreeMap<[u8; SHA256_BLOCK_SIZE], [u8; SHA256_BLOCK_SIZE]>

#[derive(Debug)]
pub struct Rootfs {
pub metadatas: Vec<BlobRef>,
pub metadatas: Vec<Vec<Inode>>,
pub fs_verity_data: VerityData,
pub manifest_version: u64,
}

impl Rootfs {
pub fn open<R: Read>(f: R) -> Result<Rootfs> {
let message_reader = serialize::read_message(f, ::capnp::message::ReaderOptions::new())?;
let rootfs = message_reader.get_root::<crate::metadata_capnp::rootfs::Reader<'_>>()?;
Self::from_capnp(rootfs)
impl TryFrom<RootfsReader> for Rootfs {
type Error = WireFormatError;
fn try_from(rootfs_reader: RootfsReader) -> Result<Self> {
Rootfs::from_capnp(rootfs_reader.reader.get()?)
}
}

impl Rootfs {
pub fn from_capnp(reader: crate::metadata_capnp::rootfs::Reader<'_>) -> Result<Self> {
let metadatas = reader.get_metadatas()?;

let metadata_vec = metadatas
let metadata_vec = reader
.get_metadatas()?
.iter()
.map(BlobRef::from_capnp)
.collect::<Result<Vec<BlobRef>>>()?;
.map(InodeVector::from_capnp)
.collect::<Result<Vec<Vec<_>>>>()?;

let capnp_verities = reader.get_fs_verity_data()?;
let mut fs_verity_data = VerityData::new();
Expand Down Expand Up @@ -78,7 +77,7 @@ impl Rootfs {
for (i, metadata) in self.metadatas.iter().enumerate() {
// we already checked that the length of metadatas fits inside a u32
let mut capnp_metadata = capnp_metadatas.reborrow().get(i as u32);
metadata.fill_capnp(&mut capnp_metadata);
InodeVector::fill_capnp(metadata, &mut capnp_metadata)?;
}

let verity_data_len = self.fs_verity_data.len().try_into()?;
Expand All @@ -95,6 +94,72 @@ impl Rootfs {
}
}

pub struct RootfsReader {
reader: message::TypedReader<
::capnp::serialize::BufferSegments<Mmap>,
crate::metadata_capnp::rootfs::Owned,
>,
}

impl RootfsReader {
pub fn open(f: fs::File) -> Result<Self> {
// We know the loaded message is safe, so we're allowing unlimited reads.
let unlimited_reads = message::ReaderOptions {
traversal_limit_in_words: None,
nesting_limit: 64,
};
let mmapped_region = unsafe { MmapOptions::new().map_copy_read_only(&f)? };
let segments = serialize::BufferSegments::new(mmapped_region, unlimited_reads)?;
let reader = message::Reader::new(segments, unlimited_reads).into_typed();

Ok(Self { reader })
}

pub fn get_manifest_version(&self) -> Result<u64> {
Ok(self.reader.get()?.get_manifest_version())
}

pub fn get_verity_data(&self) -> Result<VerityData> {
let mut fs_verity_data = VerityData::new();

let capnp_verities = self.reader.get()?.get_fs_verity_data()?;
for capnp_verity in capnp_verities {
let digest = capnp_verity.get_digest()?.try_into()?;
let verity = capnp_verity.get_verity()?.try_into()?;
fs_verity_data.insert(digest, verity);
}
Ok(fs_verity_data)
}

pub fn find_inode(&self, ino: u64) -> Result<Inode> {
for layer in self.reader.get()?.get_metadatas()?.iter() {
let inode_vector = InodeVector { reader: layer };

if let Some(inode) = inode_vector.find_inode(ino)? {
let inode = Inode::from_capnp(inode)?;
if let InodeMode::Wht = inode.mode {
// TODO: seems like this should really be an Option.
return Err(WireFormatError::from_errno(Errno::ENOENT));
}
return Ok(inode);
}
}

Err(WireFormatError::from_errno(Errno::ENOENT))
}

pub fn max_inode(&self) -> Result<Ino> {
let mut max: Ino = 1;
for layer in self.reader.get()?.get_metadatas()?.iter() {
let inode_vector = InodeVector { reader: layer };
if let Some(ino) = inode_vector.max_ino()? {
max = std::cmp::max(ino, max)
}
}
Ok(max)
}
}

// TODO: should this be an ociv1 digest and include size and media type?
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct BlobRef {
Expand Down Expand Up @@ -699,32 +764,16 @@ impl Xattr {
}
}

pub struct MetadataBlob {
reader: message::TypedReader<
::capnp::serialize::BufferSegments<Mmap>,
crate::metadata_capnp::inode_vector::Owned,
>,
pub struct InodeVector<'a> {
reader: crate::metadata_capnp::inode_vector::Reader<'a>,
}

impl MetadataBlob {
pub fn new(f: fs::File) -> Result<MetadataBlob> {
// We know the loaded message is safe, so we're allowing unlimited reads.
let unlimited_reads = message::ReaderOptions {
traversal_limit_in_words: None,
nesting_limit: 64,
};
let mmapped_region = unsafe { MmapOptions::new().map_copy_read_only(&f)? };
let segments = serialize::BufferSegments::new(mmapped_region, unlimited_reads)?;
let reader = message::Reader::new(segments, unlimited_reads).into_typed();

Ok(MetadataBlob { reader })
}

impl<'a> InodeVector<'a> {
pub fn get_inode_vector(
&self,
) -> ::capnp::Result<::capnp::struct_list::Reader<'_, crate::metadata_capnp::inode::Owned>>
{
self.reader.get()?.get_inodes()
self.reader.get_inodes()
}

pub fn find_inode(&self, ino: Ino) -> Result<Option<crate::metadata_capnp::inode::Reader<'_>>> {
Expand Down Expand Up @@ -759,6 +808,32 @@ impl MetadataBlob {
let last_index = inodes.len() - 1;
Ok(Some(inodes.get(last_index).get_ino()))
}

pub fn from_capnp(
reader: crate::metadata_capnp::inode_vector::Reader<'a>,
) -> Result<Vec<Inode>> {
reader
.get_inodes()?
.iter()
.map(|inode| Inode::from_capnp(inode))
.collect()
}

fn fill_capnp(
inodes: &[Inode],
builder: &mut crate::metadata_capnp::inode_vector::Builder<'_>,
) -> Result<()> {
let inodes_len = inodes.len().try_into()?;
let mut capnp_inodes = builder.reborrow().init_inodes(inodes_len);

for (i, inode) in inodes.iter().enumerate() {
// we already checked that the length of pfs_inodes fits inside a u32
let mut capnp_inode = capnp_inodes.reborrow().get(i as u32);
inode.fill_capnp(&mut capnp_inode)?;
}

Ok(())
}
}

#[derive(Debug, Clone, PartialEq, Eq)]
Expand Down
Loading

0 comments on commit 970362a

Please sign in to comment.