From 5bd3599e3578848d93bafce1fa638713b9b2b6ce Mon Sep 17 00:00:00 2001 From: Griffin Berlstein Date: Mon, 11 Nov 2024 19:56:23 +0000 Subject: [PATCH 1/4] [Data Converter] add experimental dat-style serialization (#2347) I did some silly hacking late last week that might get us marginally closer to unifying all the data marshaling under a single tool. This adds the ability to specify a `--to dat` (or one of the aliases) and an output directory which will generate the hex encoded files that verilator/icarus expect. There are some minor differences with the dat files generated by the python flow. Mainly, the python flow truncates leading zeroes in the encoding while I've elected to retain them in the interest of keeping things simple. Python generates: ``` 4B 53 21 5D 1E 5E 2B B 3C 60 ``` while the data-converter generates ``` 0000004B 00000053 00000021 0000005D 0000001E 0000005E 0000002B 0000000B 0000003C 00000060 ``` This also adds the ability to deserialize this style of data dump but is slightly brittle at the moment since it expect the following: - the data header is exactly that used by the tool and is cbor encoded in a file named `header`. The python currently json encodes things in a file named `shape` - the input data includes all leading zeroes Both these assumptions can probably be relaxed in the future in the interest of robustness --- interp/src/serialization/data_dump.rs | 30 ++-- tools/cider-data-converter/src/main.rs | 198 +++++++++++++++++++++---- 2 files changed, 192 insertions(+), 36 deletions(-) diff --git a/interp/src/serialization/data_dump.rs b/interp/src/serialization/data_dump.rs index 0f386d75f..d676814c5 100644 --- a/interp/src/serialization/data_dump.rs +++ b/interp/src/serialization/data_dump.rs @@ -148,6 +148,10 @@ impl MemoryDeclaration { self.format.width() } + pub fn bytes_per_entry(&self) -> u32 { + self.format.width().div_ceil(8) + } + pub fn signed(&self) -> bool { self.format.signed() } @@ -172,6 +176,17 @@ impl DataHeader { .iter() .fold(0, |acc, mem| acc + mem.byte_count()) } + + pub fn serialize(&self) -> Result, SerializationError> { + let mut header_str = Vec::new(); + ciborium::ser::into_writer(&self, &mut header_str)?; + Ok(header_str) + } + + pub fn deserialize(data: &[u8]) -> Result { + let header: Self = ciborium::from_reader(data)?; + Ok(header) + } } #[derive(Debug, PartialEq)] @@ -237,13 +252,11 @@ impl DataDump { self.push_memory(declaration, data) } - // TODO Griffin: handle the errors properly - pub fn serialize( + pub fn serialize( &self, - writer: &mut dyn std::io::Write, + mut writer: W, ) -> Result<(), SerializationError> { - let mut header_str = Vec::new(); - ciborium::ser::into_writer(&self.header, &mut header_str)?; + let header_str = self.header.serialize()?; writer.write_all(&Self::MAGIC_NUMBER)?; let len_bytes: u32 = header_str @@ -257,9 +270,8 @@ impl DataDump { Ok(()) } - // TODO Griffin: handle the errors properly - pub fn deserialize( - reader: &mut dyn std::io::Read, + pub fn deserialize( + mut reader: R, ) -> Result { let mut magic_number = [0u8; 4]; reader.read_exact(&mut magic_number).map_err(|e| { @@ -291,7 +303,7 @@ impl DataDump { SerializationError::IoError(e) } })?; - let header: DataHeader = ciborium::from_reader(raw_header.as_slice())?; + let header = DataHeader::deserialize(&raw_header)?; let mut data: Vec = Vec::with_capacity(header.data_size()); diff --git a/tools/cider-data-converter/src/main.rs b/tools/cider-data-converter/src/main.rs index e2c5c7099..e44fea9cb 100644 --- a/tools/cider-data-converter/src/main.rs +++ b/tools/cider-data-converter/src/main.rs @@ -1,9 +1,11 @@ use argh::FromArgs; use cider_data_converter::{converter, json_data::JsonData}; -use interp::serialization::{self, SerializationError}; +use core::str; +use interp::serialization::{self, DataDump, SerializationError}; +use itertools::Itertools; use std::{ fs::File, - io::{self, Read, Write}, + io::{self, BufRead, BufReader, BufWriter, Read, Write}, path::PathBuf, str::FromStr, }; @@ -12,6 +14,8 @@ use thiserror::Error; const JSON_EXTENSION: &str = "data"; const CIDER_EXTENSION: &str = "dump"; +const HEADER_FILENAME: &str = "header"; + #[derive(Error)] enum CiderDataConverterError { #[error("Failed to read file: {0}")] @@ -36,18 +40,27 @@ impl std::fmt::Debug for CiderDataConverterError { } } -enum Action { - ToDataDump, - ToJson, +/// What are we converting the input to +#[derive(Debug, Clone, Copy)] +enum Target { + /// Cider's Single-file DataDump format + DataDump, + /// Verilator/icarus directory format + Dat, + /// Human readable output JSON + Json, } -impl FromStr for Action { +impl FromStr for Target { type Err = CiderDataConverterError; fn from_str(s: &str) -> Result { match s.to_lowercase().as_str() { - "json" => Ok(Action::ToJson), - "cider" | "dump" | "data-dump" => Ok(Action::ToDataDump), + "json" => Ok(Target::Json), + "cider" | "dump" | "data-dump" => Ok(Target::DataDump), + "dat" | "verilog-dat" | "verilog" | "verilator" | "icarus" => { + Ok(Target::Dat) + } _ => Err(CiderDataConverterError::BadToArgument(s.to_string())), } } @@ -71,7 +84,7 @@ struct Opts { /// optional specification of what action to perform. Can be "cider" or /// "json". If not provided, the converter will try to guess based on file names #[argh(option, short = 't', long = "to")] - action: Option, + action: Option, /// whether to use quotes around floating point numbers in the output. This /// exists solely for backwards compatibility with the old display format. @@ -82,18 +95,6 @@ struct Opts { fn main() -> Result<(), CiderDataConverterError> { let mut opts: Opts = argh::from_env(); - let mut input: Box = opts - .input_path - .as_ref() - .map(|path| File::open(path).map(|x| Box::new(x) as Box)) - .unwrap_or(Ok(Box::new(io::stdin())))?; - - let mut output: Box = opts - .output_path - .as_ref() - .map(|path| File::create(path).map(|x| Box::new(x) as Box)) - .unwrap_or(Ok(Box::new(io::stdout())))?; - // if no action is specified, try to guess based on file extensions if opts.action.is_none() && (opts.input_path.as_ref().is_some_and(|x| { @@ -102,7 +103,7 @@ fn main() -> Result<(), CiderDataConverterError> { x.extension().map_or(false, |y| y == CIDER_EXTENSION) })) { - opts.action = Some(Action::ToDataDump); + opts.action = Some(Target::DataDump); } else if opts.action.is_none() && (opts.output_path.as_ref().is_some_and(|x| { x.extension().map_or(false, |x| x == JSON_EXTENSION) @@ -110,20 +111,87 @@ fn main() -> Result<(), CiderDataConverterError> { x.extension().map_or(false, |x| x == CIDER_EXTENSION) })) { - opts.action = Some(Action::ToJson); + opts.action = Some(Target::Json); } if let Some(action) = opts.action { match action { - Action::ToDataDump => { + Target::DataDump => { + let (mut input, mut output) = get_io_handles(&opts)?; + let parsed_json: JsonData = serde_json::from_reader(&mut input)?; converter::convert_to_data_dump(&parsed_json, opts.round_float) .serialize(&mut output)?; } - Action::ToJson => { - let data_dump = - serialization::DataDump::deserialize(&mut input)?; + Target::Json => { + let data_dump = if let Some(path) = &opts.input_path { + if path.is_dir() { + // we are converting from a dat directory rather than a + // dump + + let header = { + let mut header_file = + File::open(path.join(HEADER_FILENAME))?; + let mut raw_header = vec![]; + header_file.read_to_end(&mut raw_header)?; + + serialization::DataHeader::deserialize(&raw_header)? + }; + + let mut data: Vec = vec![]; + + for mem_dec in &header.memories { + let starting_len = data.len(); + let mem_file = BufReader::new(File::open( + path.join(&mem_dec.name), + )?); + + let mut line_data = vec![]; + for line in mem_file.lines() { + let line = line?; + for pair in &line.chars().chunks(2) { + // there has got to be a better way to do this... + let string = + pair.into_iter().collect::(); + let val = u8::from_str_radix(&string, 16) + .expect("invalid hex"); + line_data.push(val); + } + // TODO griffin: handle inputs that are + // truncated or otherwise shorter than expected + + assert!( + line_data.len() + == (mem_dec.bytes_per_entry() as usize) + ); + // reverse the byte order to get the expected + // little endian and reuse the vec + data.extend(line_data.drain(..).rev()) + } + + assert_eq!( + data.len() - starting_len, + mem_dec.byte_count() + ); + } + + DataDump { header, data } + } else { + // we are converting from a dump file + serialization::DataDump::deserialize( + &mut get_read_handle(&opts)?, + )? + } + } else { + // we are converting from a dump file + serialization::DataDump::deserialize(&mut get_read_handle( + &opts, + )?)? + }; + + let mut output = get_output_handle(&opts)?; + let json_data = converter::convert_from_data_dump( &data_dump, opts.use_quotes, @@ -134,6 +202,51 @@ fn main() -> Result<(), CiderDataConverterError> { serde_json::to_string_pretty(&json_data)? )?; } + Target::Dat => { + let mut input = get_read_handle(&opts)?; + let parsed_json: JsonData = + serde_json::from_reader(&mut input)?; + let data = converter::convert_to_data_dump( + &parsed_json, + opts.round_float, + ); + + if let Some(path) = opts.output_path { + if path.exists() && !path.is_dir() { + // TODO griffin: Make this an actual error + panic!("Output path exists but is not a directory") + } else if !path.exists() { + std::fs::create_dir(&path)?; + } + + let mut header_output = File::create(path.join("header"))?; + header_output.write_all(&data.header.serialize()?)?; + + for memory in &data.header.memories { + let file = File::create(path.join(&memory.name))?; + let mut writer = BufWriter::new(file); + for bytes in data + .get_data(&memory.name) + .unwrap() + .chunks_exact(memory.bytes_per_entry() as usize) + { + // data file seems to expect lsb on the right + // for the moment electing to print out every byte + // and do so with two hex digits per byte rather + // than truncating leading zeroes. No need to do + // anything fancy here. + for byte in bytes.iter().rev() { + write!(writer, "{byte:02X}")?; + } + + writeln!(writer)?; + } + } + } else { + // TODO griffin: Make this an actual error + panic!("Output path not specified, this is required for the dat target") + } + } } } else { // Since we can't guess based on input/output file names and no target @@ -143,3 +256,34 @@ fn main() -> Result<(), CiderDataConverterError> { Ok(()) } + +#[allow(clippy::type_complexity)] +fn get_io_handles( + opts: &Opts, +) -> Result<(Box, Box), CiderDataConverterError> { + let input = get_read_handle(opts)?; + let output = get_output_handle(opts)?; + Ok((input, output)) +} + +fn get_output_handle( + opts: &Opts, +) -> Result, CiderDataConverterError> { + let output: Box = opts + .output_path + .as_ref() + .map(|path| File::create(path).map(|x| Box::new(x) as Box)) + .unwrap_or(Ok(Box::new(io::stdout())))?; + Ok(output) +} + +fn get_read_handle( + opts: &Opts, +) -> Result, CiderDataConverterError> { + let input: Box = opts + .input_path + .as_ref() + .map(|path| File::open(path).map(|x| Box::new(x) as Box)) + .unwrap_or(Ok(Box::new(io::stdin())))?; + Ok(input) +} From 33255f99918f8158c226b5823be44c00d2128a70 Mon Sep 17 00:00:00 2001 From: Ethan Uppal <113849268+ethanuppal@users.noreply.github.com> Date: Tue, 12 Nov 2024 11:11:00 -0500 Subject: [PATCH 2/4] [cider2] Make minimally-sized clonable simulator (#2345) Creates a new `BaseSimulator` object that can be used independently of `Simulator`. Also, things are cloneable now. The rationale is we can now write a model checker using cider by cloning cider contexts at forks. --- interp/Cargo.toml | 1 - .../src/flatten/primitives/combinational.rs | 15 ++ interp/src/flatten/primitives/macros.rs | 6 + interp/src/flatten/primitives/prim_trait.rs | 11 +- .../src/flatten/primitives/stateful/math.rs | 24 ++ .../flatten/primitives/stateful/memories.rs | 28 +++ interp/src/flatten/primitives/utils.rs | 1 + .../src/flatten/structures/environment/env.rs | 236 +++++++++++++----- .../src/flatten/structures/environment/mod.rs | 2 +- .../structures/environment/program_counter.rs | 2 +- interp/src/flatten/structures/indexed_map.rs | 3 +- interp/src/flatten/structures/thread.rs | 4 +- 12 files changed, 258 insertions(+), 75 deletions(-) diff --git a/interp/Cargo.toml b/interp/Cargo.toml index 537518f77..8d99d1a84 100644 --- a/interp/Cargo.toml +++ b/interp/Cargo.toml @@ -48,7 +48,6 @@ baa = { version = "0.14.0", features = ["bigint", "serde1", "fraction1"] } fst-writer = "0.2.1" bon = "2.3" - [dev-dependencies] proptest = "1.0.0" diff --git a/interp/src/flatten/primitives/combinational.rs b/interp/src/flatten/primitives/combinational.rs index da6ddb1d5..8df70e435 100644 --- a/interp/src/flatten/primitives/combinational.rs +++ b/interp/src/flatten/primitives/combinational.rs @@ -11,6 +11,7 @@ use baa::{BitVecOps, BitVecValue}; use super::prim_trait::UpdateResult; +#[derive(Clone)] pub struct StdConst { value: BitVecValue, out: GlobalPortIdx, @@ -44,8 +45,13 @@ impl Primitive for StdConst { fn has_stateful(&self) -> bool { false } + + fn clone_boxed(&self) -> Box { + Box::new(self.clone()) + } } +#[derive(Clone)] pub struct StdMux { base: GlobalPortIdx, } @@ -80,6 +86,10 @@ impl Primitive for StdMux { fn has_stateful(&self) -> bool { false } + + fn clone_boxed(&self) -> Box { + Box::new(self.clone()) + } } comb_primitive!(StdNot(input [0]) -> (out [1]) { @@ -275,6 +285,7 @@ comb_primitive!(StdUnsynSmod[WIDTH](left [0], right [1]) -> (out [2]) { Ok(Some(BitVecValue::from_big_int(&res, WIDTH))) }); +#[derive(Clone)] pub struct StdUndef(GlobalPortIdx); impl StdUndef { @@ -288,4 +299,8 @@ impl Primitive for StdUndef { port_map.write_undef(self.0)?; Ok(UpdateStatus::Unchanged) } + + fn clone_boxed(&self) -> Box { + Box::new(self.clone()) + } } diff --git a/interp/src/flatten/primitives/macros.rs b/interp/src/flatten/primitives/macros.rs index 7a34274e4..bf75c6f1b 100644 --- a/interp/src/flatten/primitives/macros.rs +++ b/interp/src/flatten/primitives/macros.rs @@ -102,6 +102,12 @@ macro_rules! comb_primitive { false } + fn clone_boxed(&self) -> Box { + Box::new(Self { + base_port: self.base_port, + $($($param: self.$param,)+)? + }) + } } }; diff --git a/interp/src/flatten/primitives/prim_trait.rs b/interp/src/flatten/primitives/prim_trait.rs index ecfb0b2e5..199adce45 100644 --- a/interp/src/flatten/primitives/prim_trait.rs +++ b/interp/src/flatten/primitives/prim_trait.rs @@ -134,6 +134,8 @@ pub trait Primitive { fn dump_memory_state(&self) -> Option> { None } + + fn clone_boxed(&self) -> Box; } pub trait RaceDetectionPrimitive: Primitive { @@ -158,10 +160,13 @@ pub trait RaceDetectionPrimitive: Primitive { /// Get a reference to the underlying primitive. Unfortunately cannot add an /// optional default implementation due to size rules fn as_primitive(&self) -> &dyn Primitive; + + fn clone_boxed_rd(&self) -> Box; } /// An empty primitive implementation used for testing. It does not do anything /// and has no ports of any kind +#[derive(Clone, Copy)] pub struct DummyPrimitive; impl DummyPrimitive { @@ -170,4 +175,8 @@ impl DummyPrimitive { } } -impl Primitive for DummyPrimitive {} +impl Primitive for DummyPrimitive { + fn clone_boxed(&self) -> Box { + Box::new(*self) + } +} diff --git a/interp/src/flatten/primitives/stateful/math.rs b/interp/src/flatten/primitives/stateful/math.rs index 30e659810..c8fb091e4 100644 --- a/interp/src/flatten/primitives/stateful/math.rs +++ b/interp/src/flatten/primitives/stateful/math.rs @@ -10,6 +10,7 @@ use crate::flatten::{ use baa::{BitVecOps, BitVecValue, WidthInt}; use num_traits::Euclid; +#[derive(Clone)] pub struct StdMultPipe { base_port: GlobalPortIdx, pipeline: ShiftBuffer<(PortValue, PortValue), DEPTH>, @@ -32,6 +33,10 @@ impl StdMultPipe { } impl Primitive for StdMultPipe { + fn clone_boxed(&self) -> Box { + Box::new(self.clone()) + } + fn exec_comb(&self, port_map: &mut PortMap) -> UpdateResult { ports![&self.base_port; out: Self::OUT, done: Self::DONE]; @@ -106,6 +111,7 @@ impl Primitive for StdMultPipe { } } +#[derive(Clone)] pub struct StdDivPipe { base_port: GlobalPortIdx, pipeline: ShiftBuffer<(PortValue, PortValue), DEPTH>, @@ -132,6 +138,10 @@ impl StdDivPipe { impl Primitive for StdDivPipe { + fn clone_boxed(&self) -> Box { + Box::new(self.clone()) + } + fn exec_comb(&self, port_map: &mut PortMap) -> UpdateResult { ports![&self.base_port; out_quot: Self::OUT_QUOTIENT, @@ -253,6 +263,10 @@ impl Sqrt { } impl Primitive for Sqrt { + fn clone_boxed(&self) -> Box { + Box::new(self.clone()) + } + fn exec_comb(&self, port_map: &mut PortMap) -> UpdateResult { ports![&self.base_port; out: Self::OUT, done: Self::DONE]; @@ -309,6 +323,7 @@ impl Primitive for Sqrt { } } +#[derive(Clone)] pub struct FxpMultPipe { base_port: GlobalPortIdx, pipeline: ShiftBuffer<(PortValue, PortValue), DEPTH>, @@ -339,6 +354,10 @@ impl FxpMultPipe { } impl Primitive for FxpMultPipe { + fn clone_boxed(&self) -> Box { + Box::new(self.clone()) + } + fn exec_comb(&self, port_map: &mut PortMap) -> UpdateResult { ports![&self.base_port; out: Self::OUT, done: Self::DONE]; @@ -422,6 +441,7 @@ impl Primitive for FxpMultPipe { } } +#[derive(Clone)] pub struct FxpDivPipe { base_port: GlobalPortIdx, pipeline: ShiftBuffer<(PortValue, PortValue), DEPTH>, @@ -460,6 +480,10 @@ impl FxpDivPipe { impl Primitive for FxpDivPipe { + fn clone_boxed(&self) -> Box { + Box::new(self.clone()) + } + fn exec_comb(&self, port_map: &mut PortMap) -> UpdateResult { ports![&self.base_port; out_quot: Self::OUT_QUOTIENT, diff --git a/interp/src/flatten/primitives/stateful/memories.rs b/interp/src/flatten/primitives/stateful/memories.rs index 7b2f5680b..8365907f2 100644 --- a/interp/src/flatten/primitives/stateful/memories.rs +++ b/interp/src/flatten/primitives/stateful/memories.rs @@ -27,6 +27,7 @@ use crate::{ use baa::{BitVecOps, BitVecValue, WidthInt}; +#[derive(Clone)] pub struct StdReg { base_port: GlobalPortIdx, internal_state: ValueWithClock, @@ -55,6 +56,10 @@ impl StdReg { } impl Primitive for StdReg { + fn clone_boxed(&self) -> Box { + Box::new(self.clone()) + } + fn exec_cycle(&mut self, port_map: &mut PortMap) -> UpdateResult { ports![&self.base_port; input: Self::IN, @@ -139,6 +144,10 @@ impl Primitive for StdReg { } impl RaceDetectionPrimitive for StdReg { + fn clone_boxed_rd(&self) -> Box { + Box::new(self.clone()) + } + fn as_primitive(&self) -> &dyn Primitive { self } @@ -176,6 +185,7 @@ impl RaceDetectionPrimitive for StdReg { } } +#[derive(Clone)] pub struct MemDx { shape: Shape, } @@ -389,6 +399,7 @@ impl MemDx { } } +#[derive(Clone)] pub struct CombMem { base_port: GlobalPortIdx, internal_state: Vec, @@ -508,6 +519,10 @@ impl CombMem { } impl Primitive for CombMem { + fn clone_boxed(&self) -> Box { + Box::new(self.clone()) + } + fn exec_comb(&self, port_map: &mut PortMap) -> UpdateResult { let addr: Option = self .addresser @@ -609,6 +624,10 @@ impl Primitive for CombMem { } impl RaceDetectionPrimitive for CombMem { + fn clone_boxed_rd(&self) -> Box { + Box::new(self.clone()) + } + fn as_primitive(&self) -> &dyn Primitive { self } @@ -674,6 +693,7 @@ impl RaceDetectionPrimitive for CombMem { } } +#[derive(Clone)] pub struct SeqMem { base_port: GlobalPortIdx, internal_state: Vec, @@ -812,6 +832,10 @@ impl SeqMem { } impl Primitive for SeqMem { + fn clone_boxed(&self) -> Box { + Box::new(self.clone()) + } + fn exec_comb(&self, port_map: &mut PortMap) -> UpdateResult { let done_signal = port_map.insert_val_general( self.done(), @@ -915,6 +939,10 @@ impl Primitive for SeqMem { } impl RaceDetectionPrimitive for SeqMem { + fn clone_boxed_rd(&self) -> Box { + Box::new(self.clone()) + } + fn as_primitive(&self) -> &dyn Primitive { self } diff --git a/interp/src/flatten/primitives/utils.rs b/interp/src/flatten/primitives/utils.rs index 6b91e17a3..51ab75fd5 100644 --- a/interp/src/flatten/primitives/utils.rs +++ b/interp/src/flatten/primitives/utils.rs @@ -38,6 +38,7 @@ pub(crate) fn int_sqrt(i: &BigUint) -> BigUint { } /// A shift buffer of a fixed size +#[derive(Clone)] pub struct ShiftBuffer { buffer: VecDeque>, } diff --git a/interp/src/flatten/structures/environment/env.rs b/interp/src/flatten/structures/environment/env.rs index 479619a81..65769f4d1 100644 --- a/interp/src/flatten/structures/environment/env.rs +++ b/interp/src/flatten/structures/environment/env.rs @@ -156,6 +156,7 @@ pub(crate) type RefPortMap = IndexedMap>; pub(crate) type AssignmentRange = IndexRange; +#[derive(Clone)] pub(crate) struct ComponentLedger { pub(crate) index_bases: BaseIndices, pub(crate) comp_id: ComponentIdx, @@ -192,6 +193,24 @@ pub(crate) enum CellLedger { Component(ComponentLedger), } +impl Clone for CellLedger { + fn clone(&self) -> Self { + match self { + Self::Primitive { cell_dyn } => Self::Primitive { + cell_dyn: cell_dyn.clone_boxed(), + }, + Self::RaceDetectionPrimitive { cell_dyn } => { + Self::RaceDetectionPrimitive { + cell_dyn: cell_dyn.clone_boxed_rd(), + } + } + Self::Component(component_ledger) => { + Self::Component(component_ledger.clone()) + } + } + } +} + impl From for CellLedger { fn from(v: ComponentLedger) -> Self { Self::Component(v) @@ -302,7 +321,7 @@ impl PinnedPorts { } } -#[derive(Debug)] +#[derive(Debug, Clone)] pub struct Environment + Clone> { /// A map from global port IDs to their current values. ports: PortMap, @@ -1285,24 +1304,44 @@ impl + Clone> Environment { } } +/// The core functionality of a simulator. Clonable. +#[derive(Clone)] +pub struct BaseSimulator + Clone> { + env: Environment, + conf: RuntimeConfig, +} + /// A wrapper struct for the environment that provides the functions used to /// simulate the actual program. /// /// This is just to keep the simulation logic under a different namespace than /// the environment to avoid confusion pub struct Simulator + Clone> { - env: Environment, + base: BaseSimulator, wave: Option, - conf: RuntimeConfig, } impl + Clone> Simulator { - pub fn new( - env: Environment, + pub fn build_simulator( + ctx: C, + data_file: &Option, wave_file: &Option, - conf: RuntimeConfig, - ) -> Self { - // open the wave form file and declare all signals + runtime_config: RuntimeConfig, + ) -> Result { + let data_dump = data_file + .as_ref() + .map(|path| { + let mut file = std::fs::File::open(path)?; + DataDump::deserialize(&mut file) + }) + // flip to a result of an option + .map_or(Ok(None), |res| res.map(Some))?; + let env = Environment::new( + ctx, + data_dump, + runtime_config.check_data_race, + runtime_config.get_logging_config(), + ); let wave = wave_file.as_ref().map(|p| match WaveWriter::open(p, &env) { Ok(w) => w, @@ -1310,9 +1349,120 @@ impl + Clone> Simulator { todo!("deal more gracefully with error: {err:?}") } }); - let mut output = Self { env, wave, conf }; - output.set_root_go_high(); - output + Ok(Self { + base: BaseSimulator::new(env, runtime_config), + wave, + }) + } + + pub fn is_done(&self) -> bool { + self.base.is_done() + } + + pub fn step(&mut self) -> CiderResult<()> { + self.base.step() + } + + pub fn converge(&mut self) -> CiderResult<()> { + self.base.converge() + } + + pub fn get_currently_running_groups( + &self, + ) -> impl Iterator + '_ { + self.base.get_currently_running_groups() + } + + pub fn is_group_running(&self, group_idx: GroupIdx) -> bool { + self.base.is_group_running(group_idx) + } + + pub fn print_pc(&self) { + self.base.print_pc(); + } + + pub fn format_cell_state( + &self, + cell_idx: GlobalCellIdx, + print_code: PrintCode, + name: Option<&str>, + ) -> Option { + self.base.format_cell_state(cell_idx, print_code, name) + } + + pub fn format_cell_ports( + &self, + cell_idx: GlobalCellIdx, + print_code: PrintCode, + name: Option<&str>, + ) -> String { + self.base.format_cell_ports(cell_idx, print_code, name) + } + + pub fn format_port_value( + &self, + port_idx: GlobalPortIdx, + print_code: PrintCode, + ) -> String { + self.base.format_port_value(port_idx, print_code) + } + + pub fn traverse_name_vec( + &self, + name: &[String], + ) -> Result { + self.base.traverse_name_vec(name) + } + + pub fn get_full_name(&self, nameable: N) -> String + where + N: GetFullName, + { + self.base.get_full_name(nameable) + } + + pub(crate) fn env(&self) -> &Environment { + self.base.env() + } + + /// Evaluate the entire program + pub fn run_program(&mut self) -> CiderResult<()> { + if self.base.conf.debug_logging { + info!(self.base.env().logger, "Starting program execution"); + } + + match self.base.run_program_inner(self.wave.as_mut()) { + Ok(_) => { + if self.base.conf.debug_logging { + info!(self.base.env().logger, "Finished program execution"); + } + Ok(()) + } + Err(e) => { + if self.base.conf.debug_logging { + slog::error!( + self.base.env().logger, + "Program execution failed with error: {}", + e.red() + ); + } + Err(e) + } + } + } + + pub fn dump_memories( + &self, + dump_registers: bool, + all_mems: bool, + ) -> DataDump { + self.base.dump_memories(dump_registers, all_mems) + } +} + +impl + Clone> BaseSimulator { + pub fn new(env: Environment, conf: RuntimeConfig) -> Self { + Self { env, conf } } pub(crate) fn env(&self) -> &Environment { @@ -1332,33 +1482,6 @@ impl + Clone> Simulator { self.env } - pub fn build_simulator( - ctx: C, - data_file: &Option, - wave_file: &Option, - runtime_config: RuntimeConfig, - ) -> Result { - let data_dump = data_file - .as_ref() - .map(|path| { - let mut file = std::fs::File::open(path)?; - DataDump::deserialize(&mut file) - }) - // flip to a result of an option - .map_or(Ok(None), |res| res.map(Some))?; - - Ok(Simulator::new( - Environment::new( - ctx, - data_dump, - runtime_config.check_data_race, - runtime_config.get_logging_config(), - ), - wave_file, - runtime_config, - )) - } - pub fn is_group_running(&self, group_idx: GroupIdx) -> bool { self.env.is_group_running(group_idx) } @@ -1418,7 +1541,7 @@ impl + Clone> Simulator { } // =========================== simulation functions =========================== -impl + Clone> Simulator { +impl + Clone> BaseSimulator { #[inline] fn lookup_global_port_id(&self, port: GlobalPortRef) -> GlobalPortIdx { match port { @@ -2161,43 +2284,20 @@ impl + Clone> Simulator { .unwrap_or_default() } - /// Evaluate the entire program - pub fn run_program(&mut self) -> CiderResult<()> { - if self.conf.debug_logging { - info!(self.env.logger, "Starting program execution"); - } - - match self.run_program_inner() { - Ok(_) => { - if self.conf.debug_logging { - info!(self.env.logger, "Finished program execution"); - } - Ok(()) - } - Err(e) => { - if self.conf.debug_logging { - slog::error!( - self.env.logger, - "Program execution failed with error: {}", - e.red() - ); - } - Err(e) - } - } - } - - fn run_program_inner(&mut self) -> Result<(), BoxedCiderError> { + pub fn run_program_inner( + &mut self, + mut wave: Option<&mut WaveWriter>, + ) -> Result<(), BoxedCiderError> { let mut time = 0; while !self.is_done() { - if let Some(wave) = self.wave.as_mut() { + if let Some(wave) = wave.as_mut() { wave.write_values(time, &self.env.ports)?; } // self.print_pc(); self.step()?; time += 1; } - if let Some(wave) = self.wave.as_mut() { + if let Some(wave) = wave { wave.write_values(time, &self.env.ports)?; } Ok(()) diff --git a/interp/src/flatten/structures/environment/mod.rs b/interp/src/flatten/structures/environment/mod.rs index f5133e74a..40808bfbe 100644 --- a/interp/src/flatten/structures/environment/mod.rs +++ b/interp/src/flatten/structures/environment/mod.rs @@ -5,7 +5,7 @@ mod program_counter; mod traverser; mod wave; -pub use env::{Environment, PortMap, Simulator}; +pub use env::{BaseSimulator, Environment, PortMap, Simulator}; pub use traverser::{Path, PathError, PathResolution}; pub(crate) use env::CellLedger; diff --git a/interp/src/flatten/structures/environment/program_counter.rs b/interp/src/flatten/structures/environment/program_counter.rs index 3ce817a84..dc8223d24 100644 --- a/interp/src/flatten/structures/environment/program_counter.rs +++ b/interp/src/flatten/structures/environment/program_counter.rs @@ -375,7 +375,7 @@ impl WithEntry { /// The program counter for the whole program execution. Wraps over a vector of /// the active leaf statements for each component instance. -#[derive(Debug, Default)] +#[derive(Debug, Default, Clone)] pub(crate) struct ProgramCounter { vec: Vec, par_map: HashMap, diff --git a/interp/src/flatten/structures/indexed_map.rs b/interp/src/flatten/structures/indexed_map.rs index 137788b09..e3f24c315 100644 --- a/interp/src/flatten/structures/indexed_map.rs +++ b/interp/src/flatten/structures/indexed_map.rs @@ -4,7 +4,7 @@ use std::{ ops::{self, Index}, }; -#[derive(Debug)] +#[derive(Debug, Clone)] pub struct IndexedMap where K: IndexRef, @@ -143,6 +143,7 @@ where Self::new() } } + #[allow(dead_code)] pub struct IndexedMapRangeIterator<'range, 'data, K, D> where diff --git a/interp/src/flatten/structures/thread.rs b/interp/src/flatten/structures/thread.rs index 938990dba..8ca8a5bc9 100644 --- a/interp/src/flatten/structures/thread.rs +++ b/interp/src/flatten/structures/thread.rs @@ -9,7 +9,7 @@ use super::{ pub struct ThreadIdx(NonZeroU32); impl_index_nonzero!(ThreadIdx); -#[derive(Debug)] +#[derive(Debug, Clone)] pub struct ThreadInfo { parent: Option, clock_id: ClockIdx, @@ -25,7 +25,7 @@ impl ThreadInfo { } } -#[derive(Debug)] +#[derive(Debug, Clone)] pub struct ThreadMap { map: IndexedMap, } From dd55f4a2ac4316f8729987f9d58b209963bd1002 Mon Sep 17 00:00:00 2001 From: Elias Castro <127804425+eliascxstro@users.noreply.github.com> Date: Tue, 12 Nov 2024 13:41:35 -0500 Subject: [PATCH 3/4] Seq (#2346) Configured various files and edited the parser to print out nodes in the format we talked about. Biggest function implemented was string_path in **program_counter.rs** --- .../src/debugger/commands/command_parser.rs | 17 ++++-- interp/src/debugger/commands/commands.pest | 3 +- interp/src/debugger/commands/core.rs | 9 ++- interp/src/debugger/debugger_core.rs | 15 +++-- .../src/flatten/structures/environment/env.rs | 24 ++++++++ .../structures/environment/program_counter.rs | 61 +++++++++++++++++++ 6 files changed, 119 insertions(+), 10 deletions(-) diff --git a/interp/src/debugger/commands/command_parser.rs b/interp/src/debugger/commands/command_parser.rs index 60af379f6..c48179d1a 100644 --- a/interp/src/debugger/commands/command_parser.rs +++ b/interp/src/debugger/commands/command_parser.rs @@ -1,5 +1,8 @@ -use super::core::{ - Command, ParsedBreakPointID, ParsedGroupName, PrintMode, WatchPosition, +use super::{ + core::{ + Command, ParsedBreakPointID, ParsedGroupName, PrintMode, WatchPosition, + }, + PrintCommand, }; use baa::WidthInt; use pest_consume::{match_nodes, Error, Parser}; @@ -21,10 +24,15 @@ impl CommandParser { fn EOI(_input: Node) -> ParseResult<()> { Ok(()) } + fn code_calyx(_input: Node) -> ParseResult<()> { Ok(()) } + fn code_nodes(_input: Node) -> ParseResult<()> { + Ok(()) + } + // ---------------------- fn help(_input: Node) -> ParseResult { @@ -60,8 +68,9 @@ impl CommandParser { fn comm_where(input: Node) -> ParseResult { Ok(match_nodes!(input.into_children(); - [code_calyx(_)] => Command::PrintPC(true), - [] => Command::PrintPC(false), + [code_calyx(_)] => Command::PrintPC(PrintCommand::PrintCalyx), + [code_nodes(_)] => Command::PrintPC(PrintCommand::PrintNodes), + [] => Command::PrintPC(PrintCommand::Normal), )) } diff --git a/interp/src/debugger/commands/commands.pest b/interp/src/debugger/commands/commands.pest index 799971648..7a1920c68 100644 --- a/interp/src/debugger/commands/commands.pest +++ b/interp/src/debugger/commands/commands.pest @@ -14,6 +14,7 @@ pc_s = { ^"s" } pc_ufx = { ^"u." ~ num } pc_sfx = { ^"s." ~ num } code_calyx = { ^"calyx" } +code_nodes = {^"nodes"} print_code = { "\\" ~ (pc_ufx | pc_sfx | pc_s | pc_un) @@ -67,7 +68,7 @@ disable_watch = { (^"disable-watch " | ^"disw ") ~ brk_id+ } exit = { ^"exit" | ^"quit" } -comm_where = { (^"where" | "pc") ~ (code_calyx)? } +comm_where = { (^"where" | "pc") ~ (code_calyx | code_nodes)? } explain = { ^"explain" } diff --git a/interp/src/debugger/commands/core.rs b/interp/src/debugger/commands/core.rs index 46f86279b..37148ef8a 100644 --- a/interp/src/debugger/commands/core.rs +++ b/interp/src/debugger/commands/core.rs @@ -299,6 +299,13 @@ impl From<(Vec, Option, PrintMode)> for PrintTuple { } } +// Different types of printing commands +pub enum PrintCommand { + Normal, + PrintCalyx, + PrintNodes, +} + /// A command that can be sent to the debugger. pub enum Command { /// Advance the execution by a given number of steps (cycles). @@ -345,7 +352,7 @@ pub enum Command { PrintMode, ), /// Print the current program counter - PrintPC(bool), + PrintPC(PrintCommand), /// Show command examples Explain, /// Restart the debugger from the beginning of the execution. Command history, breakpoints, watchpoints, etc. are preserved. diff --git a/interp/src/debugger/debugger_core.rs b/interp/src/debugger/debugger_core.rs index 837869f9f..e9fc878c2 100644 --- a/interp/src/debugger/debugger_core.rs +++ b/interp/src/debugger/debugger_core.rs @@ -6,7 +6,9 @@ use super::{ }; use crate::{ configuration::RuntimeConfig, - debugger::{source::SourceMap, unwrap_error_message}, + debugger::{ + commands::PrintCommand, source::SourceMap, unwrap_error_message, + }, errors::{CiderError, CiderResult}, flatten::{ flat_ir::prelude::GroupIdx, @@ -368,10 +370,15 @@ impl + Clone> Debugger { Command::InfoWatch => self .debugging_context .print_watchpoints(self.interpreter.env()), - Command::PrintPC(_override_flag) => { - self.interpreter.print_pc(); - } + Command::PrintPC(print_mode) => match print_mode { + PrintCommand::Normal | PrintCommand::PrintCalyx => { + self.interpreter.print_pc(); + } + PrintCommand::PrintNodes => { + self.interpreter.print_pc_string(); + } + }, Command::Explain => { print!("{}", Command::get_explain_string()) } diff --git a/interp/src/flatten/structures/environment/env.rs b/interp/src/flatten/structures/environment/env.rs index 65769f4d1..7b7360a57 100644 --- a/interp/src/flatten/structures/environment/env.rs +++ b/interp/src/flatten/structures/environment/env.rs @@ -357,6 +357,11 @@ impl + Clone> Environment { pub fn ctx(&self) -> &Context { self.ctx.as_ref() } + + pub fn pc_iter(&self) -> impl Iterator { + self.pc.iter().map(|(_, x)| x) + } + /// Returns the full name and port list of each cell in the context pub fn iter_cells( &self, @@ -857,6 +862,17 @@ impl + Clone> Environment { } } + pub fn print_pc_string(&self) { + let ctx = self.ctx.as_ref(); + for node in self.pc_iter() { + println!( + "{}: {}", + self.get_full_name(node.comp), + node.string_path(ctx) + ); + } + } + fn get_name_from_cell_and_parent( &self, parent: GlobalCellIdx, @@ -1458,6 +1474,10 @@ impl + Clone> Simulator { ) -> DataDump { self.base.dump_memories(dump_registers, all_mems) } + + pub fn print_pc_string(&self) { + self.base.print_pc_string() + } } impl + Clone> BaseSimulator { @@ -1516,6 +1536,10 @@ impl + Clone> BaseSimulator { self.env.print_pc() } + pub fn print_pc_string(&self) { + self.env.print_pc_string() + } + /// Pins the port with the given name to the given value. This may only be /// used for input ports on the entrypoint component (excluding the go port) /// and will panic if used otherwise. Intended for external use. diff --git a/interp/src/flatten/structures/environment/program_counter.rs b/interp/src/flatten/structures/environment/program_counter.rs index dc8223d24..918303141 100644 --- a/interp/src/flatten/structures/environment/program_counter.rs +++ b/interp/src/flatten/structures/environment/program_counter.rs @@ -59,6 +59,67 @@ impl ControlPoint { false } } + + /// Returns a string showing the path from the root node to input node. This + /// path is displayed in the minimal metadata path syntax. + pub fn string_path(&self, ctx: &Context) -> String { + let path = SearchPath::find_path_from_root(self.control_node_idx, ctx); + let mut path_vec = path.path; + + // Remove first element since we know it is a root + path_vec.remove(0); + let mut string_path = String::new(); + string_path.push('.'); + let control_map = &ctx.primary.control; + let mut count = -1; + let mut body = false; + let mut if_branches: HashMap = HashMap::new(); + for search_node in path_vec { + // The control_idx should exist in the map, so we shouldn't worry about it + // exploding. First SearchNode is root, hence "." + let control_idx = search_node.node; + let control_node = control_map.get(control_idx).unwrap(); + match control_node { + // These are terminal nodes + // ControlNode::Empty(_) => "empty", + // ControlNode::Invoke(_) => "invoke", + // ControlNode::Enable(_) => "enable", + + // These have unbounded children + // ControlNode::Seq(_) => "seq", + // ControlNode::Par(_) => "par", + + // Special cases + ControlNode::If(if_node) => { + if_branches.insert(if_node.tbranch(), String::from("t")); + if_branches.insert(if_node.tbranch(), String::from("f")); + } + ControlNode::While(_) => { + body = true; + } + ControlNode::Repeat(_) => { + body = true; + } + _ => {} + }; + + let control_type = if body { + body = false; + count = -1; + String::from("b") + } else if if_branches.contains_key(&control_idx) { + let (_, branch) = + if_branches.get_key_value(&control_idx).unwrap(); + branch.clone() + } else { + count += 1; + count.to_string() + }; + + string_path = string_path + "-" + &control_type; + } + string_path + } } #[derive(Debug, Clone)] From fa298e93c23ea7658e1b8dbf56d2145c6b65976a Mon Sep 17 00:00:00 2001 From: Griffin Berlstein Date: Thu, 14 Nov 2024 15:11:18 +0000 Subject: [PATCH 4/4] [Data converter] Improve the hex parsing & general functionality (#2352) I spent some more time hacking on this instead of spending my time in a more productive way. This does the following: - switch the `dat` deserializing from custom matching stuff to a proper `nom` parser that accounts for comments and leading `0x` tags. - `dat` files can now parse values with leading zeroes truncated (though we continue to generate `dat` files with the leading zeroes included) - Output `dat` files will be generated by default as `MEMNAME.dat` though this can be customized with `-e` flag. I.e. `-e out` will generate `MEMNAME.out` - Similarly, when reading in a `dat` directory, the tool will look for `MEMNAME.dat` which can be retargeted via `-e` flag. - The tool will also infer the `--to json` target when given a directory as input --- Cargo.lock | 19 ++- tools/cider-data-converter/Cargo.toml | 1 + tools/cider-data-converter/src/dat_parser.rs | 133 +++++++++++++++++++ tools/cider-data-converter/src/lib.rs | 1 + tools/cider-data-converter/src/main.rs | 90 ++++++++----- 5 files changed, 213 insertions(+), 31 deletions(-) create mode 100644 tools/cider-data-converter/src/dat_parser.rs diff --git a/Cargo.lock b/Cargo.lock index 7b5156844..7eba77637 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -544,6 +544,7 @@ dependencies = [ "argh", "interp", "itertools 0.11.0", + "nom 7.1.3", "num-bigint", "num-rational", "num-traits", @@ -1623,6 +1624,12 @@ version = "2.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "523dc4f511e55ab87b694dc30d0f820d60906ef06413f93d4d7a1385599cc149" +[[package]] +name = "minimal-lexical" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" + [[package]] name = "miniz_oxide" version = "0.7.2" @@ -1674,6 +1681,16 @@ dependencies = [ "version_check 0.1.5", ] +[[package]] +name = "nom" +version = "7.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" +dependencies = [ + "memchr", + "minimal-lexical", +] + [[package]] name = "nu-ansi-term" version = "0.46.0" @@ -2409,7 +2426,7 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5318bfeed779c64075ce317c81462ed54dc00021be1c6b34957d798e11a68bdb" dependencies = [ - "nom", + "nom 4.2.3", "serde", ] diff --git a/tools/cider-data-converter/Cargo.toml b/tools/cider-data-converter/Cargo.toml index a5755b406..a690023d3 100644 --- a/tools/cider-data-converter/Cargo.toml +++ b/tools/cider-data-converter/Cargo.toml @@ -17,6 +17,7 @@ thiserror = "1.0.59" num-bigint = { version = "0.4.6" } num-rational = { version = "0.4.2" } num-traits = { version = "0.2.19" } +nom = "7.1.3" [dev-dependencies] proptest = "1.0.0" diff --git a/tools/cider-data-converter/src/dat_parser.rs b/tools/cider-data-converter/src/dat_parser.rs new file mode 100644 index 000000000..88f99d6ba --- /dev/null +++ b/tools/cider-data-converter/src/dat_parser.rs @@ -0,0 +1,133 @@ +use nom::{ + branch::alt, + bytes::complete::{tag, take_while_m_n}, + character::complete::{anychar, line_ending, multispace0}, + combinator::{eof, map_res, opt}, + error::Error, + multi::{many1, many_till}, + sequence::{preceded, tuple}, + IResult, +}; + +fn is_hex_digit(c: char) -> bool { + c.is_ascii_hexdigit() +} + +fn from_hex(input: &str) -> Result { + u8::from_str_radix(input, 16) +} + +fn parse_hex(input: &str) -> IResult<&str, u8> { + map_res(take_while_m_n(1, 2, is_hex_digit), from_hex)(input) +} + +/// Parse a single line of hex characters into a vector of bytes in the order +/// the characters are given, i.e. reversed. +fn hex_line(input: &str) -> IResult<&str, LineOrComment> { + // strip any leading whitespace + let (input, bytes) = preceded( + tuple((multispace0, opt(tag("0x")))), + many1(parse_hex), + )(input)?; + + Ok((input, LineOrComment::Line(bytes))) +} + +fn comment(input: &str) -> IResult<&str, LineOrComment> { + // skip any whitespace + let (input, _) = multispace0(input)?; + let (input, _) = tag("//")(input)?; + let (input, _) = many_till(anychar, alt((line_ending, eof)))(input)?; + Ok((input, LineOrComment::Comment)) +} +/// Parse a line which only contains whitespace +fn empty_line(input: &str) -> IResult<&str, LineOrComment> { + // skip any whitespace + let (input, _) = multispace0(input)?; + Ok((input, LineOrComment::EmptyLine)) +} + +pub fn line_or_comment( + input: &str, +) -> Result>> { + let (_, res) = alt((hex_line, comment, empty_line))(input)?; + Ok(res) +} + +#[derive(Debug, PartialEq)] +pub enum LineOrComment { + Line(Vec), + Comment, + EmptyLine, +} + +/// Parse a single line of hex characters, or a comment. Returns None if it's a +/// comment or an empty line and Some(Vec) if it's a hex line. Panics on a +/// parse error. +/// +/// For the fallible version, see `line_or_comment`. +pub fn unwrap_line_or_comment(input: &str) -> Option> { + match line_or_comment(input).expect("hex parse failed") { + LineOrComment::Line(vec) => Some(vec), + LineOrComment::Comment => None, + LineOrComment::EmptyLine => None, + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_comment() { + assert_eq!(comment("// comment"), Ok(("", LineOrComment::Comment))); + assert_eq!(comment("// comment\n"), Ok(("", LineOrComment::Comment))); + } + + #[test] + fn test_hex_line() { + assert_eq!(hex_line("0x01"), Ok(("", LineOrComment::Line(vec![1])))); + assert_eq!(hex_line("0x02"), Ok(("", LineOrComment::Line(vec![2])))); + assert_eq!(hex_line("0x03"), Ok(("", LineOrComment::Line(vec![3])))); + assert_eq!(hex_line("0x04"), Ok(("", LineOrComment::Line(vec![4])))); + assert_eq!(hex_line("0x05"), Ok(("", LineOrComment::Line(vec![5])))); + assert_eq!(hex_line("0x06"), Ok(("", LineOrComment::Line(vec![6])))); + assert_eq!(hex_line("0x07"), Ok(("", LineOrComment::Line(vec![7])))); + assert_eq!(hex_line("0x08"), Ok(("", LineOrComment::Line(vec![8])))); + assert_eq!(hex_line("0x09"), Ok(("", LineOrComment::Line(vec![9])))); + assert_eq!(hex_line("0x0a"), Ok(("", LineOrComment::Line(vec![10])))); + assert_eq!(hex_line("0x0b"), Ok(("", LineOrComment::Line(vec![11])))); + assert_eq!(hex_line("0x0c"), Ok(("", LineOrComment::Line(vec![12])))); + assert_eq!(hex_line("0x0d"), Ok(("", LineOrComment::Line(vec![13])))); + assert_eq!(hex_line("0x0e"), Ok(("", LineOrComment::Line(vec![14])))); + assert_eq!(hex_line("0x0f"), Ok(("", LineOrComment::Line(vec![15])))); + assert_eq!(hex_line("0xff"), Ok(("", LineOrComment::Line(vec![255])))); + assert_eq!( + hex_line("0x00ff"), + Ok(("", LineOrComment::Line(vec![0, 255]))) + ); + } + + #[test] + fn test_from_hex() { + assert_eq!(from_hex("0"), Ok(0)); + assert_eq!(from_hex("1"), Ok(1)); + assert_eq!(from_hex("2"), Ok(2)); + assert_eq!(from_hex("3"), Ok(3)); + assert_eq!(from_hex("4"), Ok(4)); + assert_eq!(from_hex("5"), Ok(5)); + assert_eq!(from_hex("6"), Ok(6)); + assert_eq!(from_hex("7"), Ok(7)); + assert_eq!(from_hex("8"), Ok(8)); + assert_eq!(from_hex("9"), Ok(9)); + assert_eq!(from_hex("a"), Ok(10)); + assert_eq!(from_hex("b"), Ok(11)); + assert_eq!(from_hex("c"), Ok(12)); + assert_eq!(from_hex("d"), Ok(13)); + assert_eq!(from_hex("e"), Ok(14)); + assert_eq!(from_hex("f"), Ok(15)); + + assert_eq!(from_hex("FF"), Ok(255)); + assert_eq!(from_hex("ff"), Ok(255)); + } +} diff --git a/tools/cider-data-converter/src/lib.rs b/tools/cider-data-converter/src/lib.rs index 7e1dadc1b..31092d73e 100644 --- a/tools/cider-data-converter/src/lib.rs +++ b/tools/cider-data-converter/src/lib.rs @@ -1,2 +1,3 @@ pub mod converter; +pub mod dat_parser; pub mod json_data; diff --git a/tools/cider-data-converter/src/main.rs b/tools/cider-data-converter/src/main.rs index e44fea9cb..a95d910e8 100644 --- a/tools/cider-data-converter/src/main.rs +++ b/tools/cider-data-converter/src/main.rs @@ -1,11 +1,13 @@ use argh::FromArgs; -use cider_data_converter::{converter, json_data::JsonData}; +use cider_data_converter::{ + converter, dat_parser::unwrap_line_or_comment, json_data::JsonData, +}; use core::str; use interp::serialization::{self, DataDump, SerializationError}; -use itertools::Itertools; use std::{ fs::File, io::{self, BufRead, BufReader, BufWriter, Read, Write}, + iter::repeat, path::PathBuf, str::FromStr, }; @@ -13,6 +15,7 @@ use thiserror::Error; const JSON_EXTENSION: &str = "data"; const CIDER_EXTENSION: &str = "dump"; +const DAT_EXTENSION: &str = "dat"; const HEADER_FILENAME: &str = "header"; @@ -32,6 +35,14 @@ enum CiderDataConverterError { #[error(transparent)] DataDumpError(#[from] SerializationError), + + #[error( + "Missing output path. This is required for the \"to dat\" conversion" + )] + MissingDatOutputPath, + + #[error("Output path for \"to dat\" exists but it is a file")] + DatOutputPathIsFile, } impl std::fmt::Debug for CiderDataConverterError { @@ -90,6 +101,12 @@ struct Opts { /// exists solely for backwards compatibility with the old display format. #[argh(switch, long = "legacy-quotes")] use_quotes: bool, + + /// the file extension to use for the output/input file when parsing to and + /// from the dat target. If not provided, the extension is assumed to be .dat + #[argh(option, short = 'e', long = "dat-file-extension")] + #[argh(default = "String::from(DAT_EXTENSION)")] + file_extension: String, } fn main() -> Result<(), CiderDataConverterError> { @@ -97,19 +114,27 @@ fn main() -> Result<(), CiderDataConverterError> { // if no action is specified, try to guess based on file extensions if opts.action.is_none() + // input is .json && (opts.input_path.as_ref().is_some_and(|x| { x.extension().map_or(false, |y| y == JSON_EXTENSION) - }) || opts.output_path.as_ref().is_some_and(|x| { + }) + // output is .dump + || opts.output_path.as_ref().is_some_and(|x| { x.extension().map_or(false, |y| y == CIDER_EXTENSION) })) { opts.action = Some(Target::DataDump); } else if opts.action.is_none() + // output is .json && (opts.output_path.as_ref().is_some_and(|x| { x.extension().map_or(false, |x| x == JSON_EXTENSION) - }) || opts.input_path.as_ref().is_some_and(|x| { + }) + // input is .dump + || opts.input_path.as_ref().is_some_and(|x| { x.extension().map_or(false, |x| x == CIDER_EXTENSION) - })) + }) + // input is a directory (suggesting a deserialization from dat) + || opts.input_path.as_ref().is_some_and(|x| x.is_dir())) { opts.action = Some(Target::Json); } @@ -144,30 +169,31 @@ fn main() -> Result<(), CiderDataConverterError> { for mem_dec in &header.memories { let starting_len = data.len(); let mem_file = BufReader::new(File::open( - path.join(&mem_dec.name), + path.join(format!( + "{}.{}", + mem_dec.name, opts.file_extension + )), )?); - let mut line_data = vec![]; for line in mem_file.lines() { let line = line?; - for pair in &line.chars().chunks(2) { - // there has got to be a better way to do this... - let string = - pair.into_iter().collect::(); - let val = u8::from_str_radix(&string, 16) - .expect("invalid hex"); - line_data.push(val); + if let Some(line_data) = + unwrap_line_or_comment(&line) + { + assert!( + line_data.len() + <= mem_dec.bytes_per_entry() + as usize, + "line data too long" + ); + + let padding = (mem_dec.bytes_per_entry() + as usize) + - line_data.len(); + + data.extend(line_data.into_iter().rev()); + data.extend(repeat(0u8).take(padding)) } - // TODO griffin: handle inputs that are - // truncated or otherwise shorter than expected - - assert!( - line_data.len() - == (mem_dec.bytes_per_entry() as usize) - ); - // reverse the byte order to get the expected - // little endian and reuse the vec - data.extend(line_data.drain(..).rev()) } assert_eq!( @@ -213,17 +239,22 @@ fn main() -> Result<(), CiderDataConverterError> { if let Some(path) = opts.output_path { if path.exists() && !path.is_dir() { - // TODO griffin: Make this an actual error - panic!("Output path exists but is not a directory") + return Err( + CiderDataConverterError::DatOutputPathIsFile, + ); } else if !path.exists() { std::fs::create_dir(&path)?; } - let mut header_output = File::create(path.join("header"))?; + let mut header_output = + File::create(path.join(HEADER_FILENAME))?; header_output.write_all(&data.header.serialize()?)?; for memory in &data.header.memories { - let file = File::create(path.join(&memory.name))?; + let file = File::create(path.join(format!( + "{}.{}", + memory.name, opts.file_extension + )))?; let mut writer = BufWriter::new(file); for bytes in data .get_data(&memory.name) @@ -243,8 +274,7 @@ fn main() -> Result<(), CiderDataConverterError> { } } } else { - // TODO griffin: Make this an actual error - panic!("Output path not specified, this is required for the dat target") + return Err(CiderDataConverterError::MissingDatOutputPath); } } }