diff --git a/README.md b/README.md index 51e3209..55c8357 100644 --- a/README.md +++ b/README.md @@ -80,10 +80,10 @@ with the following options and the estimated elapsed time. | Header | Corruption Check | Body | Output JSON | Elapsed | Throughput | | - | - | - | - | - | - | -| ✔ | | | | 68.0 µs | | -| ✔ | ✔ | ✔ | | 6.6 ms | 223 MiB/s | -| ✔ | | ✔ | | 6.3 ms | 232 MiB/s | -| ✔ | ✔ | ✔ | ✔ | 35 ms | 531 MiB/s ^1 | +| ✔ | | | | 32 µs | | +| ✔ | ✔ | ✔ | | 5.1 ms | 290 MiB/s | +| ✔ | | ✔ | | 4.8 ms | 315 MiB/s | +| ✔ | ✔ | ✔ | ✔ | 31 ms | 600 MiB/s ^1 | ^1: JSON serialization throughput includes the amount of JSON produced diff --git a/src/errors.rs b/src/errors.rs index 44c2dcd..6b6205d 100644 --- a/src/errors.rs +++ b/src/errors.rs @@ -1,5 +1,6 @@ use crate::data::ATTRIBUTES; use crate::network::{ActorId, Frame, NewActor, ObjectId, StreamId, UpdatedAttribute}; +use crate::{AttributeTag, CacheInfo}; use fnv::FnvHashMap; use std::error::Error; use std::fmt; @@ -105,12 +106,12 @@ impl Display for AttributeError { #[derive(PartialEq, Debug, Clone)] pub struct FrameContext { - pub objects: Vec<String>, - pub object_attributes: FnvHashMap<ObjectId, FnvHashMap<StreamId, ObjectId>>, - pub frames: Vec<Frame>, - pub actors: FnvHashMap<ActorId, ObjectId>, - pub new_actors: Vec<NewActor>, - pub updated_actors: Vec<UpdatedAttribute>, + pub(crate) objects: Vec<String>, + pub(crate) object_attributes: Vec<Option<CacheInfo>>, + pub(crate) frames: Vec<Frame>, + pub(crate) actors: FnvHashMap<ActorId, ObjectId>, + pub(crate) new_actors: Vec<NewActor>, + pub(crate) updated_actors: Vec<UpdatedAttribute>, } impl FrameContext { @@ -124,55 +125,48 @@ impl FrameContext { } fn display_new_actor(&self, f: &mut fmt::Formatter<'_>, actor: &NewActor) -> fmt::Result { - write!( + writeln!(f)?; + writeln!(f, "Last new actor:")?; + writeln!(f, "{}", "-".repeat("Last new actor:".len()))?; + writeln!(f, "actor id: {}", actor.actor_id)?; + writeln!( f, - "(id: {}, nameId: {}, objId: {}, objName: {}, initial trajectory: {:?})", - actor.actor_id, + "name id: {}", actor .name_id .map(|x| x.to_string()) - .unwrap_or_else(|| String::from("<none>")), - actor.object_id, - self.object_ind_to_string(actor.object_id), - actor.initial_trajectory - ) - } - - fn display_update(&self, f: &mut fmt::Formatter<'_>, attr: &UpdatedAttribute) -> fmt::Result { - let actor_entry = self.actors.get(&attr.actor_id); - let actor_obj_name = actor_entry.and_then(|x| self.objects.get(usize::from(*x))); - let stream_obj_name = self.objects.get(usize::from(attr.object_id)); - - write!( + .unwrap_or_else(|| String::from("<none>")) + )?; + writeln!(f, "object id: {}", actor.object_id)?; + writeln!( f, - "(actor stream id / object id / name: {} / ", - attr.actor_id + "object name: {}", + self.object_ind_to_string(actor.object_id) )?; - if let Some(actor_id) = actor_entry { - write!(f, "{} / ", actor_id) - } else { - write!(f, "<none> / ") - }?; + writeln!(f, "location: {:?}", actor.initial_trajectory.location)?; + writeln!(f, "rotation: {:?}", actor.initial_trajectory.rotation) + } - if let Some(name) = actor_obj_name { - write!(f, "{}, ", name) + fn display_update(&self, f: &mut fmt::Formatter<'_>, attr: &UpdatedAttribute) -> fmt::Result { + writeln!(f)?; + writeln!(f, "Last actor update:")?; + writeln!(f, "{}", "-".repeat("Last actor update:".len()))?; + + writeln!(f, "actor id: {}", attr.actor_id)?; + if let Some(object_id) = self.actors.get(&attr.actor_id) { + writeln!(f, "object id: {}", object_id)?; + writeln!(f, "object name: {}", self.object_ind_to_string(*object_id))?; } else { - write!(f, "<none>, ") - }?; - - write!( + writeln!(f, "object id: <none>")?; + }; + writeln!(f, "attribute stream id: {}", attr.stream_id)?; + writeln!(f, "attribute object id: {}", attr.object_id)?; + writeln!( f, - "attribute stream id / object id / name: {} / {} / ", - attr.stream_id, attr.object_id + "attribute object name: {}", + self.object_ind_to_string(attr.object_id) )?; - - if let Some(name) = stream_obj_name { - write!(f, "{}", name) - } else { - write!(f, "<none>") - }?; - - write!(f, ", attribute: {:?})", attr.attribute) + writeln!(f, "attribute: {:?}", attr.attribute) } fn most_recent_frame_with_data(&self) -> Option<(usize, &Frame)> { @@ -187,32 +181,37 @@ impl FrameContext { impl fmt::Display for FrameContext { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { let last_frame = self.frames.last(); - write!( + writeln!( f, - "on frame: {} (time: {} delta: {}), ", + "Current frame: {} (time: {} delta: {})", self.frames.len(), last_frame.map(|x| x.time).unwrap_or_default(), last_frame.map(|x| x.delta).unwrap_or_default() )?; if let Some(updated) = self.updated_actors.last() { - write!(f, "last updated actor: ")?; - self.display_update(f, updated) + return self.display_update(f, updated); } else if let Some(new) = self.new_actors.last() { - write!(f, "last new actor: ")?; + return self.display_new_actor(f, new); + } + + writeln!(f, "- No actor information decoded")?; + + let Some((frame_idx, frame)) = self.most_recent_frame_with_data() else { + return Ok(()); + }; + + writeln!( + f, + "Retrace frame: {} (time: {}, delta: {})", + frame_idx, frame.time, frame.delta + )?; + + if let Some(updated) = frame.updated_actors.last() { + self.display_update(f, updated) + } else if let Some(new) = frame.new_actors.last() { self.display_new_actor(f, new) - } else if let Some((frame_idx, frame)) = self.most_recent_frame_with_data() { - write!(f, "backtracking to frame {}, ", frame_idx)?; - if let Some(updated) = frame.updated_actors.last() { - write!(f, "last updated actor: ")?; - self.display_update(f, updated) - } else if let Some(new) = frame.new_actors.last() { - write!(f, "last new actor: ")?; - self.display_new_actor(f, new) - } else { - write!(f, "it didn't decode anything") - } } else { - write!(f, "it didn't decode anything") + writeln!(f, "- No actor information decoded") } } } @@ -278,35 +277,28 @@ impl FrameError { let objs_with_attr = context .object_attributes .iter() - .flat_map(|(obj_id, attrs)| { - attrs - .iter() - .map(move |(attr_id, attr_obj_id)| (obj_id, attr_obj_id, attr_id)) - }) - .filter(|&(_, _, stream_id)| stream_id == attribute_stream) + .filter_map(|x| x.as_ref()) + .filter_map(|c| c.attributes.get(*attribute_stream)) .collect::<Vec<_>>(); let obj = context .object_attributes - .get(actor_object) - .and_then(|x| x.get(attribute_stream)); - - if let Some(attr_obj_id) = obj { - if let Some(name) = context.objects.get(usize::from(*attr_obj_id)) { - if let Some(attr) = ATTRIBUTES.get(name.as_str()) { - write!( - f, - "found attribute {} ({:?}) on {} in network cache data. ", - name, attr, actor_obj_name - )?; - } else { - write!(f, "found attribute {} (unknown to boxcars) on {} in network cache data. This is likely due to a rocket league update or an atypical replay. File a bug report!", name, actor_obj_name)?; - - // No need for further context so we return early. - return Ok(()); - } + .get(usize::from(*actor_object)) + .and_then(|x| x.as_ref()) + .and_then(|x| x.attributes.get(*attribute_stream)); + + if let Some(attr_obj) = obj { + if attr_obj.attribute != AttributeTag::NotImplemented { + write!( + f, + "found attribute ({:?}) on {} in network cache data. ", + attr_obj.attribute, actor_obj_name + )?; } else { - write!(f, "found attribute on {} in network cache data, but not in replay object data, ", actor_obj_name)?; + writeln!(f, "attribute: {}", context.objects.get(usize::from(attr_obj.object_id)).map_or("<unknown>", |v| v))?; + + // No need for further context so we return early. + return Ok(()); } } else { write!( @@ -318,17 +310,7 @@ impl FrameError { let mut obj_attr_names = objs_with_attr .iter() - .filter_map(|&(obj_id, attr_obj_id, _)| { - context - .objects - .get(usize::from(*obj_id)) - .and_then(|obj_name| { - context - .objects - .get(usize::from(*attr_obj_id)) - .map(|attr_name| (obj_name, attr_name)) - }) - }) + .filter_map(|attr| context.objects.get(usize::from(attr.object_id))) .collect::<Vec<_>>(); obj_attr_names.sort(); @@ -336,7 +318,6 @@ impl FrameError { let mut unknown_attributes = obj_attr_names .iter() - .map(|(_obj_name, attr_name)| attr_name) .filter(|x| !ATTRIBUTES.contains_key(x.as_str())) .cloned() .cloned() @@ -347,7 +328,7 @@ impl FrameError { let stringify_names = obj_attr_names .iter() - .map(|(obj_name, attr_name)| format!("({}: {})", obj_name, attr_name)) + .map(|attr_name| format!("({})", attr_name)) .collect::<Vec<_>>(); write!(f, "searching all attributes with the same stream id, ")?; @@ -386,7 +367,13 @@ impl Display for FrameError { FrameError::ObjectIdOutOfRange {obj} => write!(f, "new actor object id out of range: {}", obj), FrameError::MissingActor {actor} => write!(f, "attribute update references unknown actor: {}", actor), FrameError::MissingCache {actor, actor_object} => write!(f, "no known attributes found for actor id / object id: {} / {}", actor, actor_object), - FrameError::MissingAttribute {actor, actor_object, attribute_stream} => write!(f, "attribute unknown or not implemented: actor id / actor object id / attribute id: {} / {} / {}", actor, actor_object, attribute_stream), + FrameError::MissingAttribute {actor, actor_object, attribute_stream} =>{ + writeln!(f, "attribute unknown or not implemented:")?; + writeln!(f, "{}", "-".repeat(10))?; + writeln!(f, "actor id: {}", actor)?; + writeln!(f, "actor object id: {}", actor_object)?; + writeln!(f, "attribute stream id: {}", attribute_stream) + }, FrameError::AttributeError {actor, actor_object, attribute_stream, error} => write!(f, "attribute decoding error encountered: {} for actor id / actor object id / attribute id: {} / {} / {}", error, actor, actor_object, attribute_stream), } } @@ -436,9 +423,14 @@ impl Display for NetworkError { ), NetworkError::TooManyFrames(size) => write!(f, "Too many frames to decode: {}", size), NetworkError::FrameError(err, context) => { - write!(f, "Error decoding frame: {}. ", err)?; + write!(f, "Error decoding frame: {}", err)?; + if !matches!(err, FrameError::MissingAttribute { .. }) { + write!(f, ". ")?; + } + err.contextualize(f, context)?; - write!(f, " Context: {}", context) + writeln!(f)?; + write!(f, "{}", context) } } } diff --git a/src/network/frame_decoder.rs b/src/network/frame_decoder.rs index cf80990..aab55ff 100644 --- a/src/network/frame_decoder.rs +++ b/src/network/frame_decoder.rs @@ -10,6 +10,110 @@ use crate::network::models::{ use crate::network::{CacheInfo, VersionTriplet}; use crate::parser::ReplayBody; +#[derive(Debug)] +pub(crate) struct RawSegmentedArray<T> { + array: Vec<Option<T>>, + map: FnvHashMap<usize, T>, +} + +impl<T> RawSegmentedArray<T> { + pub(crate) fn new(size: usize) -> Self { + let mut array = Vec::with_capacity(size); + array.resize_with(size, || None); + Self { + array, + map: FnvHashMap::default(), + } + } + + pub(crate) fn insert(&mut self, key: usize, value: T) { + match self.array.get_mut(key) { + Some(entry) => { + *entry = Some(value); + } + None => { + self.map.insert(key, value); + } + }; + } + + pub(crate) fn get(&self, key: usize) -> Option<&T> { + match self.array.get(key) { + Some(x) => x.as_ref(), + None => self.map.get(&key), + } + } + + pub(crate) fn delete(&mut self, key: usize) { + match self.array.get(key) { + Some(_) => {} // skip removing + None => { + self.map.remove(&key); + } + }; + } +} + +impl<T: Clone> Clone for RawSegmentedArray<T> { + fn clone(&self) -> Self { + Self { + array: self.array.clone(), + map: self.map.clone(), + } + } +} + +impl<T: PartialEq> PartialEq for RawSegmentedArray<T> { + fn eq(&self, other: &Self) -> bool { + self.array == other.array && self.map == other.map + } +} + +#[derive(Debug)] +pub(crate) struct SegmentedArray<K, V> { + raw: RawSegmentedArray<V>, + marker: std::marker::PhantomData<K>, +} + +impl<K, V: Clone> Clone for SegmentedArray<K, V> { + fn clone(&self) -> Self { + Self { + raw: self.raw.clone(), + marker: std::marker::PhantomData, + } + } +} + +impl<K, V: PartialEq> PartialEq for SegmentedArray<K, V> { + fn eq(&self, other: &Self) -> bool { + self.raw == other.raw + } +} + +impl<K, V> SegmentedArray<K, V> +where + K: Into<usize>, +{ + pub(crate) fn new(size: usize) -> Self { + Self { + raw: RawSegmentedArray::new(size), + marker: std::marker::PhantomData, + } + } + + pub(crate) fn insert(&mut self, key: K, value: V) { + self.raw.insert(key.into(), value); + } + + pub(crate) fn get(&self, key: K) -> Option<&V> { + self.raw.get(key.into()) + } + + pub(crate) fn delete(&mut self, key: K) { + self.raw.delete(key.into()); + } +} + pub(crate) struct FrameDecoder<'a, 'b: 'a> { pub frames_len: usize, pub product_decoder: ProductValueDecoder, @@ -17,7 +121,7 @@ pub(crate) struct FrameDecoder<'a, 'b: 'a> { pub channel_bits: u32, pub body: &'a ReplayBody<'b>, pub spawns: &'a Vec<SpawnTrajectory>, - pub object_ind_attributes: FnvHashMap<ObjectId, CacheInfo<'a>>, + pub object_ind_attributes: Vec<Option<CacheInfo>>, pub version: VersionTriplet, pub is_lan: bool, pub is_rl_223: bool, @@ -68,12 +172,12 @@ impl<'a, 'b> FrameDecoder<'a, 'b> { }) } - fn decode_frame( - &self, + fn decode_frame<'c>( + &'c self, attr_decoder: &AttributeDecoder, bits: &mut LittleEndianReader<'_>, buf: &mut [u8], - actors: &mut FnvHashMap<ActorId, ObjectId>, + actors: &mut SegmentedArray<ActorId, (ObjectId, &'c CacheInfo)>, new_actors: &mut Vec<NewActor>, deleted_actors: &mut Vec<ActorId>, updated_actors: &mut Vec<UpdatedAttribute>, @@ -123,24 +227,24 @@ impl<'a, 'b> FrameDecoder<'a, 'b> { // Insert the new actor so we can keep track of it for attribute // updates. It's common for an actor id to already exist, so we // overwrite it. - actors.insert(actor.actor_id, actor.object_id); + let cache_info = self + .object_ind_attributes + .get(usize::from(actor.object_id)) + .and_then(|x| x.as_ref()) + .ok_or(FrameError::MissingCache { + actor: actor_id, + actor_object: actor.object_id, + })?; + + actors.insert(actor.actor_id, (actor.object_id, cache_info)); new_actors.push(actor); } else { // We'll be updating an existing actor with some attributes so we need - // to track down what the actor's type is - let object_id = actors - .get(&actor_id) + // to track down what the actor's type is and what attributes are available + let (object_id, cache_info) = actors + .get(actor_id) .ok_or(FrameError::MissingActor { actor: actor_id })?; - // Once we have the type we need to look up what attributes are - // available for said type - let cache_info = self.object_ind_attributes.get(object_id).ok_or( - FrameError::MissingCache { - actor: actor_id, - actor_object: *object_id, - }, - )?; - // While there are more attributes to update for our actor: while bits .read_bit() @@ -164,7 +268,7 @@ impl<'a, 'b> FrameDecoder<'a, 'b> { // decoding function. Experience has told me replays that fail to // parse, fail to do so here, so a large chunk is dedicated to // generating an error message with context - let attr = cache_info.attributes.get(&stream_id).ok_or( + let attr = cache_info.attributes.get(stream_id).ok_or( FrameError::MissingAttribute { actor: actor_id, actor_object: *object_id, @@ -198,7 +302,7 @@ impl<'a, 'b> FrameDecoder<'a, 'b> { } } else { deleted_actors.push(actor_id); - actors.remove(&actor_id); + actors.delete(actor_id); } } @@ -219,7 +323,7 @@ impl<'a, 'b> FrameDecoder<'a, 'b> { }; let mut frames: Vec<Frame> = Vec::with_capacity(self.frames_len); - let mut actors = FnvHashMap::default(); + let mut actors = SegmentedArray::new(200); let mut bits = LittleEndianReader::new(self.body.network_data); let mut new_actors = Vec::new(); let mut updated_actors = Vec::new(); @@ -242,22 +346,25 @@ impl<'a, 'b> FrameDecoder<'a, 'b> { e, Box::new(FrameContext { objects: self.body.objects.clone(), - object_attributes: self - .object_ind_attributes + object_attributes: self.object_ind_attributes.clone(), + frames: frames.clone(), + actors: actors + .raw + .array .iter() - .map(|(key, value)| { - ( - *key, - value - .attributes - .iter() - .map(|(key2, value)| (*key2, value.object_id)) - .collect(), - ) + .enumerate() + .filter_map(|(i, x)| { + let (obj_id, _) = x.as_ref()?; + Some((ActorId(i as i32), *obj_id)) }) + .chain( + actors + .raw + .map + .iter() + .map(|(k, (o, _))| (ActorId(*k as i32), *o)), + ) .collect(), - frames: frames.clone(), - actors: actors.clone(), new_actors: new_actors.clone(), updated_actors: updated_actors.clone(), }), diff --git a/src/network/mod.rs b/src/network/mod.rs index 580af9c..290cb9b 100644 --- a/src/network/mod.rs +++ b/src/network/mod.rs @@ -14,19 +14,20 @@ use crate::models::*; use crate::network::frame_decoder::FrameDecoder; use crate::parser::ReplayBody; use fnv::FnvHashMap; +use frame_decoder::SegmentedArray; use std::cmp; -#[derive(Debug)] -pub(crate) struct CacheInfo<'a> { - max_prop_id: u32, - prop_id_bits: u32, - attributes: &'a FnvHashMap<StreamId, ObjectAttribute>, +#[derive(PartialEq, Debug, Clone)] +pub(crate) struct CacheInfo { + pub(crate) max_prop_id: u32, + pub(crate) prop_id_bits: u32, + pub(crate) attributes: SegmentedArray<StreamId, ObjectAttribute>, } #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub(crate) struct ObjectAttribute { - attribute: AttributeTag, - object_id: ObjectId, + pub(crate) attribute: AttributeTag, + pub(crate) object_id: ObjectId, } #[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] @@ -134,28 +135,37 @@ pub(crate) fn parse(header: &Header, body: &ReplayBody) -> Result<NetworkFrames, ); } - let object_ind_attributes: FnvHashMap<ObjectId, CacheInfo> = object_ind_attrs - .iter() - .map(|(obj_id, attrs)| { - let id = *obj_id; - let max = attrs - .keys() - .map(|&x| i32::from(x)) - .max() - .unwrap_or(2) - .saturating_add(1); - - let max_bit_width = crate::bits::bit_width(max as u64); - Ok(( - id, - CacheInfo { - max_prop_id: max as u32, - prop_id_bits: cmp::max(max_bit_width, 1) - 1, - attributes: attrs, - }, - )) - }) - .collect::<Result<FnvHashMap<_, _>, NetworkError>>()?; + let mut object_ind_attributes: Vec<Option<CacheInfo>> = Vec::with_capacity(body.objects.len()); + object_ind_attributes.resize_with(body.objects.len(), || None); + + let iter = object_ind_attrs.into_iter().map(|(obj_id, attrs)| { + let id = obj_id; + let max = attrs + .keys() + .map(|&x| i32::from(x)) + .max() + .unwrap_or(2) + .saturating_add(1); + let mut attributes = SegmentedArray::new(64); + for (k, v) in attrs { + attributes.insert(k, v); + } + + let max_bit_width = crate::bits::bit_width(max as u64); + Ok(( + id, + CacheInfo { + max_prop_id: max as u32, + prop_id_bits: cmp::max(max_bit_width, 1) - 1, + attributes, + }, + )) + }); + + for x in iter { + let (object, cache) = x?; + object_ind_attributes[object.0 as usize] = Some(cache); + } let product_decoder = ProductValueDecoder::create(version, &object_index); diff --git a/src/network/models.rs b/src/network/models.rs index 964486c..f2ae35c 100644 --- a/src/network/models.rs +++ b/src/network/models.rs @@ -285,6 +285,12 @@ impl From<StreamId> for i32 { } } +impl From<StreamId> for usize { + fn from(val: StreamId) -> Self { + val.0 as usize + } +} + impl fmt::Display for StreamId { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, "{}", self.0) @@ -302,6 +308,12 @@ impl From<ActorId> for i32 { } } +impl From<ActorId> for usize { + fn from(val: ActorId) -> Self { + val.0 as usize + } +} + impl fmt::Display for ActorId { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { write!(f, "{}", self.0)