From 9bb26f913da37e02e6bbc1b635b7a4dc1844bd57 Mon Sep 17 00:00:00 2001 From: bskrlj Date: Fri, 6 Jan 2023 11:28:27 +0100 Subject: [PATCH 1/2] cargo fmt defaults --- src/block_ffm.rs | 778 +++++++++------ src/block_helpers.rs | 204 ++-- src/block_loss_functions.rs | 183 ++-- src/block_lr.rs | 189 ++-- src/block_misc.rs | 951 ++++++++++-------- src/block_neural.rs | 805 +++++++++------- src/block_normalize.rs | 252 +++-- src/block_relu.rs | 146 ++- src/cache.rs | 139 +-- src/cmdline.rs | 8 +- src/consts.rs | 9 +- src/feature_buffer.rs | 649 +++++++++---- src/feature_transform_executor.rs | 260 +++-- src/feature_transform_implementations.rs | 1004 +++++++++++-------- src/feature_transform_parser.rs | 459 +++++---- src/graph.rs | 435 +++++---- src/lib.rs | 44 +- src/main.rs | 29 +- src/model_instance.rs | 538 ++++++----- src/multithread_helpers.rs | 42 +- src/optimizer.rs | 120 +-- src/parser.rs | 1118 ++++++++++++++-------- src/persistence.rs | 76 +- src/port_buffer.rs | 10 +- src/regressor.rs | 613 ++++++++---- src/serving.rs | 378 ++++---- src/vwmap.rs | 144 +-- 27 files changed, 5765 insertions(+), 3818 deletions(-) diff --git a/src/block_ffm.rs b/src/block_ffm.rs index a50a2d0d..509cc9b6 100644 --- a/src/block_ffm.rs +++ b/src/block_ffm.rs @@ -1,34 +1,31 @@ -use std::any::Any; -use std::io; -use merand48::*; use core::arch::x86_64::*; +use merand48::*; +use std::any::Any; use std::error::Error; -use std::mem::{self, MaybeUninit}; use std::f32::consts::PI; +use std::io; +use std::mem::{self, MaybeUninit}; -use crate::optimizer; -use crate::regressor; -use crate::model_instance; -use crate::feature_buffer; -use crate::port_buffer; -use crate::consts; use crate::block_helpers; +use crate::consts; +use crate::feature_buffer; use crate::graph; use crate::graph::BlockGraph; +use crate::model_instance; +use crate::optimizer; +use crate::port_buffer; +use crate::regressor; +use block_helpers::WeightAndOptimizerData; use optimizer::OptimizerTrait; use regressor::BlockTrait; -use block_helpers::{WeightAndOptimizerData}; - -const FFM_STACK_BUF_LEN:usize= 32768; -const FFM_CONTRA_BUF_LEN:usize = 16384; +const FFM_STACK_BUF_LEN: usize = 32768; +const FFM_CONTRA_BUF_LEN: usize = 16384; +const SQRT_OF_ONE_HALF: f32 = 0.70710678118; -const SQRT_OF_ONE_HALF:f32 = 0.70710678118; - - -pub struct BlockFFM { +pub struct BlockFFM { pub optimizer_ffm: L, pub local_data_ffm_values: Vec, pub ffm_k: u32, @@ -39,24 +36,22 @@ pub struct BlockFFM { pub output_offset: usize, } - macro_rules! specialize_1f32 { ( $input_expr:expr, $output_const:ident, $code_block:block ) => { - if $input_expr == 1.0 { - const $output_const:f32 = 1.0; - $code_block - } else { - let $output_const:f32 = $input_expr; - $code_block - } - }; + if $input_expr == 1.0 { + const $output_const: f32 = 1.0; + $code_block + } else { + let $output_const: f32 = $input_expr; + $code_block + } + }; } - macro_rules! specialize_k { - ( $input_expr: expr, + ( $input_expr: expr, $output_const: ident, $wsumbuf: ident, $code_block: block ) => { @@ -70,7 +65,6 @@ macro_rules! specialize_k { }; } - impl BlockFFM { fn set_weights(&mut self, lower_bound: f32, difference: f32) { for i in 0..self.ffm_weights_len { @@ -82,24 +76,29 @@ impl BlockFFM { } pub fn new_ffm_block( - bg: &mut graph::BlockGraph, - mi: &model_instance::ModelInstance) - -> Result> { - + bg: &mut graph::BlockGraph, + mi: &model_instance::ModelInstance, +) -> Result> { let block = match mi.optimizer { - model_instance::Optimizer::AdagradLUT => new_ffm_block_without_weights::(&mi), - model_instance::Optimizer::AdagradFlex => new_ffm_block_without_weights::(&mi), - model_instance::Optimizer::SGD => new_ffm_block_without_weights::(&mi) - }.unwrap(); + model_instance::Optimizer::AdagradLUT => { + new_ffm_block_without_weights::(&mi) + } + model_instance::Optimizer::AdagradFlex => { + new_ffm_block_without_weights::(&mi) + } + model_instance::Optimizer::SGD => { + new_ffm_block_without_weights::(&mi) + } + } + .unwrap(); let mut block_outputs = bg.add_node(block, vec![]).unwrap(); assert_eq!(block_outputs.len(), 1); Ok(block_outputs.pop().unwrap()) } - - -fn new_ffm_block_without_weights(mi: &model_instance::ModelInstance) -> Result, Box> { - +fn new_ffm_block_without_weights( + mi: &model_instance::ModelInstance, +) -> Result, Box> { let ffm_num_fields = mi.ffm_fields.len() as u32; let mut reg_ffm = BlockFFM:: { weights: Vec::new(), @@ -113,10 +112,14 @@ fn new_ffm_block_without_weights(mi: &model_instance }; if mi.ffm_k > 0 { - - reg_ffm.optimizer_ffm.init(mi.ffm_learning_rate, mi.ffm_power_t, mi.ffm_init_acc_gradient); + reg_ffm.optimizer_ffm.init( + mi.ffm_learning_rate, + mi.ffm_power_t, + mi.ffm_init_acc_gradient, + ); // At the end we add "spillover buffer", so we can do modulo only on the base address and add offset - reg_ffm.ffm_weights_len = (1 << mi.ffm_bit_precision) + (mi.ffm_fields.len() as u32 * reg_ffm.ffm_k); + reg_ffm.ffm_weights_len = + (1 << mi.ffm_bit_precision) + (mi.ffm_fields.len() as u32 * reg_ffm.ffm_k); } // Verify that forward pass will have enough stack for temporary buffer @@ -128,119 +131,122 @@ fn new_ffm_block_without_weights(mi: &model_instance Ok(Box::new(reg_ffm)) } - - - - - - -impl BlockTrait for BlockFFM { +impl BlockTrait for BlockFFM { fn as_any(&mut self) -> &mut dyn Any { self } - fn allocate_and_init_weights(&mut self, mi: &model_instance::ModelInstance) { - self.weights =vec![WeightAndOptimizerData::{weight:0.0, optimizer_data: self.optimizer_ffm.initial_data()}; self.ffm_weights_len as usize]; - - match mi.ffm_initialization_type.as_str() { - "default" => { - if mi.ffm_k > 0 { - if mi.ffm_init_width == 0.0 { - // Initialization that has showed to work ok for us, like in ffm.pdf, but centered around zero and further divided by 50 - let ffm_one_over_k_root = 1.0 / (self.ffm_k as f32).sqrt() / 50.0; - for i in 0..self.ffm_weights_len { - self.weights[i as usize].weight = (1.0 * merand48((self.ffm_weights_len as usize+ i as usize) as u64)-0.5) * ffm_one_over_k_root; - self.weights[i as usize].optimizer_data = self.optimizer_ffm.initial_data(); - } - } else { - let zero_half_band_width = mi.ffm_init_width * mi.ffm_init_zero_band * 0.5; - let band_width = mi.ffm_init_width * (1.0 - mi.ffm_init_zero_band); - for i in 0..self.ffm_weights_len { - let mut w = merand48(i as u64) * band_width - band_width * 0.5; - if w > 0.0 { - w += zero_half_band_width ; - } else { - w -= zero_half_band_width; - } - w += mi.ffm_init_center; - self.weights[i as usize].weight = w; - self.weights[i as usize].optimizer_data = self.optimizer_ffm.initial_data(); - } - - } - } - }, - "xavier_custom_mask" => { - // MASK[U [-(1/sqrt(n)), 1/sqrt(n)]] - let lower_bound: f32 = -1.0/(self.ffm_weights_len as f32).sqrt(); - let upper_bound: f32 = 1.0/(self.ffm_weights_len as f32).sqrt(); - let difference = upper_bound - lower_bound; - - for i in 0..self.ffm_weights_len { - let mut w = merand48(i as u64) as f32; - if w < (self.ffm_weights_len / 10) as f32 { - w = 0.001; - } else { - w = 0.0; - } - self.weights[i as usize].weight = w; - self.weights[i as usize].optimizer_data = self.optimizer_ffm.initial_data(); - } - }, - "xavier" => { - // U [-(1/sqrt(n)), 1/sqrt(n)] - let lower_bound: f32 = -1.0/(self.ffm_weights_len as f32).sqrt(); - let upper_bound: f32 = 1.0/(self.ffm_weights_len as f32).sqrt(); - let difference = upper_bound - lower_bound; - self.set_weights(lower_bound, difference); - - }, - "xavier_normalized" => { - // U [-(sqrt(6)/sqrt(n + m)), sqrt(6)/sqrt(n + m)] + we assume symmetric input-output - - let lower_bound: f32 = -6_f32.sqrt() / (2 * self.ffm_weights_len) as f32; - let upper_bound: f32 = 6_f32.sqrt() / (2 * self.ffm_weights_len) as f32; - let difference = upper_bound - lower_bound; - self.set_weights(lower_bound, difference); - - }, - "he" => { - // G (0.0, sqrt(2/n)) + Box Muller-ish transform - - for i in 0..self.ffm_weights_len { - - // could use both, but not critical in this case - let seed_var_first = merand48(i as u64); - let seed_var_second = merand48(u64::pow(i as u64, 2)); - let normal_var = (-2.0 * seed_var_first.ln()).sqrt() * (2.0 * PI as f32 * seed_var_second).cos(); - - self.weights[i as usize].weight = normal_var + (2.0 / self.ffm_weights_len as f32).sqrt(); - self.weights[i as usize].optimizer_data = self.optimizer_ffm.initial_data(); - } - - }, - "constant" => { - // generic constant initialization (sanity check) - - for i in 0..self.ffm_weights_len { - let mut w = 1.0; - self.weights[i as usize].weight = w; - self.weights[i as usize].optimizer_data = self.optimizer_ffm.initial_data(); - } - - }, - _ => {panic!("Please select a valid activation function.")} - } - } + self.weights = vec![ + WeightAndOptimizerData:: { + weight: 0.0, + optimizer_data: self.optimizer_ffm.initial_data() + }; + self.ffm_weights_len as usize + ]; + + match mi.ffm_initialization_type.as_str() { + "default" => { + if mi.ffm_k > 0 { + if mi.ffm_init_width == 0.0 { + // Initialization that has showed to work ok for us, like in ffm.pdf, but centered around zero and further divided by 50 + let ffm_one_over_k_root = 1.0 / (self.ffm_k as f32).sqrt() / 50.0; + for i in 0..self.ffm_weights_len { + self.weights[i as usize].weight = (1.0 + * merand48((self.ffm_weights_len as usize + i as usize) as u64) + - 0.5) + * ffm_one_over_k_root; + self.weights[i as usize].optimizer_data = + self.optimizer_ffm.initial_data(); + } + } else { + let zero_half_band_width = mi.ffm_init_width * mi.ffm_init_zero_band * 0.5; + let band_width = mi.ffm_init_width * (1.0 - mi.ffm_init_zero_band); + for i in 0..self.ffm_weights_len { + let mut w = merand48(i as u64) * band_width - band_width * 0.5; + if w > 0.0 { + w += zero_half_band_width; + } else { + w -= zero_half_band_width; + } + w += mi.ffm_init_center; + self.weights[i as usize].weight = w; + self.weights[i as usize].optimizer_data = + self.optimizer_ffm.initial_data(); + } + } + } + } + "xavier_custom_mask" => { + // MASK[U [-(1/sqrt(n)), 1/sqrt(n)]] + let lower_bound: f32 = -1.0 / (self.ffm_weights_len as f32).sqrt(); + let upper_bound: f32 = 1.0 / (self.ffm_weights_len as f32).sqrt(); + let difference = upper_bound - lower_bound; + + for i in 0..self.ffm_weights_len { + let mut w = merand48(i as u64) as f32; + if w < (self.ffm_weights_len / 10) as f32 { + w = 0.001; + } else { + w = 0.0; + } + self.weights[i as usize].weight = w; + self.weights[i as usize].optimizer_data = self.optimizer_ffm.initial_data(); + } + } + "xavier" => { + // U [-(1/sqrt(n)), 1/sqrt(n)] + let lower_bound: f32 = -1.0 / (self.ffm_weights_len as f32).sqrt(); + let upper_bound: f32 = 1.0 / (self.ffm_weights_len as f32).sqrt(); + let difference = upper_bound - lower_bound; + self.set_weights(lower_bound, difference); + } + "xavier_normalized" => { + // U [-(sqrt(6)/sqrt(n + m)), sqrt(6)/sqrt(n + m)] + we assume symmetric input-output + let lower_bound: f32 = -6_f32.sqrt() / (2 * self.ffm_weights_len) as f32; + let upper_bound: f32 = 6_f32.sqrt() / (2 * self.ffm_weights_len) as f32; + let difference = upper_bound - lower_bound; + self.set_weights(lower_bound, difference); + } + "he" => { + // G (0.0, sqrt(2/n)) + Box Muller-ish transform + + for i in 0..self.ffm_weights_len { + // could use both, but not critical in this case + let seed_var_first = merand48(i as u64); + let seed_var_second = merand48(u64::pow(i as u64, 2)); + let normal_var = (-2.0 * seed_var_first.ln()).sqrt() + * (2.0 * PI as f32 * seed_var_second).cos(); + + self.weights[i as usize].weight = + normal_var + (2.0 / self.ffm_weights_len as f32).sqrt(); + self.weights[i as usize].optimizer_data = self.optimizer_ffm.initial_data(); + } + } + "constant" => { + // generic constant initialization (sanity check) + + for i in 0..self.ffm_weights_len { + let mut w = 1.0; + self.weights[i as usize].weight = w; + self.weights[i as usize].optimizer_data = self.optimizer_ffm.initial_data(); + } + } + _ => { + panic!("Please select a valid activation function.") + } + } + } fn get_num_output_values(&self, output: graph::OutputSlot) -> usize { assert!(output.get_output_index() == 0); return (self.ffm_num_fields * self.ffm_num_fields) as usize; } - fn get_num_output_slots(&self) -> usize { 1 } + fn get_num_output_slots(&self) -> usize { + 1 + } fn set_input_offset(&mut self, input: graph::InputSlot, offset: usize) { panic!("You cannnot set_input_offset() for BlockFFM"); @@ -251,20 +257,20 @@ impl BlockTrait for BlockFFM { self.output_offset = offset; } - #[inline(always)] - fn forward_backward(&mut self, - further_blocks: &mut [Box], - fb: &feature_buffer::FeatureBuffer, - pb: &mut port_buffer::PortBuffer, - update:bool) { + fn forward_backward( + &mut self, + further_blocks: &mut [Box], + fb: &feature_buffer::FeatureBuffer, + pb: &mut port_buffer::PortBuffer, + update: bool, + ) { debug_assert!(self.output_offset != usize::MAX); let mut wsum = 0.0; let local_data_ffm_len = fb.ffm_buffer.len() * (self.ffm_k * fb.ffm_fields_count) as usize; unsafe { - macro_rules! core_macro { ( $local_data_ffm_values:ident @@ -276,17 +282,17 @@ impl BlockTrait for BlockFFM { let mut local_data_ffm_values = $local_data_ffm_values; // let mut local_data_ffm_values = &mut $local_data_ffm_values; - + let ffm_weights = &mut self.weights; let fc = (fb.ffm_fields_count * self.ffm_k) as usize; let mut contra_fields: [f32; FFM_CONTRA_BUF_LEN] = MaybeUninit::uninit().assume_init(); let field_embedding_len = self.field_embedding_len; specialize_k!(self.ffm_k, FFMK, wsumbuf, { /* first prepare two things: - - transposed contra vectors in contra_fields - + - transposed contra vectors in contra_fields - - for each vector we sum up all the features within a field - and at the same time transpose it, so we can later directly multiply them with individual feature embeddings - - cache of gradients in local_data_ffm_values + - cache of gradients in local_data_ffm_values - we will use these gradients later in backward pass */ @@ -304,7 +310,7 @@ impl BlockTrait for BlockFFM { *contra_fields.get_unchecked_mut(zfc + k) = 0.0; } zfc += fc; - } + } continue; } let mut feature_num = 0; @@ -313,7 +319,7 @@ impl BlockTrait for BlockFFM { let left_hash = fb.ffm_buffer.get_unchecked(ffm_buffer_index); let mut addr = left_hash.hash as usize; let mut zfc:usize = field_index_ffmk as usize; - + specialize_1f32!(left_hash.value, LEFT_HASH_VALUE, { if feature_num == 0 { for z in 0..fb.ffm_fields_count { @@ -339,7 +345,7 @@ impl BlockTrait for BlockFFM { feature_num += 1; } } - + let mut ffm_values_offset = 0; for (i, left_hash) in fb.ffm_buffer.iter().enumerate() { let contra_offset = (left_hash.contra_field_index * fb.ffm_fields_count) as usize; @@ -377,7 +383,7 @@ impl BlockTrait for BlockFFM { }); // End of macro specialize_1f32! for LEFT_HASH_VALUE } }); - + block_helpers::forward_backward(further_blocks, fb, pb, update); if update { @@ -410,41 +416,51 @@ impl BlockTrait for BlockFFM { return } } // End of macro - if local_data_ffm_len < FFM_STACK_BUF_LEN { // Fast-path - using on-stack data structures - let mut local_data_ffm_values: [f32; FFM_STACK_BUF_LEN as usize] = MaybeUninit::uninit().assume_init();//[0.0; FFM_STACK_BUF_LEN as usize]; + let mut local_data_ffm_values: [f32; FFM_STACK_BUF_LEN as usize] = + MaybeUninit::uninit().assume_init(); //[0.0; FFM_STACK_BUF_LEN as usize]; core_macro!(local_data_ffm_values); - } else { // Slow-path - using heap data structures if local_data_ffm_len > self.local_data_ffm_values.len() { - self.local_data_ffm_values.reserve(local_data_ffm_len - self.local_data_ffm_values.len() + 1024); + self.local_data_ffm_values + .reserve(local_data_ffm_len - self.local_data_ffm_values.len() + 1024); } let mut local_data_ffm_values = &mut self.local_data_ffm_values; - + core_macro!(local_data_ffm_values); - } + } } // unsafe end } - - fn forward(&self, further_blocks: &[Box], - fb: &feature_buffer::FeatureBuffer, - pb: &mut port_buffer::PortBuffer, - ) { + + fn forward( + &self, + further_blocks: &[Box], + fb: &feature_buffer::FeatureBuffer, + pb: &mut port_buffer::PortBuffer, + ) { debug_assert!(self.output_offset != usize::MAX); let num_outputs = (self.ffm_num_fields * self.ffm_num_fields) as usize; - let myslice = &mut pb.tape[self.output_offset .. (self.output_offset + num_outputs)]; + let myslice = &mut pb.tape[self.output_offset..(self.output_offset + num_outputs)]; myslice.fill(0.0); unsafe { let ffm_weights = &self.weights; if true { - _mm_prefetch(mem::transmute::<&f32, &i8>(&ffm_weights.get_unchecked(fb.ffm_buffer.get_unchecked(0).hash as usize).weight), _MM_HINT_T0); + _mm_prefetch( + mem::transmute::<&f32, &i8>( + &ffm_weights + .get_unchecked(fb.ffm_buffer.get_unchecked(0).hash as usize) + .weight, + ), + _MM_HINT_T0, + ); let field_embedding_len = self.field_embedding_len as usize; - let mut contra_fields: [f32; FFM_STACK_BUF_LEN] = MaybeUninit::uninit().assume_init(); + let mut contra_fields: [f32; FFM_STACK_BUF_LEN] = + MaybeUninit::uninit().assume_init(); specialize_k!(self.ffm_k, FFMK, wsumbuf, { /* We first prepare "contra_fields" or collapsed field embeddings, where we sum all individual feature embeddings @@ -453,24 +469,46 @@ impl BlockTrait for BlockFFM { - handle values on diagonal - we want to be able to exclude self-interactions later (we pre-substract from wsum) - optimize for just copying the embedding over when looking at first feature of the field, and add embeddings for the rest - optiize for very common case of value of the feature being 1.0 - avoid multiplications - - - */ - + - + */ + let mut ffm_buffer_index = 0; for field_index in 0..fb.ffm_fields_count { let field_index_ffmk = field_index * FFMK; let offset = (field_index_ffmk * fb.ffm_fields_count) as usize; // first we handle fields with no features - if ffm_buffer_index >= fb.ffm_buffer.len() || - fb.ffm_buffer.get_unchecked(ffm_buffer_index).contra_field_index > field_index_ffmk { - for z in 0..field_embedding_len as usize { // first time we see this field - just overwrite + if ffm_buffer_index >= fb.ffm_buffer.len() + || fb + .ffm_buffer + .get_unchecked(ffm_buffer_index) + .contra_field_index + > field_index_ffmk + { + for z in 0..field_embedding_len as usize { + // first time we see this field - just overwrite *contra_fields.get_unchecked_mut(offset + z) = 0.0; } continue; - } + } let mut feature_num = 0; - while ffm_buffer_index < fb.ffm_buffer.len() && fb.ffm_buffer.get_unchecked(ffm_buffer_index).contra_field_index == field_index_ffmk { - _mm_prefetch(mem::transmute::<&f32, &i8>(&ffm_weights.get_unchecked(fb.ffm_buffer.get_unchecked(ffm_buffer_index+1).hash as usize).weight), _MM_HINT_T0); + while ffm_buffer_index < fb.ffm_buffer.len() + && fb + .ffm_buffer + .get_unchecked(ffm_buffer_index) + .contra_field_index + == field_index_ffmk + { + _mm_prefetch( + mem::transmute::<&f32, &i8>( + &ffm_weights + .get_unchecked( + fb.ffm_buffer.get_unchecked(ffm_buffer_index + 1).hash + as usize, + ) + .weight, + ), + _MM_HINT_T0, + ); let left_hash = fb.ffm_buffer.get_unchecked(ffm_buffer_index); let left_hash_hash = left_hash.hash as usize; let left_hash_value = left_hash.value; @@ -478,25 +516,37 @@ impl BlockTrait for BlockFFM { let field_embedding_len2 = field_embedding_len / FFMK as usize; specialize_1f32!(left_hash_value, LEFT_HASH_VALUE, { if feature_num == 0 { - for z in 0..field_embedding_len { // first feature of the field - just overwrite - *contra_fields.get_unchecked_mut(offset + z) = ffm_weights.get_unchecked(left_hash_hash + z).weight * LEFT_HASH_VALUE; + for z in 0..field_embedding_len { + // first feature of the field - just overwrite + *contra_fields.get_unchecked_mut(offset + z) = + ffm_weights.get_unchecked(left_hash_hash + z).weight + * LEFT_HASH_VALUE; } } else { - for z in 0..field_embedding_len { // additional features of the field - addition - *contra_fields.get_unchecked_mut(offset + z) += ffm_weights.get_unchecked(left_hash_hash + z).weight * LEFT_HASH_VALUE; + for z in 0..field_embedding_len { + // additional features of the field - addition + *contra_fields.get_unchecked_mut(offset + z) += + ffm_weights.get_unchecked(left_hash_hash + z).weight + * LEFT_HASH_VALUE; } } - let vv = SQRT_OF_ONE_HALF * LEFT_HASH_VALUE; // To avoid one additional multiplication, we square root 0.5 into vv + let vv = SQRT_OF_ONE_HALF * LEFT_HASH_VALUE; // To avoid one additional multiplication, we square root 0.5 into vv for k in 0..FFMK as usize { - let ss = ffm_weights.get_unchecked(left_hash_hash + field_index_ffmk as usize + k).weight * vv; - myslice[(contra_offset2 * (fb.ffm_fields_count + 1)) as usize] -= ss * ss; + let ss = ffm_weights + .get_unchecked( + left_hash_hash + field_index_ffmk as usize + k, + ) + .weight + * vv; + myslice + [(contra_offset2 * (fb.ffm_fields_count + 1)) as usize] -= + ss * ss; } }); ffm_buffer_index += 1; feature_num += 1; } } - for f1 in 0..fb.ffm_fields_count as usize { let f1_offset = f1 * field_embedding_len as usize; @@ -505,33 +555,33 @@ impl BlockTrait for BlockFFM { let mut f2_offset_ffmk = f1_offset + f1_ffmk; let mut f1_offset_ffmk = f1_offset + f1_ffmk; // This is self-interaction - for k in 0..FFMK as usize{ + for k in 0..FFMK as usize { let v = contra_fields.get_unchecked(f1_offset_ffmk + k); myslice[f1_offset2 + f1] += v * v * 0.5; } - for f2 in f1+1..fb.ffm_fields_count as usize { + for f2 in f1 + 1..fb.ffm_fields_count as usize { f2_offset_ffmk += field_embedding_len as usize; f1_offset_ffmk += FFMK as usize; for k in 0..FFMK { - myslice[f1 * fb.ffm_fields_count as usize + f2] += - contra_fields.get_unchecked(f1_offset_ffmk + k as usize) * - contra_fields.get_unchecked(f2_offset_ffmk + k as usize) * 0.5; - myslice[f2 * fb.ffm_fields_count as usize + f1] += - contra_fields.get_unchecked(f1_offset_ffmk + k as usize) * - contra_fields.get_unchecked(f2_offset_ffmk + k as usize) * 0.5; - + myslice[f1 * fb.ffm_fields_count as usize + f2] += contra_fields + .get_unchecked(f1_offset_ffmk + k as usize) + * contra_fields.get_unchecked(f2_offset_ffmk + k as usize) + * 0.5; + myslice[f2 * fb.ffm_fields_count as usize + f1] += contra_fields + .get_unchecked(f1_offset_ffmk + k as usize) + * contra_fields.get_unchecked(f2_offset_ffmk + k as usize) + * 0.5; } } - } }); } else { // Old straight-forward method. As soon as we have multiple feature values per field, it is slower let ffm_weights = &self.weights; - specialize_k!(self.ffm_k, FFMK, wsumbuf, { + specialize_k!(self.ffm_k, FFMK, wsumbuf, { for (i, left_hash) in fb.ffm_buffer.iter().enumerate() { - for right_hash in fb.ffm_buffer.get_unchecked(i+1 ..).iter() { + for right_hash in fb.ffm_buffer.get_unchecked(i + 1..).iter() { //if left_hash.contra_field_index == right_hash.contra_field_index { // continue // not combining within a field //} @@ -539,75 +589,98 @@ impl BlockTrait for BlockFFM { let lindex = (left_hash.hash + right_hash.contra_field_index) as u32; let rindex = (right_hash.hash + left_hash.contra_field_index) as u32; for k in 0..FFMK { - let left_hash_weight = ffm_weights.get_unchecked((lindex+k) as usize).weight; - let right_hash_weight = ffm_weights.get_unchecked((rindex+k) as usize).weight; - *wsumbuf.get_unchecked_mut(k as usize) += left_hash_weight * right_hash_weight * joint_value; + let left_hash_weight = + ffm_weights.get_unchecked((lindex + k) as usize).weight; + let right_hash_weight = + ffm_weights.get_unchecked((rindex + k) as usize).weight; + *wsumbuf.get_unchecked_mut(k as usize) += + left_hash_weight * right_hash_weight * joint_value; } } - } }); } } block_helpers::forward(further_blocks, fb, pb); - } - + fn get_serialized_len(&self) -> usize { return self.ffm_weights_len as usize; } - fn read_weights_from_buf(&mut self, input_bufreader: &mut dyn io::Read) -> Result<(), Box> { + fn read_weights_from_buf( + &mut self, + input_bufreader: &mut dyn io::Read, + ) -> Result<(), Box> { block_helpers::read_weights_from_buf(&mut self.weights, input_bufreader) } - fn write_weights_to_buf(&self, output_bufwriter: &mut dyn io::Write) -> Result<(), Box> { + fn write_weights_to_buf( + &self, + output_bufwriter: &mut dyn io::Write, + ) -> Result<(), Box> { block_helpers::write_weights_to_buf(&self.weights, output_bufwriter) } - fn read_weights_from_buf_into_forward_only(&self, input_bufreader: &mut dyn io::Read, forward: &mut Box) -> Result<(), Box> { - let mut forward = forward.as_any().downcast_mut::>().unwrap(); - block_helpers::read_weights_only_from_buf2::(self.ffm_weights_len as usize, &mut forward.weights, input_bufreader) + fn read_weights_from_buf_into_forward_only( + &self, + input_bufreader: &mut dyn io::Read, + forward: &mut Box, + ) -> Result<(), Box> { + let mut forward = forward + .as_any() + .downcast_mut::>() + .unwrap(); + block_helpers::read_weights_only_from_buf2::( + self.ffm_weights_len as usize, + &mut forward.weights, + input_bufreader, + ) } /// Sets internal state of weights based on some completely object-dependent parameters - fn testing_set_weights(&mut self, aa: i32, bb: i32, index: usize, w: &[f32]) -> Result<(), Box> { + fn testing_set_weights( + &mut self, + aa: i32, + bb: i32, + index: usize, + w: &[f32], + ) -> Result<(), Box> { self.weights[index].weight = w[0]; self.weights[index].optimizer_data = self.optimizer_ffm.initial_data(); Ok(()) } } - - - mod tests { // Note this useful idiom: importing names from outer (for mod tests) scope. use super::*; use crate::block_loss_functions; - use crate::model_instance::Optimizer; use crate::feature_buffer; use crate::feature_buffer::HashAndValueAndSeq; + use crate::model_instance::Optimizer; use crate::vwmap; use block_helpers::{slearn2, spredict2}; use crate::assert_epsilon; - - fn ffm_vec(v:Vec, ffm_fields_count: u32) -> feature_buffer::FeatureBuffer { + fn ffm_vec( + v: Vec, + ffm_fields_count: u32, + ) -> feature_buffer::FeatureBuffer { feature_buffer::FeatureBuffer { - label: 0.0, - example_importance: 1.0, - example_number: 0, - lr_buffer: Vec::new(), - ffm_buffer: v, - ffm_fields_count: ffm_fields_count, + label: 0.0, + example_importance: 1.0, + example_number: 0, + lr_buffer: Vec::new(), + ffm_buffer: v, + ffm_fields_count: ffm_fields_count, } } - fn ffm_init(block_ffm: &mut Box) -> () { + fn ffm_init(block_ffm: &mut Box) -> () { let mut block_ffm = block_ffm.as_any().downcast_mut::>().unwrap(); - + for i in 0..block_ffm.weights.len() { block_ffm.weights[i].weight = 1.0; block_ffm.weights[i].optimizer_data = block_ffm.optimizer_ffm.initial_data(); @@ -616,7 +689,7 @@ mod tests { #[test] fn test_ffm_k1() { - let mut mi = model_instance::ModelInstance::new_empty().unwrap(); + let mut mi = model_instance::ModelInstance::new_empty().unwrap(); mi.learning_rate = 0.1; mi.ffm_learning_rate = 0.1; mi.power_t = 0.0; @@ -627,8 +700,6 @@ mod tests { mi.ffm_fields = vec![vec![], vec![]]; // This isn't really used mi.optimizer = Optimizer::AdagradLUT; - - // Nothing can be learned from a single field in FFMs let mut bg = BlockGraph::new(); let ffm_block = new_ffm_block(&mut bg, &mi).unwrap(); @@ -637,10 +708,16 @@ mod tests { bg.allocate_and_init_weights(&mi); let mut pb = bg.new_port_buffer(); - let fb = ffm_vec(vec![HashAndValueAndSeq{hash:1, value: 1.0, contra_field_index: 0}], - 1); // saying we have 1 field isn't entirely correct + let fb = ffm_vec( + vec![HashAndValueAndSeq { + hash: 1, + value: 1.0, + contra_field_index: 0, + }], + 1, + ); // saying we have 1 field isn't entirely correct assert_epsilon!(spredict2(&mut bg, &fb, &mut pb, true), 0.5); - assert_epsilon!(slearn2 (&mut bg, &fb, &mut pb, true), 0.5); + assert_epsilon!(slearn2(&mut bg, &fb, &mut pb, true), 0.5); // With two fields, things start to happen // Since fields depend on initial randomization, these tests are ... peculiar. @@ -654,15 +731,26 @@ mod tests { let mut pb = bg.new_port_buffer(); ffm_init::(&mut bg.blocks_final[0]); - let fb = ffm_vec(vec![ - HashAndValueAndSeq{hash:1, value: 1.0, contra_field_index: 0}, - HashAndValueAndSeq{hash:100, value: 1.0, contra_field_index: mi.ffm_k} - ], 2); + let fb = ffm_vec( + vec![ + HashAndValueAndSeq { + hash: 1, + value: 1.0, + contra_field_index: 0, + }, + HashAndValueAndSeq { + hash: 100, + value: 1.0, + contra_field_index: mi.ffm_k, + }, + ], + 2, + ); assert_epsilon!(spredict2(&mut bg, &fb, &mut pb, true), 0.7310586); - assert_eq!(slearn2 (&mut bg, &fb, &mut pb, true), 0.7310586); - - assert_epsilon!(spredict2(&mut bg, &fb, &mut pb,true), 0.7024794); - assert_eq!(slearn2 (&mut bg, &fb, &mut pb, true), 0.7024794); + assert_eq!(slearn2(&mut bg, &fb, &mut pb, true), 0.7310586); + + assert_epsilon!(spredict2(&mut bg, &fb, &mut pb, true), 0.7024794); + assert_eq!(slearn2(&mut bg, &fb, &mut pb, true), 0.7024794); // Two fields, use values mi.optimizer = Optimizer::AdagradLUT; @@ -673,20 +761,30 @@ mod tests { bg.allocate_and_init_weights(&mi); ffm_init::(&mut bg.blocks_final[0]); - let fb = ffm_vec(vec![ - HashAndValueAndSeq{hash:1, value: 2.0, contra_field_index: 0}, - HashAndValueAndSeq{hash:100, value: 2.0, contra_field_index: mi.ffm_k * 1} - ], 2); - assert_eq!(spredict2(&mut bg, &fb, &mut pb,true), 0.98201376); + let fb = ffm_vec( + vec![ + HashAndValueAndSeq { + hash: 1, + value: 2.0, + contra_field_index: 0, + }, + HashAndValueAndSeq { + hash: 100, + value: 2.0, + contra_field_index: mi.ffm_k * 1, + }, + ], + 2, + ); + assert_eq!(spredict2(&mut bg, &fb, &mut pb, true), 0.98201376); assert_eq!(slearn2(&mut bg, &fb, &mut pb, true), 0.98201376); - assert_eq!(spredict2(&mut bg, &fb, &mut pb,true), 0.81377685); + assert_eq!(spredict2(&mut bg, &fb, &mut pb, true), 0.81377685); assert_eq!(slearn2(&mut bg, &fb, &mut pb, true), 0.81377685); } - #[test] fn test_ffm_k4() { - let mut mi = model_instance::ModelInstance::new_empty().unwrap(); + let mut mi = model_instance::ModelInstance::new_empty().unwrap(); mi.learning_rate = 0.1; mi.ffm_learning_rate = 0.1; mi.power_t = 0.0; @@ -703,14 +801,19 @@ mod tests { bg.finalize(); bg.allocate_and_init_weights(&mi); - - let mut pb = bg.new_port_buffer(); - let fb = ffm_vec(vec![HashAndValueAndSeq{hash:1, value: 1.0, contra_field_index: 0}], 1); - assert_eq!(spredict2(&mut bg, &fb, &mut pb,true), 0.5); + let fb = ffm_vec( + vec![HashAndValueAndSeq { + hash: 1, + value: 1.0, + contra_field_index: 0, + }], + 1, + ); + assert_eq!(spredict2(&mut bg, &fb, &mut pb, true), 0.5); assert_eq!(slearn2(&mut bg, &fb, &mut pb, true), 0.5); - assert_eq!(spredict2(&mut bg, &fb, &mut pb,true), 0.5); + assert_eq!(spredict2(&mut bg, &fb, &mut pb, true), 0.5); assert_eq!(slearn2(&mut bg, &fb, &mut pb, true), 0.5); // With two fields, things start to happen @@ -723,14 +826,25 @@ mod tests { bg.allocate_and_init_weights(&mi); ffm_init::(&mut bg.blocks_final[0]); - let fb = ffm_vec(vec![ - HashAndValueAndSeq{hash:1, value: 1.0, contra_field_index: 0}, - HashAndValueAndSeq{hash:100, value: 1.0, contra_field_index: mi.ffm_k * 1} - ], 2); - assert_eq!(spredict2(&mut bg, &fb, &mut pb,true), 0.98201376); - assert_eq!(slearn2 (&mut bg, &fb, &mut pb, true), 0.98201376); - assert_eq!(spredict2(&mut bg, &fb, &mut pb,true), 0.96277946); - assert_eq!(slearn2 (&mut bg, &fb, &mut pb, true), 0.96277946); + let fb = ffm_vec( + vec![ + HashAndValueAndSeq { + hash: 1, + value: 1.0, + contra_field_index: 0, + }, + HashAndValueAndSeq { + hash: 100, + value: 1.0, + contra_field_index: mi.ffm_k * 1, + }, + ], + 2, + ); + assert_eq!(spredict2(&mut bg, &fb, &mut pb, true), 0.98201376); + assert_eq!(slearn2(&mut bg, &fb, &mut pb, true), 0.98201376); + assert_eq!(spredict2(&mut bg, &fb, &mut pb, true), 0.96277946); + assert_eq!(slearn2(&mut bg, &fb, &mut pb, true), 0.96277946); // Two fields, use values mi.optimizer = Optimizer::AdagradLUT; @@ -741,17 +855,27 @@ mod tests { bg.allocate_and_init_weights(&mi); ffm_init::(&mut bg.blocks_final[0]); - let fb = ffm_vec(vec![ - HashAndValueAndSeq{hash:1, value: 2.0, contra_field_index: 0}, - HashAndValueAndSeq{hash:100, value: 2.0, contra_field_index: mi.ffm_k * 1} - ], 2); - assert_eq!(spredict2(&mut bg, &fb, &mut pb,true), 0.9999999); + let fb = ffm_vec( + vec![ + HashAndValueAndSeq { + hash: 1, + value: 2.0, + contra_field_index: 0, + }, + HashAndValueAndSeq { + hash: 100, + value: 2.0, + contra_field_index: mi.ffm_k * 1, + }, + ], + 2, + ); + assert_eq!(spredict2(&mut bg, &fb, &mut pb, true), 0.9999999); assert_eq!(slearn2(&mut bg, &fb, &mut pb, true), 0.9999999); - assert_eq!(spredict2(&mut bg, &fb, &mut pb,true), 0.99685884); + assert_eq!(spredict2(&mut bg, &fb, &mut pb, true), 0.99685884); assert_eq!(slearn2(&mut bg, &fb, &mut pb, true), 0.99685884); } - #[test] fn test_ffm_multivalue() { let vw_map_string = r#" @@ -766,7 +890,7 @@ B,featureB mi.ffm_bit_precision = 18; mi.ffm_power_t = 0.0; mi.ffm_learning_rate = 0.1; - mi.ffm_fields = vec![vec![],vec![]]; + mi.ffm_fields = vec![vec![], vec![]]; mi.optimizer = Optimizer::AdagradLUT; let mut bg = BlockGraph::new(); @@ -775,22 +899,36 @@ B,featureB bg.finalize(); bg.allocate_and_init_weights(&mi); - let mut pb = bg.new_port_buffer(); let mut p: f32; ffm_init::(&mut bg.blocks_final[0]); - let fbuf = &ffm_vec(vec![ - HashAndValueAndSeq{hash:1, value: 1.0, contra_field_index: 0}, - HashAndValueAndSeq{hash:3 * 1000, value: 1.0, contra_field_index: 0}, - HashAndValueAndSeq{hash:100, value: 2.0, contra_field_index: mi.ffm_k * 1} - ], 2); - assert_epsilon!(spredict2(&mut bg, &fbuf, &mut pb,true), 0.9933072); + let fbuf = &ffm_vec( + vec![ + HashAndValueAndSeq { + hash: 1, + value: 1.0, + contra_field_index: 0, + }, + HashAndValueAndSeq { + hash: 3 * 1000, + value: 1.0, + contra_field_index: 0, + }, + HashAndValueAndSeq { + hash: 100, + value: 2.0, + contra_field_index: mi.ffm_k * 1, + }, + ], + 2, + ); + assert_epsilon!(spredict2(&mut bg, &fbuf, &mut pb, true), 0.9933072); assert_eq!(slearn2(&mut bg, &fbuf, &mut pb, true), 0.9933072); - assert_epsilon!(spredict2(&mut bg, &fbuf, &mut pb,false), 0.9395168); + assert_epsilon!(spredict2(&mut bg, &fbuf, &mut pb, false), 0.9395168); assert_eq!(slearn2(&mut bg, &fbuf, &mut pb, false), 0.9395168); - assert_epsilon!(spredict2(&mut bg, &fbuf, &mut pb,false), 0.9395168); + assert_epsilon!(spredict2(&mut bg, &fbuf, &mut pb, false), 0.9395168); assert_eq!(slearn2(&mut bg, &fbuf, &mut pb, false), 0.9395168); } @@ -804,7 +942,7 @@ B,featureB let mut mi = model_instance::ModelInstance::new_empty().unwrap(); mi.ffm_k = 4; mi.ffm_bit_precision = 18; - mi.ffm_fields = vec![vec![],vec![]]; + mi.ffm_fields = vec![vec![], vec![]]; mi.optimizer = Optimizer::AdagradLUT; let mut bg = BlockGraph::new(); @@ -816,15 +954,30 @@ B,featureB let mut pb = bg.new_port_buffer(); ffm_init::(&mut bg.blocks_final[0]); - let fbuf = &ffm_vec(vec![ - HashAndValueAndSeq{hash:1, value: 1.0, contra_field_index: 0}, - HashAndValueAndSeq{hash:3 * 1000, value: 1.0, contra_field_index: 0}, - HashAndValueAndSeq{hash:100, value: 2.0, contra_field_index: mi.ffm_k * 1} - ], 2); - - assert_eq!(spredict2(&mut bg, &fbuf,&mut pb, true), 1.0); + let fbuf = &ffm_vec( + vec![ + HashAndValueAndSeq { + hash: 1, + value: 1.0, + contra_field_index: 0, + }, + HashAndValueAndSeq { + hash: 3 * 1000, + value: 1.0, + contra_field_index: 0, + }, + HashAndValueAndSeq { + hash: 100, + value: 2.0, + contra_field_index: mi.ffm_k * 1, + }, + ], + 2, + ); + + assert_eq!(spredict2(&mut bg, &fbuf, &mut pb, true), 1.0); assert_eq!(slearn2(&mut bg, &fbuf, &mut pb, true), 1.0); - assert_eq!(spredict2(&mut bg, &fbuf,&mut pb, false), 0.9949837); + assert_eq!(spredict2(&mut bg, &fbuf, &mut pb, false), 0.9949837); assert_eq!(slearn2(&mut bg, &fbuf, &mut pb, false), 0.9949837); assert_eq!(slearn2(&mut bg, &fbuf, &mut pb, false), 0.9949837); } @@ -834,7 +987,7 @@ B,featureB // This test is useful to check if we don't by accient forget to initialize any of the collapsed // embeddings for the field, when field has no instances of a feature in it // We do by having three-field situation where only the middle field has features - let mut mi = model_instance::ModelInstance::new_empty().unwrap(); + let mut mi = model_instance::ModelInstance::new_empty().unwrap(); mi.learning_rate = 0.1; mi.ffm_learning_rate = 0.1; mi.power_t = 0.0; @@ -852,10 +1005,8 @@ B,featureB bg.finalize(); bg.allocate_and_init_weights(&mi); - let mut pb = bg.new_port_buffer(); - // With two fields, things start to happen // Since fields depend on initial randomization, these tests are ... peculiar. mi.optimizer = Optimizer::AdagradFlex; @@ -866,18 +1017,39 @@ B,featureB bg.allocate_and_init_weights(&mi); ffm_init::(&mut bg.blocks_final[0]); - let fb = ffm_vec(vec![ - HashAndValueAndSeq{hash:1, value: 1.0, contra_field_index: 0}, - HashAndValueAndSeq{hash:5, value: 1.0, contra_field_index: mi.ffm_k * 1}, - HashAndValueAndSeq{hash:100, value: 1.0, contra_field_index: mi.ffm_k * 2} - ], 3); + let fb = ffm_vec( + vec![ + HashAndValueAndSeq { + hash: 1, + value: 1.0, + contra_field_index: 0, + }, + HashAndValueAndSeq { + hash: 5, + value: 1.0, + contra_field_index: mi.ffm_k * 1, + }, + HashAndValueAndSeq { + hash: 100, + value: 1.0, + contra_field_index: mi.ffm_k * 2, + }, + ], + 3, + ); assert_epsilon!(spredict2(&mut bg, &fb, &mut pb, true), 0.95257413); - assert_eq!(slearn2 (&mut bg, &fb, &mut pb, false), 0.95257413); + assert_eq!(slearn2(&mut bg, &fb, &mut pb, false), 0.95257413); // here we intentionally have just the middle field - let fb = ffm_vec(vec![HashAndValueAndSeq{hash:5, value: 1.0, contra_field_index: mi.ffm_k * 1}], 3); - assert_eq!(spredict2(&mut bg, &fb,&mut pb,true), 0.5); - assert_eq!(slearn2 (&mut bg, &fb, &mut pb, true), 0.5); - + let fb = ffm_vec( + vec![HashAndValueAndSeq { + hash: 5, + value: 1.0, + contra_field_index: mi.ffm_k * 1, + }], + 3, + ); + assert_eq!(spredict2(&mut bg, &fb, &mut pb, true), 0.5); + assert_eq!(slearn2(&mut bg, &fb, &mut pb, true), 0.5); } } diff --git a/src/block_helpers.rs b/src/block_helpers.rs index f605f8d4..fb7a65cd 100644 --- a/src/block_helpers.rs +++ b/src/block_helpers.rs @@ -1,57 +1,61 @@ -use std::error::Error; use crate::optimizer::OptimizerTrait; +use std::error::Error; use std::io; -use std::io::{Read}; +use std::io::Read; -use std::slice; -use std::mem::{self}; -use std::cmp::min; -use crate::optimizer::OptimizerSGD; use crate::feature_buffer; -use crate::regressor::BlockTrait; -use crate::port_buffer; use crate::graph; +use crate::optimizer::OptimizerSGD; +use crate::port_buffer; +use crate::regressor::BlockTrait; +use std::cmp::min; +use std::mem::{self}; +use std::slice; #[derive(Clone, Debug)] #[repr(C)] pub struct Weight { - pub weight: f32, + pub weight: f32, } #[derive(Clone, Debug)] #[repr(C)] -pub struct OptimizerData { +pub struct OptimizerData { pub optimizer_data: L::PerWeightStore, } #[derive(Clone, Debug)] #[repr(C)] -pub struct WeightAndOptimizerData { - pub weight: f32, +pub struct WeightAndOptimizerData { + pub weight: f32, pub optimizer_data: L::PerWeightStore, } - #[macro_export] macro_rules! assert_epsilon { ($x:expr, $y:expr) => { - let x = $x; // Make sure we evaluate only once + let x = $x; // Make sure we evaluate only once let y = $y; - if !(x - y < 0.000005 && y - x < 0.000005) { println!("Expectation: {}, Got: {}", y, x); panic!(); } - } + if !(x - y < 0.000005 && y - x < 0.000005) { + println!("Expectation: {}, Got: {}", y, x); + panic!(); + } + }; } - - - // It's OK! I am a limo driver! -pub fn read_weights_from_buf(weights: &mut Vec, input_bufreader: &mut dyn io::Read) -> Result<(), Box> { +pub fn read_weights_from_buf( + weights: &mut Vec, + input_bufreader: &mut dyn io::Read, +) -> Result<(), Box> { if weights.len() == 0 { return Err(format!("Loading weights to unallocated weighs buffer"))?; } unsafe { - let mut buf_view:&mut [u8] = slice::from_raw_parts_mut(weights.as_mut_ptr() as *mut u8, - weights.len() *mem::size_of::()); + let mut buf_view: &mut [u8] = slice::from_raw_parts_mut( + weights.as_mut_ptr() as *mut u8, + weights.len() * mem::size_of::(), + ); input_bufreader.read_exact(&mut buf_view)?; } Ok(()) @@ -59,33 +63,44 @@ pub fn read_weights_from_buf(weights: &mut Vec, input_bufreader: &mut dyn // We get a vec here just so we easily know the type... // Skip amount of bytes that a weights vector would be -pub fn skip_weights_from_buf(weights_len: usize, weights: &Vec, input_bufreader: &mut dyn Read) -> Result<(), Box> { - let bytes_skip = weights_len *mem::size_of::(); - io::copy(&mut input_bufreader.take(bytes_skip as u64), &mut io::sink())?; +pub fn skip_weights_from_buf( + weights_len: usize, + weights: &Vec, + input_bufreader: &mut dyn Read, +) -> Result<(), Box> { + let bytes_skip = weights_len * mem::size_of::(); + io::copy( + &mut input_bufreader.take(bytes_skip as u64), + &mut io::sink(), + )?; Ok(()) } - - - - -pub fn write_weights_to_buf(weights: &Vec, output_bufwriter: &mut dyn io::Write) -> Result<(), Box> { +pub fn write_weights_to_buf( + weights: &Vec, + output_bufwriter: &mut dyn io::Write, +) -> Result<(), Box> { if weights.len() == 0 { assert!(false); return Err(format!("Writing weights of unallocated weights buffer"))?; } unsafe { - let buf_view:&[u8] = slice::from_raw_parts(weights.as_ptr() as *const u8, - weights.len() *mem::size_of::()); - output_bufwriter.write_all(buf_view)?; + let buf_view: &[u8] = slice::from_raw_parts( + weights.as_ptr() as *const u8, + weights.len() * mem::size_of::(), + ); + output_bufwriter.write_all(buf_view)?; } Ok(()) } - -pub fn read_weights_only_from_buf2(weights_len: usize, out_weights: &mut Vec>, input_bufreader: &mut dyn io::Read) -> Result<(), Box> { - const BUF_LEN:usize = 1024 * 1024; - let mut in_weights: Vec> = Vec::with_capacity(BUF_LEN as usize); +pub fn read_weights_only_from_buf2( + weights_len: usize, + out_weights: &mut Vec>, + input_bufreader: &mut dyn io::Read, +) -> Result<(), Box> { + const BUF_LEN: usize = 1024 * 1024; + let mut in_weights: Vec> = Vec::with_capacity(BUF_LEN as usize); let mut remaining_weights = weights_len; let mut out_idx: usize = 0; if weights_len != out_weights.len() { @@ -96,8 +111,10 @@ pub fn read_weights_only_from_buf2(weights_len: usize, out_wei while remaining_weights > 0 { let chunk_size = min(remaining_weights, BUF_LEN); in_weights.set_len(chunk_size); - let mut in_weights_view:&mut [u8] = slice::from_raw_parts_mut(in_weights.as_mut_ptr() as *mut u8, - chunk_size * mem::size_of::>()); + let mut in_weights_view: &mut [u8] = slice::from_raw_parts_mut( + in_weights.as_mut_ptr() as *mut u8, + chunk_size * mem::size_of::>(), + ); input_bufreader.read_exact(&mut in_weights_view)?; for w in &in_weights { out_weights.get_unchecked_mut(out_idx).weight = w.weight; @@ -105,75 +122,94 @@ pub fn read_weights_only_from_buf2(weights_len: usize, out_wei } remaining_weights -= chunk_size; } - } + } Ok(()) } - #[inline(always)] -pub fn get_input_output_borrows(i: &mut Vec, - start1: usize, len1: usize, - start2: usize, len2: usize) -> (&mut [f32], &mut [f32]) { - debug_assert!((start1 >= start2+len2) || (start2 >= start1+len1), "start1: {}, len1: {}, start2: {}, len2 {}", start1, len1, start2, len2); +pub fn get_input_output_borrows( + i: &mut Vec, + start1: usize, + len1: usize, + start2: usize, + len2: usize, +) -> (&mut [f32], &mut [f32]) { + debug_assert!( + (start1 >= start2 + len2) || (start2 >= start1 + len1), + "start1: {}, len1: {}, start2: {}, len2 {}", + start1, + len1, + start2, + len2 + ); unsafe { if start2 > start1 { let (rest, second) = i.split_at_mut(start2); let (rest, first) = rest.split_at_mut(start1); - return (first.get_unchecked_mut(0..len1), second.get_unchecked_mut(0..len2)) + return ( + first.get_unchecked_mut(0..len1), + second.get_unchecked_mut(0..len2), + ); } else { let (rest, first) = i.split_at_mut(start1); let (rest, second) = rest.split_at_mut(start2); - return (first.get_unchecked_mut(0..len1), second.get_unchecked_mut(0..len2)) - + return ( + first.get_unchecked_mut(0..len1), + second.get_unchecked_mut(0..len2), + ); } } - -} - - - -pub fn slearn2<'a>(bg: &mut graph::BlockGraph, - fb: &feature_buffer::FeatureBuffer, - pb: &mut port_buffer::PortBuffer, - update: bool) -> f32 { - - pb.reset(); - let (block_run, further_blocks) = bg.blocks_final.split_at_mut(1); - block_run[0].forward_backward(further_blocks, fb, pb, update); - let prediction_probability = pb.observations[0]; - return prediction_probability } -pub fn spredict2<'a>(bg: &mut graph::BlockGraph, - fb: &feature_buffer::FeatureBuffer, - pb: &mut port_buffer::PortBuffer, - update: bool) -> f32 { - - pb.reset(); - let (block_run, further_blocks) = bg.blocks_final.split_at(1); - block_run[0].forward(further_blocks, fb, pb); - let prediction_probability = pb.observations[0]; - return prediction_probability +pub fn slearn2<'a>( + bg: &mut graph::BlockGraph, + fb: &feature_buffer::FeatureBuffer, + pb: &mut port_buffer::PortBuffer, + update: bool, +) -> f32 { + pb.reset(); + let (block_run, further_blocks) = bg.blocks_final.split_at_mut(1); + block_run[0].forward_backward(further_blocks, fb, pb, update); + let prediction_probability = pb.observations[0]; + return prediction_probability; } +pub fn spredict2<'a>( + bg: &mut graph::BlockGraph, + fb: &feature_buffer::FeatureBuffer, + pb: &mut port_buffer::PortBuffer, + update: bool, +) -> f32 { + pb.reset(); + let (block_run, further_blocks) = bg.blocks_final.split_at(1); + block_run[0].forward(further_blocks, fb, pb); + let prediction_probability = pb.observations[0]; + return prediction_probability; +} #[inline(always)] -pub fn forward_backward(further_blocks: &mut [Box], - fb: &feature_buffer::FeatureBuffer, - pb: &mut port_buffer::PortBuffer, - update:bool) { +pub fn forward_backward( + further_blocks: &mut [Box], + fb: &feature_buffer::FeatureBuffer, + pb: &mut port_buffer::PortBuffer, + update: bool, +) { match further_blocks.split_first_mut() { - Some((next_regressor, further_blocks)) => next_regressor.forward_backward(further_blocks, fb, pb, update), - None => {}, + Some((next_regressor, further_blocks)) => { + next_regressor.forward_backward(further_blocks, fb, pb, update) + } + None => {} } } #[inline(always)] -pub fn forward( further_blocks: &[Box], - fb: &feature_buffer::FeatureBuffer, - pb: &mut port_buffer::PortBuffer) { +pub fn forward( + further_blocks: &[Box], + fb: &feature_buffer::FeatureBuffer, + pb: &mut port_buffer::PortBuffer, +) { match further_blocks.split_first() { Some((next_regressor, further_blocks)) => next_regressor.forward(further_blocks, fb, pb), - None => {}, + None => {} } -} \ No newline at end of file +} diff --git a/src/block_loss_functions.rs b/src/block_loss_functions.rs index 6b5a2f10..8c29dfc4 100644 --- a/src/block_loss_functions.rs +++ b/src/block_loss_functions.rs @@ -1,18 +1,17 @@ use std::any::Any; use std::error::Error; -use crate::regressor; +use crate::block_helpers; use crate::feature_buffer; -use crate::port_buffer; -use crate::model_instance; use crate::graph; -use crate::block_helpers; +use crate::model_instance; +use crate::port_buffer; +use crate::regressor; use regressor::BlockTrait; - //use fastapprox::fast::sigmoid; // surprisingly this doesn't work very well -/* We tested standard stable logistic function, but it gives slightly +/* We tested standard stable logistic function, but it gives slightly worse logloss results than plain logistic on our data */ /* #[inline(always)] @@ -28,95 +27,100 @@ pub fn stable_logistic(t: f32) -> f32 { #[inline(always)] pub fn logistic(t: f32) -> f32 { - return (1.0+(-t).exp()).recip(); + return (1.0 + (-t).exp()).recip(); } - - pub struct BlockSigmoid { num_inputs: usize, input_offset: usize, output_offset: usize, - copy_to_result: bool + copy_to_result: bool, } - -pub fn new_logloss_block( bg: &mut graph::BlockGraph, - input: graph::BlockPtrOutput, - copy_to_result: bool) - -> Result> { +pub fn new_logloss_block( + bg: &mut graph::BlockGraph, + input: graph::BlockPtrOutput, + copy_to_result: bool, +) -> Result> { let num_inputs = bg.get_num_output_values(vec![&input]); - let block = Box::new(BlockSigmoid {num_inputs: num_inputs as usize, - input_offset: usize::MAX, - output_offset: usize::MAX, - copy_to_result: copy_to_result}); + let block = Box::new(BlockSigmoid { + num_inputs: num_inputs as usize, + input_offset: usize::MAX, + output_offset: usize::MAX, + copy_to_result: copy_to_result, + }); let mut block_outputs = bg.add_node(block, vec![input]).unwrap(); assert_eq!(block_outputs.len(), 1); Ok(block_outputs.pop().unwrap()) - } - -pub fn new_without_weights(mi: &model_instance::ModelInstance, - num_inputs: u32, - copy_to_result: bool) -> Result, Box> { - Ok(Box::new(BlockSigmoid {num_inputs: num_inputs as usize, - input_offset: usize::MAX, - output_offset: usize::MAX, - copy_to_result: copy_to_result})) +pub fn new_without_weights( + mi: &model_instance::ModelInstance, + num_inputs: u32, + copy_to_result: bool, +) -> Result, Box> { + Ok(Box::new(BlockSigmoid { + num_inputs: num_inputs as usize, + input_offset: usize::MAX, + output_offset: usize::MAX, + copy_to_result: copy_to_result, + })) } - impl BlockTrait for BlockSigmoid { - fn as_any(&mut self) -> &mut dyn Any { self } - fn get_num_output_slots(&self) -> usize {1} - + fn get_num_output_slots(&self) -> usize { + 1 + } fn get_num_output_values(&self, output: graph::OutputSlot) -> usize { assert!(output.get_output_index() == 0); 1 } - - fn set_input_offset(&mut self, input: graph::InputSlot, offset: usize) { + + fn set_input_offset(&mut self, input: graph::InputSlot, offset: usize) { assert!(input.get_input_index() == 0); assert!(self.input_offset == usize::MAX); // We only allow a single call self.input_offset = offset; } - fn set_output_offset(&mut self, output: graph::OutputSlot, offset: usize) { + fn set_output_offset(&mut self, output: graph::OutputSlot, offset: usize) { assert!(self.output_offset == usize::MAX); // We only allow a single call assert!(output.get_output_index() == 0); self.output_offset = offset; } - #[inline(always)] - fn forward_backward(&mut self, - further_blocks: &mut [Box], - fb: &feature_buffer::FeatureBuffer, - pb: &mut port_buffer::PortBuffer, - update:bool) { - + fn forward_backward( + &mut self, + further_blocks: &mut [Box], + fb: &feature_buffer::FeatureBuffer, + pb: &mut port_buffer::PortBuffer, + update: bool, + ) { debug_assert!(self.input_offset != usize::MAX); debug_assert!(self.output_offset != usize::MAX); unsafe { - - let wsum:f32 = { - let myslice = &pb.tape.get_unchecked(self.input_offset .. (self.input_offset + self.num_inputs)); + let wsum: f32 = { + let myslice = &pb + .tape + .get_unchecked(self.input_offset..(self.input_offset + self.num_inputs)); myslice.iter().sum() }; // vowpal compatibility - + let mut prediction_probability: f32; let mut general_gradient: f32; - + if wsum.is_nan() { - eprintln!("NAN prediction in example {}, forcing 0.0", fb.example_number); + eprintln!( + "NAN prediction in example {}, forcing 0.0", + fb.example_number + ); prediction_probability = logistic(0.0); general_gradient = 0.0; } else if wsum < -50.0 { @@ -127,7 +131,7 @@ impl BlockTrait for BlockSigmoid { general_gradient = 0.0; } else { prediction_probability = logistic(wsum); - general_gradient = - (fb.label - prediction_probability) * fb.example_importance; + general_gradient = -(fb.label - prediction_probability) * fb.example_importance; } //println!("General gradient: {}", general_gradient); *pb.tape.get_unchecked_mut(self.output_offset) = prediction_probability; @@ -135,51 +139,52 @@ impl BlockTrait for BlockSigmoid { pb.observations.push(prediction_probability); } block_helpers::forward_backward(further_blocks, fb, pb, update); - // replace inputs with their gradients - pb.tape.get_unchecked_mut(self.input_offset .. (self.input_offset + self.num_inputs)).fill(general_gradient); + // replace inputs with their gradients + pb.tape + .get_unchecked_mut(self.input_offset..(self.input_offset + self.num_inputs)) + .fill(general_gradient); } } - fn forward(&self, - further_blocks: &[Box], - fb: &feature_buffer::FeatureBuffer, - pb: &mut port_buffer::PortBuffer, ) { + fn forward( + &self, + further_blocks: &[Box], + fb: &feature_buffer::FeatureBuffer, + pb: &mut port_buffer::PortBuffer, + ) { unsafe { + /* if further_blocks.len() != 0 { + panic!("RegSigmoid can only be at the end of the chain!"); + }*/ + debug_assert!(self.input_offset != usize::MAX); + debug_assert!(self.output_offset != usize::MAX); + let wsum: f32 = { + let myslice = &pb + .tape + .get_unchecked(self.input_offset..(self.input_offset + self.num_inputs)); + myslice.iter().sum() + }; -/* if further_blocks.len() != 0 { - panic!("RegSigmoid can only be at the end of the chain!"); - }*/ - debug_assert!(self.input_offset != usize::MAX); - debug_assert!(self.output_offset != usize::MAX); - let wsum:f32 = { - let myslice = &pb.tape.get_unchecked(self.input_offset .. (self.input_offset + self.num_inputs)); - myslice.iter().sum() - }; - - let prediction_probability:f32; - if wsum.is_nan() { - eprintln!("NAN prediction in example {}, forcing 0.0", fb.example_number); - prediction_probability = logistic(0.0); - } else if wsum < -50.0 { - prediction_probability = logistic(-50.0); - } else if wsum > 50.0 { - prediction_probability = logistic(50.0); - } else { - prediction_probability = logistic(wsum); - } - - pb.tape[self.output_offset] = prediction_probability; - if self.copy_to_result { - pb.observations.push(prediction_probability); + let prediction_probability: f32; + if wsum.is_nan() { + eprintln!( + "NAN prediction in example {}, forcing 0.0", + fb.example_number + ); + prediction_probability = logistic(0.0); + } else if wsum < -50.0 { + prediction_probability = logistic(-50.0); + } else if wsum > 50.0 { + prediction_probability = logistic(50.0); + } else { + prediction_probability = logistic(wsum); + } + + pb.tape[self.output_offset] = prediction_probability; + if self.copy_to_result { + pb.observations.push(prediction_probability); + } + block_helpers::forward(further_blocks, fb, pb); } - block_helpers::forward(further_blocks, fb, pb); - } } - } - - - - - - diff --git a/src/block_lr.rs b/src/block_lr.rs index f988d0b2..c8502c8b 100644 --- a/src/block_lr.rs +++ b/src/block_lr.rs @@ -1,22 +1,21 @@ use std::any::Any; -use crate::optimizer; -use crate::regressor; -use crate::model_instance; use crate::feature_buffer; use crate::graph; +use crate::model_instance; +use crate::optimizer; +use crate::regressor; -use std::io; use std::error::Error; +use std::io; -use optimizer::OptimizerTrait; -use regressor::BlockTrait; use crate::block_helpers; -use block_helpers::{WeightAndOptimizerData}; use crate::port_buffer; +use block_helpers::WeightAndOptimizerData; +use optimizer::OptimizerTrait; +use regressor::BlockTrait; - -pub struct BlockLR { +pub struct BlockLR { pub weights: Vec>, pub weights_len: u32, pub optimizer_lr: L, @@ -24,166 +23,206 @@ pub struct BlockLR { pub num_combos: u32, } -fn new_lr_block_without_weights(mi: &model_instance::ModelInstance) -> Result, Box> { +fn new_lr_block_without_weights( + mi: &model_instance::ModelInstance, +) -> Result, Box> { let mut num_combos = mi.feature_combo_descs.len() as u32; if mi.add_constant_feature { num_combos += 1; } let mut reg_lr = BlockLR:: { weights: Vec::new(), - weights_len: 0, + weights_len: 0, optimizer_lr: L::new(), - output_offset: usize::MAX, + output_offset: usize::MAX, num_combos: num_combos, - }; - reg_lr.optimizer_lr.init(mi.learning_rate, mi.power_t, mi.init_acc_gradient); + reg_lr + .optimizer_lr + .init(mi.learning_rate, mi.power_t, mi.init_acc_gradient); reg_lr.weights_len = 1 << mi.bit_precision; Ok(Box::new(reg_lr)) } pub fn new_lr_block( - bg: &mut graph::BlockGraph, - mi: &model_instance::ModelInstance) - -> Result> { + bg: &mut graph::BlockGraph, + mi: &model_instance::ModelInstance, +) -> Result> { let block = match mi.optimizer { - model_instance::Optimizer::AdagradLUT => new_lr_block_without_weights::(&mi), - model_instance::Optimizer::AdagradFlex => new_lr_block_without_weights::(&mi), - model_instance::Optimizer::SGD => new_lr_block_without_weights::(&mi) - }.unwrap(); + model_instance::Optimizer::AdagradLUT => { + new_lr_block_without_weights::(&mi) + } + model_instance::Optimizer::AdagradFlex => { + new_lr_block_without_weights::(&mi) + } + model_instance::Optimizer::SGD => { + new_lr_block_without_weights::(&mi) + } + } + .unwrap(); let mut block_outputs = bg.add_node(block, vec![])?; assert_eq!(block_outputs.len(), 1); Ok(block_outputs.pop().unwrap()) } - - - -impl BlockTrait for BlockLR -{ +impl BlockTrait for BlockLR { fn as_any(&mut self) -> &mut dyn Any { self } fn allocate_and_init_weights(&mut self, mi: &model_instance::ModelInstance) { - self.weights = vec![WeightAndOptimizerData::{weight:0.0, optimizer_data: self.optimizer_lr.initial_data()}; self.weights_len as usize]; - + self.weights = vec![ + WeightAndOptimizerData:: { + weight: 0.0, + optimizer_data: self.optimizer_lr.initial_data() + }; + self.weights_len as usize + ]; } - fn get_num_output_slots(&self) -> usize {1} - + fn get_num_output_slots(&self) -> usize { + 1 + } fn get_num_output_values(&self, output: graph::OutputSlot) -> usize { assert!(output.get_output_index() == 0); self.num_combos as usize } - - fn set_input_offset(&mut self, input: graph::InputSlot, offset: usize) { + fn set_input_offset(&mut self, input: graph::InputSlot, offset: usize) { panic!("You cannnot set_input_offset() for BlockLR"); } - fn set_output_offset(&mut self, output: graph::OutputSlot, offset: usize) { + fn set_output_offset(&mut self, output: graph::OutputSlot, offset: usize) { assert!(output.get_output_index() == 0); debug_assert!(self.output_offset == usize::MAX); // We only allow a single call self.output_offset = offset; } - #[inline(always)] - fn forward_backward(&mut self, - further_blocks: &mut [Box], - fb: &feature_buffer::FeatureBuffer, - pb: &mut port_buffer::PortBuffer, - update:bool) { + fn forward_backward( + &mut self, + further_blocks: &mut [Box], + fb: &feature_buffer::FeatureBuffer, + pb: &mut port_buffer::PortBuffer, + update: bool, + ) { debug_assert!(self.output_offset != usize::MAX); - let mut wsum:f32 = 0.0; + let mut wsum: f32 = 0.0; unsafe { { - let myslice = &mut pb.tape.get_unchecked_mut(self.output_offset .. (self.output_offset + self.num_combos as usize)); + let myslice = &mut pb.tape.get_unchecked_mut( + self.output_offset..(self.output_offset + self.num_combos as usize), + ); myslice.fill(0.0); for hashvalue in fb.lr_buffer.iter() { // Prefetch couple of indexes from the future to prevent pipeline stalls due to memory latencies - // for (i, hashvalue) in fb.lr_buffer.iter().enumerate() { + // for (i, hashvalue) in fb.lr_buffer.iter().enumerate() { // _mm_prefetch(mem::transmute::<&f32, &i8>(&self.weights.get_unchecked((fb.lr_buffer.get_unchecked(i+8).hash) as usize).weight), _MM_HINT_T0); // No benefit for now - let feature_index = hashvalue.hash; - let feature_value:f32 = hashvalue.value; + let feature_index = hashvalue.hash; + let feature_value: f32 = hashvalue.value; let combo_index = hashvalue.combo_index; - let feature_weight = self.weights.get_unchecked(feature_index as usize).weight; - *myslice.get_unchecked_mut(combo_index as usize) += feature_weight * feature_value; + let feature_weight = self.weights.get_unchecked(feature_index as usize).weight; + *myslice.get_unchecked_mut(combo_index as usize) += + feature_weight * feature_value; } } block_helpers::forward_backward(further_blocks, fb, pb, update); if update { - let myslice = &mut pb.tape.get_unchecked(self.output_offset .. (self.output_offset + self.num_combos as usize)); + let myslice = &mut pb.tape.get_unchecked( + self.output_offset..(self.output_offset + self.num_combos as usize), + ); for hashvalue in fb.lr_buffer.iter() { - let feature_index = hashvalue.hash as usize; - let feature_value:f32 = hashvalue.value; + let feature_index = hashvalue.hash as usize; + let feature_value: f32 = hashvalue.value; let general_gradient = myslice.get_unchecked(hashvalue.combo_index as usize); let gradient = general_gradient * feature_value; - let update = self.optimizer_lr.calculate_update(gradient, &mut self.weights.get_unchecked_mut(feature_index).optimizer_data); + let update = self.optimizer_lr.calculate_update( + gradient, + &mut self.weights.get_unchecked_mut(feature_index).optimizer_data, + ); self.weights.get_unchecked_mut(feature_index).weight -= update; } } } // end of unsafe } - - - fn forward(&self, - further_blocks: &[Box], - fb: &feature_buffer::FeatureBuffer, - pb: &mut port_buffer::PortBuffer) { + + fn forward( + &self, + further_blocks: &[Box], + fb: &feature_buffer::FeatureBuffer, + pb: &mut port_buffer::PortBuffer, + ) { debug_assert!(self.output_offset != usize::MAX); let fbuf = &fb.lr_buffer; - let mut wsum:f32 = 0.0; - - { + let mut wsum: f32 = 0.0; + { unsafe { - let myslice = &mut pb.tape[self.output_offset .. (self.output_offset + self.num_combos as usize)]; + let myslice = &mut pb.tape + [self.output_offset..(self.output_offset + self.num_combos as usize)]; myslice.fill(0.0); for val in fbuf { let hash = val.hash as usize; - let feature_value:f32 = val.value; - *myslice.get_unchecked_mut(val.combo_index as usize) += self.weights.get_unchecked(hash).weight * feature_value; + let feature_value: f32 = val.value; + *myslice.get_unchecked_mut(val.combo_index as usize) += + self.weights.get_unchecked(hash).weight * feature_value; } } } block_helpers::forward(further_blocks, fb, pb); - } - - + fn get_serialized_len(&self) -> usize { return self.weights_len as usize; } - fn read_weights_from_buf(&mut self, input_bufreader: &mut dyn io::Read) -> Result<(), Box> { + fn read_weights_from_buf( + &mut self, + input_bufreader: &mut dyn io::Read, + ) -> Result<(), Box> { block_helpers::read_weights_from_buf(&mut self.weights, input_bufreader) - } + } - fn write_weights_to_buf(&self, output_bufwriter: &mut dyn io::Write) -> Result<(), Box> { + fn write_weights_to_buf( + &self, + output_bufwriter: &mut dyn io::Write, + ) -> Result<(), Box> { block_helpers::write_weights_to_buf(&self.weights, output_bufwriter) } - fn read_weights_from_buf_into_forward_only(&self, input_bufreader: &mut dyn io::Read, forward: &mut Box) -> Result<(), Box> { - let mut forward = forward.as_any().downcast_mut::>().unwrap(); - block_helpers::read_weights_only_from_buf2::(self.weights_len as usize, &mut forward.weights, input_bufreader) + fn read_weights_from_buf_into_forward_only( + &self, + input_bufreader: &mut dyn io::Read, + forward: &mut Box, + ) -> Result<(), Box> { + let mut forward = forward + .as_any() + .downcast_mut::>() + .unwrap(); + block_helpers::read_weights_only_from_buf2::( + self.weights_len as usize, + &mut forward.weights, + input_bufreader, + ) } /// Sets internal state of weights based on some completely object-dependent parameters - fn testing_set_weights(&mut self, aa: i32, bb: i32, index: usize, w: &[f32]) -> Result<(), Box> { + fn testing_set_weights( + &mut self, + aa: i32, + bb: i32, + index: usize, + w: &[f32], + ) -> Result<(), Box> { self.weights[index].weight = w[0]; self.weights[index].optimizer_data = self.optimizer_lr.initial_data(); Ok(()) } - - } - diff --git a/src/block_misc.rs b/src/block_misc.rs index a9a45a96..9bf1d57e 100644 --- a/src/block_misc.rs +++ b/src/block_misc.rs @@ -1,18 +1,18 @@ use std::any::Any; use std::error::Error; -use crate::regressor; +use crate::block_helpers; use crate::feature_buffer; -use crate::port_buffer; -use crate::model_instance; use crate::graph; -use crate::block_helpers; +use crate::model_instance; +use crate::port_buffer; +use crate::regressor; use regressor::BlockTrait; #[derive(PartialEq)] pub enum Observe { Forward, - Backward + Backward, } pub struct BlockObserve { @@ -22,27 +22,25 @@ pub struct BlockObserve { replace_backward_with: Option, } - -pub fn new_observe_block(bg: &mut graph::BlockGraph, - input: graph::BlockPtrOutput, - observe: Observe, - replace_backward_with: Option) - -> Result> { - +pub fn new_observe_block( + bg: &mut graph::BlockGraph, + input: graph::BlockPtrOutput, + observe: Observe, + replace_backward_with: Option, +) -> Result> { let num_inputs = bg.get_num_output_values(vec![&input]); -// println!("Inputs: {} vec: {:?}", num_inputs, input); + // println!("Inputs: {} vec: {:?}", num_inputs, input); let block = Box::new(BlockObserve { - num_inputs: num_inputs as usize, - input_offset: usize::MAX, - observe: observe, - replace_backward_with: replace_backward_with}); + num_inputs: num_inputs as usize, + input_offset: usize::MAX, + observe: observe, + replace_backward_with: replace_backward_with, + }); let mut block_outputs = bg.add_node(block, vec![input])?; assert_eq!(block_outputs.len(), 1); Ok(block_outputs.pop().unwrap()) } - - impl BlockTrait for BlockObserve { // Warning: It does not confirm to regular clean-up after itself @@ -50,22 +48,26 @@ impl BlockTrait for BlockObserve { self } - fn get_block_type(&self) -> graph::BlockType {graph::BlockType::Observe} + fn get_block_type(&self) -> graph::BlockType { + graph::BlockType::Observe + } - fn get_num_output_slots(&self) -> usize {1} // It is a pass-through + fn get_num_output_slots(&self) -> usize { + 1 + } // It is a pass-through fn get_num_output_values(&self, output: graph::OutputSlot) -> usize { assert!(output.get_output_index() == 0); - return self.num_inputs + return self.num_inputs; } - fn set_input_offset(&mut self, input: graph::InputSlot, offset: usize) { + fn set_input_offset(&mut self, input: graph::InputSlot, offset: usize) { assert!(input.get_input_index() == 0); assert!(self.input_offset == usize::MAX); // We only allow a single call self.input_offset = offset; } - fn set_output_offset(&mut self, output: graph::OutputSlot, offset: usize) { + fn set_output_offset(&mut self, output: graph::OutputSlot, offset: usize) { assert!(output.get_output_index() == 0); assert_eq!(self.input_offset, offset); // this block type has special treatment } @@ -75,65 +77,75 @@ impl BlockTrait for BlockObserve { Ok(self.input_offset) } - #[inline(always)] - fn forward_backward(&mut self, - further_blocks: &mut [Box], - fb: &feature_buffer::FeatureBuffer, - pb: &mut port_buffer::PortBuffer, - update:bool) { + fn forward_backward( + &mut self, + further_blocks: &mut [Box], + fb: &feature_buffer::FeatureBuffer, + pb: &mut port_buffer::PortBuffer, + update: bool, + ) { debug_assert!(self.input_offset != usize::MAX); if self.observe == Observe::Forward { - pb.observations.extend_from_slice(&pb.tape[self.input_offset .. (self.input_offset + self.num_inputs)]); + pb.observations.extend_from_slice( + &pb.tape[self.input_offset..(self.input_offset + self.num_inputs)], + ); } block_helpers::forward_backward(further_blocks, fb, pb, update); - + if self.observe == Observe::Backward { - pb.observations.extend_from_slice(&pb.tape[self.input_offset .. (self.input_offset + self.num_inputs)]); + pb.observations.extend_from_slice( + &pb.tape[self.input_offset..(self.input_offset + self.num_inputs)], + ); } // replace inputs with whatever we wanted match self.replace_backward_with { - Some(value) => pb.tape[self.input_offset..(self.input_offset + self.num_inputs)].fill(value), - None => {}, + Some(value) => { + pb.tape[self.input_offset..(self.input_offset + self.num_inputs)].fill(value) + } + None => {} } - } #[inline(always)] - fn forward(&self, - further_blocks: &[Box], - fb: &feature_buffer::FeatureBuffer, - pb: &mut port_buffer::PortBuffer - ) { + fn forward( + &self, + further_blocks: &[Box], + fb: &feature_buffer::FeatureBuffer, + pb: &mut port_buffer::PortBuffer, + ) { debug_assert!(self.input_offset != usize::MAX); - + if self.observe == Observe::Forward { - pb.observations.extend_from_slice(&pb.tape[self.input_offset .. (self.input_offset + self.num_inputs)]); + pb.observations.extend_from_slice( + &pb.tape[self.input_offset..(self.input_offset + self.num_inputs)], + ); } block_helpers::forward(further_blocks, fb, pb); if self.observe == Observe::Backward { - pb.observations.extend_from_slice(&pb.tape[self.input_offset .. (self.input_offset + self.num_inputs)]); + pb.observations.extend_from_slice( + &pb.tape[self.input_offset..(self.input_offset + self.num_inputs)], + ); } // replace inputs with whatever we wanted match self.replace_backward_with { - Some(value) => pb.tape[self.input_offset..(self.input_offset + self.num_inputs)].fill(value), - None => {}, + Some(value) => { + pb.tape[self.input_offset..(self.input_offset + self.num_inputs)].fill(value) + } + None => {} } - - } - } pub enum SinkType { Zero, - Untouched + Untouched, } pub struct BlockSink { @@ -142,25 +154,22 @@ pub struct BlockSink { sink_type: SinkType, } - -pub fn new_sink_block(bg: &mut graph::BlockGraph, - input: graph::BlockPtrOutput, - sink_type: SinkType) - -> Result<(), Box> { - +pub fn new_sink_block( + bg: &mut graph::BlockGraph, + input: graph::BlockPtrOutput, + sink_type: SinkType, +) -> Result<(), Box> { let num_inputs = bg.get_num_output_values(vec![&input]); let block = Box::new(BlockSink { - input_offset: usize::MAX, - num_inputs: num_inputs, - sink_type: sink_type, - }); + input_offset: usize::MAX, + num_inputs: num_inputs, + sink_type: sink_type, + }); let mut block_outputs = bg.add_node(block, vec![input])?; assert_eq!(block_outputs.len(), 0); Ok(()) } - - impl BlockTrait for BlockSink { // Warning: It does not confirm to regular clean-up after itself @@ -168,76 +177,78 @@ impl BlockTrait for BlockSink { self } - fn get_block_type(&self) -> graph::BlockType {graph::BlockType::Regular} // It is regular, as there is no special functionality. + fn get_block_type(&self) -> graph::BlockType { + graph::BlockType::Regular + } // It is regular, as there is no special functionality. - fn get_num_output_slots(&self) -> usize {0} // It is a pass-through + fn get_num_output_slots(&self) -> usize { + 0 + } // It is a pass-through fn get_num_output_values(&self, output: graph::OutputSlot) -> usize { assert!(false, "No output values in BlockSink"); 0 } - fn set_input_offset(&mut self, input: graph::InputSlot, offset: usize) { + fn set_input_offset(&mut self, input: graph::InputSlot, offset: usize) { assert!(input.get_input_index() == 0); assert!(self.input_offset == usize::MAX); // We only allow a single call self.input_offset = offset; } - fn set_output_offset(&mut self, output: graph::OutputSlot, offset: usize) { + fn set_output_offset(&mut self, output: graph::OutputSlot, offset: usize) { assert!(false, "No outputs in BlockSink"); } #[inline(always)] - fn forward_backward(&mut self, - further_blocks: &mut [Box], - fb: &feature_buffer::FeatureBuffer, - pb: &mut port_buffer::PortBuffer, - update:bool) { + fn forward_backward( + &mut self, + further_blocks: &mut [Box], + fb: &feature_buffer::FeatureBuffer, + pb: &mut port_buffer::PortBuffer, + update: bool, + ) { debug_assert!(self.input_offset != usize::MAX); block_helpers::forward_backward(further_blocks, fb, pb, update); match self.sink_type { - SinkType::Zero => {pb.tape[self.input_offset..(self.input_offset + self.num_inputs)].fill(0.0);}, - SinkType::Untouched => {}, + SinkType::Zero => { + pb.tape[self.input_offset..(self.input_offset + self.num_inputs)].fill(0.0); + } + SinkType::Untouched => {} } } #[inline(always)] - fn forward(&self, - further_blocks: &[Box], - fb: &feature_buffer::FeatureBuffer, - pb: &mut port_buffer::PortBuffer - ) { + fn forward( + &self, + further_blocks: &[Box], + fb: &feature_buffer::FeatureBuffer, + pb: &mut port_buffer::PortBuffer, + ) { debug_assert!(self.input_offset != usize::MAX); block_helpers::forward(further_blocks, fb, pb); } - } - - - - pub struct BlockConsts { pub output_offset: usize, consts: Vec, - } -pub fn new_const_block( bg: &mut graph::BlockGraph, - consts: Vec) - -> Result> { - let block = Box::new(BlockConsts { output_offset: usize::MAX, - consts: consts}); +pub fn new_const_block( + bg: &mut graph::BlockGraph, + consts: Vec, +) -> Result> { + let block = Box::new(BlockConsts { + output_offset: usize::MAX, + consts: consts, + }); let mut block_outputs = bg.add_node(block, vec![])?; assert_eq!(block_outputs.len(), 1); Ok(block_outputs.pop().unwrap()) - - } - - impl BlockTrait for BlockConsts { // Warning: It does not confirm to regular clean-up after itself @@ -245,61 +256,68 @@ impl BlockTrait for BlockConsts { self } - fn get_num_output_slots(&self) -> usize {1} + fn get_num_output_slots(&self) -> usize { + 1 + } fn get_num_output_values(&self, output: graph::OutputSlot) -> usize { assert!(output.get_output_index() == 0); self.consts.len() as usize } - - fn set_input_offset(&mut self, input: graph::InputSlot, offset: usize) { + + fn set_input_offset(&mut self, input: graph::InputSlot, offset: usize) { panic!("You cannnot set input_tape_index for BlockConsts"); } fn set_output_offset(&mut self, output: graph::OutputSlot, offset: usize) { - assert!(output.get_output_index() == 0, "Only supports a single output for BlockConsts"); + assert!( + output.get_output_index() == 0, + "Only supports a single output for BlockConsts" + ); self.output_offset = offset; } #[inline(always)] - fn forward_backward(&mut self, - further_blocks: &mut [Box], - fb: &feature_buffer::FeatureBuffer, - pb: &mut port_buffer::PortBuffer, - update:bool) { + fn forward_backward( + &mut self, + further_blocks: &mut [Box], + fb: &feature_buffer::FeatureBuffer, + pb: &mut port_buffer::PortBuffer, + update: bool, + ) { debug_assert!(self.output_offset != usize::MAX); - pb.tape[self.output_offset..(self.output_offset + self.consts.len())].copy_from_slice(&self.consts); + pb.tape[self.output_offset..(self.output_offset + self.consts.len())] + .copy_from_slice(&self.consts); block_helpers::forward_backward(further_blocks, fb, pb, update); } - fn forward(&self, - further_blocks: &[Box], - fb: &feature_buffer::FeatureBuffer, - pb: &mut port_buffer::PortBuffer, ) { - + fn forward( + &self, + further_blocks: &[Box], + fb: &feature_buffer::FeatureBuffer, + pb: &mut port_buffer::PortBuffer, + ) { debug_assert!(self.output_offset != usize::MAX); - pb.tape[self.output_offset..(self.output_offset + self.consts.len())].copy_from_slice(&self.consts); + pb.tape[self.output_offset..(self.output_offset + self.consts.len())] + .copy_from_slice(&self.consts); block_helpers::forward(further_blocks, fb, pb); - } - } - -pub struct BlockCopy { +pub struct BlockCopy { pub num_inputs: usize, pub input_offset: usize, pub output_offsets: Vec, } - -pub fn new_copy_block(bg: &mut graph::BlockGraph, - input: graph::BlockPtrOutput, - num_output_slots: usize, - ) -> Result, Box> { +pub fn new_copy_block( + bg: &mut graph::BlockGraph, + input: graph::BlockPtrOutput, + num_output_slots: usize, +) -> Result, Box> { let num_inputs = bg.get_num_output_values(vec![&input]); assert!(num_inputs != 0); @@ -313,9 +331,10 @@ pub fn new_copy_block(bg: &mut graph::BlockGraph, Ok(block_outputs) } -pub fn new_copy_block_2(bg: &mut graph::BlockGraph, - input: graph::BlockPtrOutput, - ) -> Result<(graph::BlockPtrOutput, graph::BlockPtrOutput), Box> { +pub fn new_copy_block_2( + bg: &mut graph::BlockGraph, + input: graph::BlockPtrOutput, +) -> Result<(graph::BlockPtrOutput, graph::BlockPtrOutput), Box> { let mut outputs = new_copy_block(bg, input, 2)?; assert!(outputs.len() == 2); let output_2 = outputs.pop().unwrap(); @@ -323,20 +342,27 @@ pub fn new_copy_block_2(bg: &mut graph::BlockGraph, Ok((output_1, output_2)) } - - impl BlockTrait for BlockCopy { fn as_any(&mut self) -> &mut dyn Any { self } - fn get_block_type(&self) -> graph::BlockType {graph::BlockType::Copy} + fn get_block_type(&self) -> graph::BlockType { + graph::BlockType::Copy + } - fn get_num_output_slots(&self) -> usize {self.output_offsets.len()} + fn get_num_output_slots(&self) -> usize { + self.output_offsets.len() + } fn get_num_output_values(&self, output: graph::OutputSlot) -> usize { - assert!(output.get_output_index() < self.output_offsets.len(), "output.get_output_index(): {}, self.output_offsets.len(): {}", output.get_output_index(), self.output_offsets.len()); - self.num_inputs // all output slots have the same number of output values + assert!( + output.get_output_index() < self.output_offsets.len(), + "output.get_output_index(): {}, self.output_offsets.len(): {}", + output.get_output_index(), + self.output_offsets.len() + ); + self.num_inputs // all output slots have the same number of output values } fn get_input_offset(&mut self, input: graph::InputSlot) -> Result> { @@ -344,7 +370,7 @@ impl BlockTrait for BlockCopy { Ok(self.input_offset) } - fn set_input_offset(&mut self, input: graph::InputSlot, offset: usize) { + fn set_input_offset(&mut self, input: graph::InputSlot, offset: usize) { assert!(input.get_input_index() == 0); assert!(self.input_offset == usize::MAX); // We only allow a single call self.input_offset = offset; @@ -362,42 +388,56 @@ impl BlockTrait for BlockCopy { } } - #[inline(always)] - fn forward_backward(&mut self, - further_blocks: &mut [Box], - fb: &feature_buffer::FeatureBuffer, - pb: &mut port_buffer::PortBuffer, - update:bool) { + fn forward_backward( + &mut self, + further_blocks: &mut [Box], + fb: &feature_buffer::FeatureBuffer, + pb: &mut port_buffer::PortBuffer, + update: bool, + ) { debug_assert!(self.input_offset != usize::MAX); //debug_assert!(self.input_offset == self.output_offsets[0]); debug_assert!(self.num_inputs > 0); - + unsafe { // CopyBlock supports two modes: // If input is the same as the first output offset, there is just one copy to be done. // let output_offset_0 = *self.output_offsets.get_unchecked(0); if self.input_offset != output_offset_0 { - pb.tape.copy_within(self.input_offset .. (self.input_offset + self.num_inputs), output_offset_0); + pb.tape.copy_within( + self.input_offset..(self.input_offset + self.num_inputs), + output_offset_0, + ); } for &output_offset in self.output_offsets.get_unchecked(1..) { debug_assert!(output_offset != usize::MAX); - pb.tape.copy_within(self.input_offset .. (self.input_offset + self.num_inputs), output_offset); - } + pb.tape.copy_within( + self.input_offset..(self.input_offset + self.num_inputs), + output_offset, + ); + } block_helpers::forward_backward(further_blocks, fb, pb, update); if update { let output_offset_0 = *self.output_offsets.get_unchecked(0); if self.input_offset != output_offset_0 { - pb.tape.copy_within(output_offset_0 .. (output_offset_0 + self.num_inputs), self.input_offset); + pb.tape.copy_within( + output_offset_0..(output_offset_0 + self.num_inputs), + self.input_offset, + ); } - + // Sum up the gradients from output to input for &output_offset in self.output_offsets.get_unchecked(1..) { - let (input_tape, output_tape) = block_helpers::get_input_output_borrows(&mut pb.tape, - self.input_offset, self.num_inputs, - output_offset, self.num_inputs); + let (input_tape, output_tape) = block_helpers::get_input_output_borrows( + &mut pb.tape, + self.input_offset, + self.num_inputs, + output_offset, + self.num_inputs, + ); for i in 0..self.num_inputs { *input_tape.get_unchecked_mut(i) += *output_tape.get_unchecked(i); } @@ -405,40 +445,44 @@ impl BlockTrait for BlockCopy { } } // unsafe end } - - fn forward(&self, further_blocks: &[Box], - fb: &feature_buffer::FeatureBuffer, - pb: &mut port_buffer::PortBuffer, - ) { - // plain copy from input to output + + fn forward( + &self, + further_blocks: &[Box], + fb: &feature_buffer::FeatureBuffer, + pb: &mut port_buffer::PortBuffer, + ) { + // plain copy from input to output unsafe { let output_offset_0 = *self.output_offsets.get_unchecked(0); if self.input_offset != output_offset_0 { - pb.tape.copy_within(self.input_offset .. (self.input_offset + self.num_inputs), output_offset_0); + pb.tape.copy_within( + self.input_offset..(self.input_offset + self.num_inputs), + output_offset_0, + ); } for &output_offset in self.output_offsets.get_unchecked(1..) { - pb.tape.copy_within(self.input_offset .. (self.input_offset + self.num_inputs), output_offset); - } + pb.tape.copy_within( + self.input_offset..(self.input_offset + self.num_inputs), + output_offset, + ); + } } block_helpers::forward(further_blocks, fb, pb); } - } - - - -pub struct BlockJoin { +pub struct BlockJoin { pub num_inputs: usize, pub input_offset: usize, pub output_offset: usize, } - -pub fn new_join_block(bg: &mut graph::BlockGraph, - inputs: Vec, - ) -> Result> { +pub fn new_join_block( + bg: &mut graph::BlockGraph, + inputs: Vec, +) -> Result> { let num_inputs = bg.get_num_output_values(inputs.iter().collect()); assert!(num_inputs != 0); @@ -456,11 +500,15 @@ impl BlockTrait for BlockJoin { fn as_any(&mut self) -> &mut dyn Any { self } - - fn get_block_type(&self) -> graph::BlockType {graph::BlockType::Join} - fn get_num_output_slots(&self) -> usize {1} - + fn get_block_type(&self) -> graph::BlockType { + graph::BlockType::Join + } + + fn get_num_output_slots(&self) -> usize { + 1 + } + fn get_num_output_values(&self, output: graph::OutputSlot) -> usize { assert!(output.get_output_index() == 0); self.num_inputs @@ -471,17 +519,28 @@ impl BlockTrait for BlockJoin { Ok(self.input_offset) } - fn set_input_offset(&mut self, input: graph::InputSlot, offset: usize) { + fn set_input_offset(&mut self, input: graph::InputSlot, offset: usize) { //assert!(input.get_input_index() <= 1); // We now support multiple joins, so no need to assume just one if input.get_input_index() == 0 { assert!(self.input_offset == usize::MAX); // We only allow a single call self.input_offset = offset; } else if input.get_input_index() >= 1 { - assert!(self.input_offset <= offset, "Output 1, error 1: Input offset: {}, num_inputs: {}, offset: {}", self.input_offset, self.num_inputs, offset); - assert!(self.input_offset + self.num_inputs >= offset, "Output 1, error 2: Input offset: {}, num_inputs: {}, offset: {}", self.input_offset, self.num_inputs, offset); + assert!( + self.input_offset <= offset, + "Output 1, error 1: Input offset: {}, num_inputs: {}, offset: {}", + self.input_offset, + self.num_inputs, + offset + ); + assert!( + self.input_offset + self.num_inputs >= offset, + "Output 1, error 2: Input offset: {}, num_inputs: {}, offset: {}", + self.input_offset, + self.num_inputs, + offset + ); } - - } + } fn set_output_offset(&mut self, output: graph::OutputSlot, offset: usize) { assert!(output.get_output_index() == 0); @@ -491,39 +550,37 @@ impl BlockTrait for BlockJoin { // WARNING: These two functions are automatically removed from the graph when executing, since they are a no-op #[inline(always)] - fn forward_backward(&mut self, - further_blocks: &mut [Box], - fb: &feature_buffer::FeatureBuffer, - pb: &mut port_buffer::PortBuffer, - update:bool) { + fn forward_backward( + &mut self, + further_blocks: &mut [Box], + fb: &feature_buffer::FeatureBuffer, + pb: &mut port_buffer::PortBuffer, + update: bool, + ) { debug_assert!(self.input_offset != usize::MAX); debug_assert!(self.output_offset != usize::MAX); debug_assert!(self.num_inputs > 0); - + block_helpers::forward_backward(further_blocks, fb, pb, update); } - - fn forward(&self, further_blocks: &[Box], - fb: &feature_buffer::FeatureBuffer, - pb: &mut port_buffer::PortBuffer, - ) { + + fn forward( + &self, + further_blocks: &[Box], + fb: &feature_buffer::FeatureBuffer, + pb: &mut port_buffer::PortBuffer, + ) { block_helpers::forward(further_blocks, fb, pb); } - } - -pub struct BlockSum { +pub struct BlockSum { pub num_inputs: usize, pub input_offset: usize, pub output_offset: usize, - } - -fn new_sum_without_weights( - num_inputs: usize, - ) -> Result, Box> { +fn new_sum_without_weights(num_inputs: usize) -> Result, Box> { assert!(num_inputs > 0); let mut rg = BlockSum { output_offset: usize::MAX, @@ -533,106 +590,104 @@ fn new_sum_without_weights( Ok(Box::new(rg)) } - - - -pub fn new_sum_block( bg: &mut graph::BlockGraph, - input: graph::BlockPtrOutput, - ) -> Result> { +pub fn new_sum_block( + bg: &mut graph::BlockGraph, + input: graph::BlockPtrOutput, +) -> Result> { let num_inputs = bg.get_num_output_values(vec![&input]); - let block = new_sum_without_weights(num_inputs).unwrap(); + let block = new_sum_without_weights(num_inputs).unwrap(); let mut block_outputs = bg.add_node(block, vec![input]).unwrap(); assert_eq!(block_outputs.len(), 1); Ok(block_outputs.pop().unwrap()) } - - - - - - impl BlockTrait for BlockSum { fn as_any(&mut self) -> &mut dyn Any { self } - fn get_num_output_slots(&self) -> usize {1} + fn get_num_output_slots(&self) -> usize { + 1 + } fn get_num_output_values(&self, output: graph::OutputSlot) -> usize { assert!(output.get_output_index() == 0); 1 } - fn set_input_offset(&mut self, input: graph::InputSlot, offset: usize) { + fn set_input_offset(&mut self, input: graph::InputSlot, offset: usize) { assert!(input.get_input_index() == 0); assert!(self.input_offset == usize::MAX); // We only allow a single call self.input_offset = offset; } - fn set_output_offset(&mut self, output: graph::OutputSlot, offset: usize) { + fn set_output_offset(&mut self, output: graph::OutputSlot, offset: usize) { assert!(output.get_output_index() == 0); assert!(self.output_offset == usize::MAX); // We only allow a single call self.output_offset = offset; } #[inline(always)] - fn forward_backward(&mut self, - further_blocks: &mut [Box], - fb: &feature_buffer::FeatureBuffer, - pb: &mut port_buffer::PortBuffer, - update:bool) { + fn forward_backward( + &mut self, + further_blocks: &mut [Box], + fb: &feature_buffer::FeatureBuffer, + pb: &mut port_buffer::PortBuffer, + update: bool, + ) { debug_assert!(self.num_inputs > 0); debug_assert!(self.output_offset != usize::MAX); debug_assert!(self.input_offset != usize::MAX); - + unsafe { - let wsum:f32 = pb.tape.get_unchecked_mut(self.input_offset .. (self.input_offset + self.num_inputs as usize)).iter().sum(); + let wsum: f32 = pb + .tape + .get_unchecked_mut( + self.input_offset..(self.input_offset + self.num_inputs as usize), + ) + .iter() + .sum(); pb.tape[self.output_offset as usize] = wsum; - + block_helpers::forward_backward(further_blocks, fb, pb, update); let general_gradient = pb.tape[self.output_offset]; if update { - pb.tape.get_unchecked_mut(self.input_offset .. (self.input_offset + self.num_inputs as usize)).fill(general_gradient); + pb.tape + .get_unchecked_mut( + self.input_offset..(self.input_offset + self.num_inputs as usize), + ) + .fill(general_gradient); } } // unsafe end } - - fn forward( &self, - further_blocks: &[Box], - fb: &feature_buffer::FeatureBuffer, - pb: &mut port_buffer::PortBuffer, - ) { + + fn forward( + &self, + further_blocks: &[Box], + fb: &feature_buffer::FeatureBuffer, + pb: &mut port_buffer::PortBuffer, + ) { debug_assert!(self.num_inputs > 0); debug_assert!(self.output_offset != usize::MAX); debug_assert!(self.input_offset != usize::MAX); - - let wsum:f32 = pb.tape[self.input_offset .. (self.input_offset + self.num_inputs as usize)].iter().sum(); + + let wsum: f32 = pb.tape[self.input_offset..(self.input_offset + self.num_inputs as usize)] + .iter() + .sum(); pb.tape[self.output_offset as usize] = wsum; block_helpers::forward(further_blocks, fb, pb); } } - - - - - - - - - - - // From a square only keep weights that are on the lower left triangle + diagonal // Why is this useful? // Because in FFM you get a square matrix of outputs, but it is symetrical across the diagonal // This means that when you bring FFM to the first neural layer, you have double parameters needlessly -slowing things down // It would be possible to modify BlockFFM output, but that is excessively complex to do. -pub struct BlockTriangle { +pub struct BlockTriangle { pub square_width: usize, pub num_inputs: usize, pub num_outputs: usize, @@ -640,10 +695,10 @@ pub struct BlockTriangle { pub output_offset: usize, } - -pub fn new_triangle_block(bg: &mut graph::BlockGraph, - input: graph::BlockPtrOutput, - ) -> Result> { +pub fn new_triangle_block( + bg: &mut graph::BlockGraph, + input: graph::BlockPtrOutput, +) -> Result> { let num_inputs = bg.get_num_output_values(vec![&input]); assert!(num_inputs != 0); @@ -665,25 +720,21 @@ pub fn new_triangle_block(bg: &mut graph::BlockGraph, Ok(block_outputs.pop().unwrap()) } - - - - -impl BlockTrait for BlockTriangle - - { +impl BlockTrait for BlockTriangle { fn as_any(&mut self) -> &mut dyn Any { self } - fn get_num_output_slots(&self) -> usize {1} + fn get_num_output_slots(&self) -> usize { + 1 + } fn get_num_output_values(&self, output: graph::OutputSlot) -> usize { assert!(output.get_output_index() == 0); self.num_outputs } - fn set_input_offset(&mut self, input: graph::InputSlot, offset: usize) { + fn set_input_offset(&mut self, input: graph::InputSlot, offset: usize) { assert!(input.get_input_index() == 0); assert!(self.input_offset == usize::MAX); // We only allow a single call self.input_offset = offset; @@ -695,31 +746,37 @@ impl BlockTrait for BlockTriangle self.output_offset = offset; } - #[inline(always)] - fn forward_backward(&mut self, - further_blocks: &mut [Box], - fb: &feature_buffer::FeatureBuffer, - pb: &mut port_buffer::PortBuffer, - update:bool) { + fn forward_backward( + &mut self, + further_blocks: &mut [Box], + fb: &feature_buffer::FeatureBuffer, + pb: &mut port_buffer::PortBuffer, + update: bool, + ) { debug_assert!(self.input_offset != usize::MAX); debug_assert!(self.output_offset != usize::MAX); debug_assert!(self.num_inputs > 0); - + unsafe { { - let (input_tape, output_tape) = block_helpers::get_input_output_borrows(&mut pb.tape, - self.input_offset, self.num_inputs, - self.output_offset, self.num_outputs); - - + let (input_tape, output_tape) = block_helpers::get_input_output_borrows( + &mut pb.tape, + self.input_offset, + self.num_inputs, + self.output_offset, + self.num_outputs, + ); + let mut output_index: usize = 0; for i in 0..self.square_width { for j in 0..i { - *output_tape.get_unchecked_mut(output_index) = *input_tape.get_unchecked(i * self.square_width + j) * 2.0; + *output_tape.get_unchecked_mut(output_index) = + *input_tape.get_unchecked(i * self.square_width + j) * 2.0; output_index += 1; } - *output_tape.get_unchecked_mut(output_index) = *input_tape.get_unchecked(i * self.square_width + i); + *output_tape.get_unchecked_mut(output_index) = + *input_tape.get_unchecked(i * self.square_width + i); output_index += 1; } } @@ -727,302 +784,350 @@ impl BlockTrait for BlockTriangle block_helpers::forward_backward(further_blocks, fb, pb, update); if update { - let (input_tape, output_tape) = block_helpers::get_input_output_borrows(&mut pb.tape, - self.input_offset, self.num_inputs, - self.output_offset, self.num_outputs); + let (input_tape, output_tape) = block_helpers::get_input_output_borrows( + &mut pb.tape, + self.input_offset, + self.num_inputs, + self.output_offset, + self.num_outputs, + ); let mut output_index: usize = 0; for i in 0..self.square_width { - for j in 0..i+1 { - *input_tape.get_unchecked_mut(i * self.square_width + j) = *output_tape.get_unchecked(output_index); - *input_tape.get_unchecked_mut(j * self.square_width + i) = *output_tape.get_unchecked(output_index); + for j in 0..i + 1 { + *input_tape.get_unchecked_mut(i * self.square_width + j) = + *output_tape.get_unchecked(output_index); + *input_tape.get_unchecked_mut(j * self.square_width + i) = + *output_tape.get_unchecked(output_index); output_index += 1; } } } - } // unsafe end } - - fn forward(&self, further_blocks: &[Box], - fb: &feature_buffer::FeatureBuffer, - pb: &mut port_buffer::PortBuffer, - ) { - unsafe { - let (input_tape, output_tape) = block_helpers::get_input_output_borrows(&mut pb.tape, - self.input_offset, self.num_inputs, - self.output_offset, self.num_outputs); - - - let mut output_index: usize = 0; - for i in 0..self.square_width { -// println!("AAAAA i: {}", i); - for j in 0..i { -// let input = input_tape.get_unchecked(i * self.square_width + j); - // println!("Output index: i: {}, j: {}, A: {}, B: {}", i, j, input_tape[i * self.square_width + j], input_tape[j * self.square_width + i]); - *output_tape.get_unchecked_mut(output_index) = *input_tape.get_unchecked(i * self.square_width + j) * 2.0; - output_index += 1; - } - *output_tape.get_unchecked_mut(output_index) = *input_tape.get_unchecked(i * self.square_width + i); + + fn forward( + &self, + further_blocks: &[Box], + fb: &feature_buffer::FeatureBuffer, + pb: &mut port_buffer::PortBuffer, + ) { + unsafe { + let (input_tape, output_tape) = block_helpers::get_input_output_borrows( + &mut pb.tape, + self.input_offset, + self.num_inputs, + self.output_offset, + self.num_outputs, + ); + + let mut output_index: usize = 0; + for i in 0..self.square_width { + // println!("AAAAA i: {}", i); + for j in 0..i { + // let input = input_tape.get_unchecked(i * self.square_width + j); + // println!("Output index: i: {}, j: {}, A: {}, B: {}", i, j, input_tape[i * self.square_width + j], input_tape[j * self.square_width + i]); + *output_tape.get_unchecked_mut(output_index) = + *input_tape.get_unchecked(i * self.square_width + j) * 2.0; output_index += 1; } + *output_tape.get_unchecked_mut(output_index) = + *input_tape.get_unchecked(i * self.square_width + i); + output_index += 1; } - block_helpers::forward(further_blocks, fb, pb); + } + block_helpers::forward(further_blocks, fb, pb); } - } - - - - - - - - - mod tests { // Note this useful idiom: importing names from outer (for mod tests) scope. use super::*; - use crate::block_misc; - use crate::feature_buffer; use crate::block_helpers::{slearn2, spredict2}; - use crate::graph::{BlockGraph}; + use crate::block_misc; use crate::block_misc::Observe; + use crate::feature_buffer; + use crate::graph::BlockGraph; fn fb_vec() -> feature_buffer::FeatureBuffer { feature_buffer::FeatureBuffer { - label: 0.0, - example_importance: 1.0, - example_number: 0, - lr_buffer: Vec::new(), - ffm_buffer: Vec::new(), - ffm_fields_count: 0, + label: 0.0, + example_importance: 1.0, + example_number: 0, + lr_buffer: Vec::new(), + ffm_buffer: Vec::new(), + ffm_fields_count: 0, } } - #[test] fn test_sum_block() { - let mut mi = model_instance::ModelInstance::new_empty().unwrap(); + let mut mi = model_instance::ModelInstance::new_empty().unwrap(); let mut bg = BlockGraph::new(); let input_block = block_misc::new_const_block(&mut bg, vec![2.0, 3.0]).unwrap(); - let observe_block_backward = block_misc::new_observe_block(&mut bg, input_block, Observe::Backward, None).unwrap(); + let observe_block_backward = + block_misc::new_observe_block(&mut bg, input_block, Observe::Backward, None).unwrap(); let sum_block = new_sum_block(&mut bg, observe_block_backward).unwrap(); - let observe_block_forward = block_misc::new_observe_block(&mut bg, sum_block, Observe::Forward, Some(1.0)).unwrap(); + let observe_block_forward = + block_misc::new_observe_block(&mut bg, sum_block, Observe::Forward, Some(1.0)).unwrap(); bg.finalize(); bg.allocate_and_init_weights(&mi); - + let mut pb = bg.new_port_buffer(); let fb = fb_vec(); - slearn2 (&mut bg, &fb, &mut pb, true); - assert_eq!(pb.observations, vec![5.0, // forward part, just a sum of 2 and 3 - 1.0, 1.0]); // backward part -- 1 is distributed to both inputs - - spredict2 (&mut bg, &fb, &mut pb, true); - assert_eq!(pb.observations, vec![5.0, // forward part, just a sum of 2 and 3 - 2.0, 3.0]); // backward part -- nothing gets updated - - - + slearn2(&mut bg, &fb, &mut pb, true); + assert_eq!( + pb.observations, + vec![ + 5.0, // forward part, just a sum of 2 and 3 + 1.0, 1.0 + ] + ); // backward part -- 1 is distributed to both inputs + + spredict2(&mut bg, &fb, &mut pb, true); + assert_eq!( + pb.observations, + vec![ + 5.0, // forward part, just a sum of 2 and 3 + 2.0, 3.0 + ] + ); // backward part -- nothing gets updated } - #[test] fn test_triangle_block() { - let mut mi = model_instance::ModelInstance::new_empty().unwrap(); + let mut mi = model_instance::ModelInstance::new_empty().unwrap(); let mut bg = BlockGraph::new(); let input_block = block_misc::new_const_block(&mut bg, vec![2.0, 4.0, 4.0, 5.0]).unwrap(); - let observe_block_backward = block_misc::new_observe_block(&mut bg, input_block, Observe::Backward, None).unwrap(); + let observe_block_backward = + block_misc::new_observe_block(&mut bg, input_block, Observe::Backward, None).unwrap(); let triangle_block = new_triangle_block(&mut bg, observe_block_backward).unwrap(); - let observe_block_forward = block_misc::new_observe_block(&mut bg, triangle_block, Observe::Forward, None).unwrap(); - let sink = block_misc::new_sink_block(&mut bg, observe_block_forward, block_misc::SinkType::Untouched).unwrap(); + let observe_block_forward = + block_misc::new_observe_block(&mut bg, triangle_block, Observe::Forward, None).unwrap(); + let sink = block_misc::new_sink_block( + &mut bg, + observe_block_forward, + block_misc::SinkType::Untouched, + ) + .unwrap(); bg.finalize(); bg.allocate_and_init_weights(&mi); - + let mut pb = bg.new_port_buffer(); let fb = fb_vec(); - slearn2 (&mut bg, &fb, &mut pb, true); - assert_eq!(pb.observations, vec![2.0, 8.0, 5.0, // forward part - 2.0, 8.0, 8.0, 5.0]); // backward part -- 3.0 gets turned into 4.0 since that is its transpose - - + slearn2(&mut bg, &fb, &mut pb, true); + assert_eq!( + pb.observations, + vec![ + 2.0, 8.0, 5.0, // forward part + 2.0, 8.0, 8.0, 5.0 + ] + ); // backward part -- 3.0 gets turned into 4.0 since that is its transpose } - #[test] fn test_copy_block() { - let mut mi = model_instance::ModelInstance::new_empty().unwrap(); + let mut mi = model_instance::ModelInstance::new_empty().unwrap(); let mut bg = BlockGraph::new(); let input_block = block_misc::new_const_block(&mut bg, vec![2.0, 3.0]).unwrap(); - let observe_block_backward = block_misc::new_observe_block(&mut bg, input_block, Observe::Backward, None).unwrap(); - let (copy_block_1, copy_block_2)= new_copy_block_2(&mut bg, observe_block_backward).unwrap(); - let observe_block_1_forward = block_misc::new_observe_block(&mut bg, copy_block_1, Observe::Forward, Some(5.0)).unwrap(); - let observe_block_2_forward = block_misc::new_observe_block(&mut bg, copy_block_2, Observe::Forward, Some(6.0)).unwrap(); + let observe_block_backward = + block_misc::new_observe_block(&mut bg, input_block, Observe::Backward, None).unwrap(); + let (copy_block_1, copy_block_2) = + new_copy_block_2(&mut bg, observe_block_backward).unwrap(); + let observe_block_1_forward = + block_misc::new_observe_block(&mut bg, copy_block_1, Observe::Forward, Some(5.0)) + .unwrap(); + let observe_block_2_forward = + block_misc::new_observe_block(&mut bg, copy_block_2, Observe::Forward, Some(6.0)) + .unwrap(); bg.finalize(); bg.allocate_and_init_weights(&mi); - + let mut pb = bg.new_port_buffer(); let fb = fb_vec(); - slearn2 (&mut bg, &fb, &mut pb, true); - assert_eq!(pb.observations, vec![2.0, 3.0, // 1st copy of forward parts - 2.0, 3.0, // 2nd copy of forward - 11.0, 11.0, ]); // backward part (6+11) - - spredict2 (&mut bg, &fb, &mut pb, false); - assert_eq!(pb.observations, vec![2.0, 3.0, // 1st copy of forward parts - 2.0, 3.0, // 2nd copy of forward - 2.0, 3.0, ]); // backward part isn't touched, it will contain whatever observe block_1 put there - - + slearn2(&mut bg, &fb, &mut pb, true); + assert_eq!( + pb.observations, + vec![ + 2.0, 3.0, // 1st copy of forward parts + 2.0, 3.0, // 2nd copy of forward + 11.0, 11.0, + ] + ); // backward part (6+11) + + spredict2(&mut bg, &fb, &mut pb, false); + assert_eq!( + pb.observations, + vec![ + 2.0, 3.0, // 1st copy of forward parts + 2.0, 3.0, // 2nd copy of forward + 2.0, 3.0, + ] + ); // backward part isn't touched, it will contain whatever observe block_1 put there } #[test] fn test_copy_block_cascade() { - let mut mi = model_instance::ModelInstance::new_empty().unwrap(); + let mut mi = model_instance::ModelInstance::new_empty().unwrap(); let mut bg = BlockGraph::new(); let input_block = block_misc::new_const_block(&mut bg, vec![2.0, 3.0]).unwrap(); - let observe_block_backward = block_misc::new_observe_block(&mut bg, input_block, Observe::Backward, None).unwrap(); - let (copy_block_1, copy_block_2) = new_copy_block_2(&mut bg, observe_block_backward).unwrap(); + let observe_block_backward = + block_misc::new_observe_block(&mut bg, input_block, Observe::Backward, None).unwrap(); + let (copy_block_1, copy_block_2) = + new_copy_block_2(&mut bg, observe_block_backward).unwrap(); let (copy_block_3, copy_block_4) = new_copy_block_2(&mut bg, copy_block_1).unwrap(); - let observe_block_1_forward = block_misc::new_observe_block(&mut bg, copy_block_2, Observe::Forward, Some(5.0)).unwrap(); - let observe_block_2_forward = block_misc::new_observe_block(&mut bg, copy_block_3, Observe::Forward, Some(6.0)).unwrap(); - let observe_block_3_forward = block_misc::new_observe_block(&mut bg, copy_block_4, Observe::Forward, Some(7.0)).unwrap(); + let observe_block_1_forward = + block_misc::new_observe_block(&mut bg, copy_block_2, Observe::Forward, Some(5.0)) + .unwrap(); + let observe_block_2_forward = + block_misc::new_observe_block(&mut bg, copy_block_3, Observe::Forward, Some(6.0)) + .unwrap(); + let observe_block_3_forward = + block_misc::new_observe_block(&mut bg, copy_block_4, Observe::Forward, Some(7.0)) + .unwrap(); bg.finalize(); bg.allocate_and_init_weights(&mi); - + let mut pb = bg.new_port_buffer(); let fb = fb_vec(); - slearn2 (&mut bg, &fb, &mut pb, true); - assert_eq!(pb.observations, vec![2.0, 3.0, // 1st copy of forward parts - 2.0, 3.0, // 2nd copy of forward - 2.0, 3.0, - 18.0, 18.0, ]); // backward part (5+6+7) - - spredict2 (&mut bg, &fb, &mut pb, false); - assert_eq!(pb.observations, vec![2.0, 3.0, // 1st copy of forward parts - 2.0, 3.0, // 2nd copy of forward - 2.0, 3.0, - 2.0, 3.0]); // backward part isn't touched, it will contain whatever observe block_1 put there - // it is from copy_block_3 since that is the last one where observe_block_2_forward does its work - + slearn2(&mut bg, &fb, &mut pb, true); + assert_eq!( + pb.observations, + vec![ + 2.0, 3.0, // 1st copy of forward parts + 2.0, 3.0, // 2nd copy of forward + 2.0, 3.0, 18.0, 18.0, + ] + ); // backward part (5+6+7) + + spredict2(&mut bg, &fb, &mut pb, false); + assert_eq!( + pb.observations, + vec![ + 2.0, 3.0, // 1st copy of forward parts + 2.0, 3.0, // 2nd copy of forward + 2.0, 3.0, 2.0, 3.0 + ] + ); // backward part isn't touched, it will contain whatever observe block_1 put there + // it is from copy_block_3 since that is the last one where observe_block_2_forward does its work } - #[test] fn test_join_1_block() { - let mut mi = model_instance::ModelInstance::new_empty().unwrap(); + let mut mi = model_instance::ModelInstance::new_empty().unwrap(); let mut bg = BlockGraph::new(); let input_block_1 = block_misc::new_const_block(&mut bg, vec![2.0, 3.0]).unwrap(); let join_block = new_join_block(&mut bg, vec![input_block_1]).unwrap(); - let observe_block = block_misc::new_observe_block(&mut bg, join_block, Observe::Forward, Some(6.0)).unwrap(); + let observe_block = + block_misc::new_observe_block(&mut bg, join_block, Observe::Forward, Some(6.0)) + .unwrap(); bg.finalize(); bg.allocate_and_init_weights(&mi); - + let mut pb = bg.new_port_buffer(); let fb = fb_vec(); - slearn2 (&mut bg, &fb, &mut pb, true); - assert_eq!(pb.observations, vec![2.0, 3.0,]); // join actually doesn't do anything + slearn2(&mut bg, &fb, &mut pb, true); + assert_eq!(pb.observations, vec![2.0, 3.0,]); // join actually doesn't do anything - spredict2 (&mut bg, &fb, &mut pb, false); - assert_eq!(pb.observations, vec![2.0, 3.0,]); // join actually doesn't do anything + spredict2(&mut bg, &fb, &mut pb, false); + assert_eq!(pb.observations, vec![2.0, 3.0,]); // join actually doesn't do anything } - #[test] fn test_join_2_blocks() { - let mut mi = model_instance::ModelInstance::new_empty().unwrap(); + let mut mi = model_instance::ModelInstance::new_empty().unwrap(); let mut bg = BlockGraph::new(); let input_block_1 = block_misc::new_const_block(&mut bg, vec![2.0, 3.0]).unwrap(); let input_block_2 = block_misc::new_const_block(&mut bg, vec![4.0, 5.0, 6.0]).unwrap(); let join_block = new_join_block(&mut bg, vec![input_block_1, input_block_2]).unwrap(); - let observe_block = block_misc::new_observe_block(&mut bg, join_block, Observe::Forward, Some(6.0)).unwrap(); + let observe_block = + block_misc::new_observe_block(&mut bg, join_block, Observe::Forward, Some(6.0)) + .unwrap(); bg.finalize(); bg.allocate_and_init_weights(&mi); - + let mut pb = bg.new_port_buffer(); let fb = fb_vec(); - slearn2 (&mut bg, &fb, &mut pb, true); - assert_eq!(pb.observations, vec![2.0, 3.0, 4.0, 5.0, 6.0]); // join actually doesn't do anything + slearn2(&mut bg, &fb, &mut pb, true); + assert_eq!(pb.observations, vec![2.0, 3.0, 4.0, 5.0, 6.0]); // join actually doesn't do anything - spredict2 (&mut bg, &fb, &mut pb, false); - assert_eq!(pb.observations, vec![2.0, 3.0, 4.0, 5.0, 6.0]); // join actually doesn't do anything + spredict2(&mut bg, &fb, &mut pb, false); + assert_eq!(pb.observations, vec![2.0, 3.0, 4.0, 5.0, 6.0]); // join actually doesn't do anything } #[test] fn test_join_3_blocks() { - let mut mi = model_instance::ModelInstance::new_empty().unwrap(); + let mut mi = model_instance::ModelInstance::new_empty().unwrap(); let mut bg = BlockGraph::new(); let input_block_3 = block_misc::new_const_block(&mut bg, vec![1.0, 2.0]).unwrap(); let input_block_2 = block_misc::new_const_block(&mut bg, vec![3.0, 4.0, 5.0]).unwrap(); let input_block_1 = block_misc::new_const_block(&mut bg, vec![6.0, 7.0]).unwrap(); - let join_block = new_join_block(&mut bg, vec![input_block_1, input_block_2, input_block_3]).unwrap(); - let observe_block = block_misc::new_observe_block(&mut bg, join_block, Observe::Forward, Some(6.0)).unwrap(); + let join_block = + new_join_block(&mut bg, vec![input_block_1, input_block_2, input_block_3]).unwrap(); + let observe_block = + block_misc::new_observe_block(&mut bg, join_block, Observe::Forward, Some(6.0)) + .unwrap(); bg.finalize(); bg.allocate_and_init_weights(&mi); - + let mut pb = bg.new_port_buffer(); let fb = fb_vec(); - slearn2 (&mut bg, &fb, &mut pb, true); + slearn2(&mut bg, &fb, &mut pb, true); // Order depends on the input parameters order, not on order of adding to graph - assert_eq!(pb.observations, vec![6.0, 7.0, 3.0, 4.0, 5.0, 1.0, 2.0]); // join actually doesn't do anything + assert_eq!(pb.observations, vec![6.0, 7.0, 3.0, 4.0, 5.0, 1.0, 2.0]); // join actually doesn't do anything - spredict2 (&mut bg, &fb, &mut pb, false); - assert_eq!(pb.observations, vec![6.0, 7.0, 3.0, 4.0, 5.0, 1.0, 2.0]); // join actually doesn't do anything + spredict2(&mut bg, &fb, &mut pb, false); + assert_eq!(pb.observations, vec![6.0, 7.0, 3.0, 4.0, 5.0, 1.0, 2.0]); // join actually doesn't do anything } #[test] fn test_join_cascading() { - let mut mi = model_instance::ModelInstance::new_empty().unwrap(); + let mut mi = model_instance::ModelInstance::new_empty().unwrap(); let mut bg = BlockGraph::new(); let input_block_1 = block_misc::new_const_block(&mut bg, vec![1.0, 2.0]).unwrap(); let input_block_2 = block_misc::new_const_block(&mut bg, vec![3.0, 4.0, 5.0]).unwrap(); let join_block_1 = new_join_block(&mut bg, vec![input_block_1, input_block_2]).unwrap(); let input_block_3 = block_misc::new_const_block(&mut bg, vec![6.0, 7.0]).unwrap(); let join_block_2 = new_join_block(&mut bg, vec![input_block_3, join_block_1]).unwrap(); - let observe_block = block_misc::new_observe_block(&mut bg, join_block_2, Observe::Forward, Some(6.0)).unwrap(); + let observe_block = + block_misc::new_observe_block(&mut bg, join_block_2, Observe::Forward, Some(6.0)) + .unwrap(); bg.finalize(); bg.allocate_and_init_weights(&mi); - + let mut pb = bg.new_port_buffer(); let fb = fb_vec(); - slearn2 (&mut bg, &fb, &mut pb, true); + slearn2(&mut bg, &fb, &mut pb, true); // Order depends on the input parameters order, not on order of adding to graph - assert_eq!(pb.observations, vec![6.0, 7.0, 1.0, 2.0, 3.0, 4.0, 5.0]); // join actually doesn't do anything + assert_eq!(pb.observations, vec![6.0, 7.0, 1.0, 2.0, 3.0, 4.0, 5.0]); // join actually doesn't do anything - spredict2 (&mut bg, &fb, &mut pb, false); - assert_eq!(pb.observations, vec![6.0, 7.0, 1.0, 2.0, 3.0, 4.0, 5.0]); // join actually doesn't do anything + spredict2(&mut bg, &fb, &mut pb, false); + assert_eq!(pb.observations, vec![6.0, 7.0, 1.0, 2.0, 3.0, 4.0, 5.0]); // join actually doesn't do anything } #[test] fn test_copy_to_join() { - let mut mi = model_instance::ModelInstance::new_empty().unwrap(); + let mut mi = model_instance::ModelInstance::new_empty().unwrap(); let mut bg = BlockGraph::new(); let input_block_1 = block_misc::new_const_block(&mut bg, vec![2.0, 3.0]).unwrap(); - let observe_block_backward = block_misc::new_observe_block(&mut bg, input_block_1, Observe::Backward, None).unwrap(); - let (copy_1, copy_2) = block_misc::new_copy_block_2(&mut bg, observe_block_backward).unwrap(); + let observe_block_backward = + block_misc::new_observe_block(&mut bg, input_block_1, Observe::Backward, None).unwrap(); + let (copy_1, copy_2) = + block_misc::new_copy_block_2(&mut bg, observe_block_backward).unwrap(); let join_block = new_join_block(&mut bg, vec![copy_1, copy_2]).unwrap(); - let observe_block = block_misc::new_observe_block(&mut bg, join_block, Observe::Forward, Some(6.0)).unwrap(); + let observe_block = + block_misc::new_observe_block(&mut bg, join_block, Observe::Forward, Some(6.0)) + .unwrap(); bg.finalize(); bg.allocate_and_init_weights(&mi); - + let mut pb = bg.new_port_buffer(); let fb = fb_vec(); - slearn2 (&mut bg, &fb, &mut pb, true); - assert_eq!(pb.observations, vec![2.0, 3.0, 2.0, 3.0, - 12.0, 12.0 - ]); // correct backwards pass + slearn2(&mut bg, &fb, &mut pb, true); + assert_eq!(pb.observations, vec![2.0, 3.0, 2.0, 3.0, 12.0, 12.0]); // correct backwards pass - spredict2 (&mut bg, &fb, &mut pb, false); - assert_eq!(pb.observations, vec![2.0, 3.0, 2.0, 3.0, - 2.0, 3.0]); // on backward pass this are leftovers + spredict2(&mut bg, &fb, &mut pb, false); + assert_eq!(pb.observations, vec![2.0, 3.0, 2.0, 3.0, 2.0, 3.0]); // on backward pass this are leftovers } - - - - - } - - diff --git a/src/block_neural.rs b/src/block_neural.rs index 32c64403..914f74f7 100644 --- a/src/block_neural.rs +++ b/src/block_neural.rs @@ -1,32 +1,31 @@ -use std::any::Any; -use std::io; -use std::mem::{self, MaybeUninit}; -use rand_distr::{Normal, Distribution, Uniform}; +use rand_distr::{Distribution, Normal, Uniform}; +use rand_xoshiro::rand_core::RngCore; use rand_xoshiro::rand_core::SeedableRng; use rand_xoshiro::Xoshiro256PlusPlus; -use rand_xoshiro::rand_core::RngCore; +use std::any::Any; use std::error::Error; +use std::io; use std::io::Error as IOError; use std::io::ErrorKind; +use std::mem::{self, MaybeUninit}; use std::slice; - -use crate::optimizer; -use crate::regressor; -use crate::model_instance; -use crate::feature_buffer; -use crate::port_buffer; use crate::block_helpers; -use crate::graph; use crate::block_misc; +use crate::feature_buffer; +use crate::graph; +use crate::model_instance; +use crate::optimizer; +use crate::port_buffer; +use crate::regressor; +use block_helpers::{OptimizerData, Weight}; use optimizer::OptimizerTrait; use regressor::BlockTrait; -use block_helpers::{Weight, OptimizerData}; use blas::*; -const MAX_NUM_INPUTS:usize= 16000; -const USE_BLAS:bool = true; +const MAX_NUM_INPUTS: usize = 16000; +const USE_BLAS: bool = true; #[derive(PartialEq, Debug)] pub enum NeuronType { @@ -38,21 +37,19 @@ pub enum NeuronType { pub enum InitType { Xavier, Hu, -// RandomFirst1, -// RandomFirst10, + // RandomFirst1, + // RandomFirst10, One, Zero, } - - -pub struct BlockNeuronLayer { +pub struct BlockNeuronLayer { pub num_inputs: usize, pub input_offset: usize, pub output_offset: usize, pub weights_len: u32, // While FFM part keeps weight and accumulation together (since memory locality is the issue) - // for NN part it is actually preferrable to have it separately + // for NN part it is actually preferrable to have it separately pub weights: Vec, pub weights_optimizer: Vec>, pub optimizer: L, @@ -68,20 +65,18 @@ pub struct BlockNeuronLayer { dropout_threshold: u32, } - - - -fn new_neuronlayer_without_weights(mi: &model_instance::ModelInstance, - num_inputs: usize, - ntype: NeuronType, - num_neurons: usize, - init_type: InitType, - dropout: f32, - max_norm: f32, - layer_norm: bool, - ) -> Result, Box> { +fn new_neuronlayer_without_weights( + mi: &model_instance::ModelInstance, + num_inputs: usize, + ntype: NeuronType, + num_neurons: usize, + init_type: InitType, + dropout: f32, + max_norm: f32, + layer_norm: bool, +) -> Result, Box> { assert!(num_neurons > 0); - assert!((num_inputs as usize )< MAX_NUM_INPUTS); + assert!((num_inputs as usize) < MAX_NUM_INPUTS); assert!(num_inputs != 0); if dropout != 0.0 { @@ -102,7 +97,7 @@ fn new_neuronlayer_without_weights(mi: &model_instan num_neurons: num_neurons, init_type: init_type, dropout: dropout, - dropout_inv: 1.0/(1.0 - dropout), + dropout_inv: 1.0 / (1.0 - dropout), max_norm: max_norm, layer_norm: layer_norm, rng: Xoshiro256PlusPlus::seed_from_u64(0 as u64), @@ -110,68 +105,90 @@ fn new_neuronlayer_without_weights(mi: &model_instan dropout_threshold: ((u32::MAX as f64) * (dropout as f64)) as u32, }; - rg.optimizer.init(mi.nn_learning_rate, mi.nn_power_t, mi.nn_init_acc_gradient); + rg.optimizer + .init(mi.nn_learning_rate, mi.nn_power_t, mi.nn_init_acc_gradient); Ok(Box::new(rg)) } - - pub fn new_neuronlayer_block( - bg: &mut graph::BlockGraph, - mi: &model_instance::ModelInstance, - input: graph::BlockPtrOutput, - ntype: NeuronType, - num_neurons: usize, - init_type: InitType, - dropout: f32, - max_norm: f32, - layer_norm: bool, - ) -> Result> { - + bg: &mut graph::BlockGraph, + mi: &model_instance::ModelInstance, + input: graph::BlockPtrOutput, + ntype: NeuronType, + num_neurons: usize, + init_type: InitType, + dropout: f32, + max_norm: f32, + layer_norm: bool, +) -> Result> { let num_inputs = bg.get_num_output_values(vec![&input]); if ntype == NeuronType::Sum { return Err(Box::new(IOError::new(ErrorKind::Other, "You should not use new_neuronlayer_block with the type NeuronType::Sum, it makes no sense - use block_misc::new_sum_block()"))); } let block = match mi.optimizer { - model_instance::Optimizer::AdagradLUT => new_neuronlayer_without_weights:: (&mi, num_inputs, ntype, num_neurons, init_type, dropout, max_norm, layer_norm), - model_instance::Optimizer::AdagradFlex => new_neuronlayer_without_weights::(&mi, num_inputs, ntype, num_neurons, init_type, dropout, max_norm, layer_norm), - model_instance::Optimizer::SGD => new_neuronlayer_without_weights:: (&mi, num_inputs, ntype, num_neurons, init_type, dropout, max_norm, layer_norm) - }.unwrap(); + model_instance::Optimizer::AdagradLUT => { + new_neuronlayer_without_weights::( + &mi, + num_inputs, + ntype, + num_neurons, + init_type, + dropout, + max_norm, + layer_norm, + ) + } + model_instance::Optimizer::AdagradFlex => { + new_neuronlayer_without_weights::( + &mi, + num_inputs, + ntype, + num_neurons, + init_type, + dropout, + max_norm, + layer_norm, + ) + } + model_instance::Optimizer::SGD => { + new_neuronlayer_without_weights::( + &mi, + num_inputs, + ntype, + num_neurons, + init_type, + dropout, + max_norm, + layer_norm, + ) + } + } + .unwrap(); let mut block_outputs = bg.add_node(block, vec![input]).unwrap(); assert_eq!(block_outputs.len(), 1); Ok(block_outputs.pop().unwrap()) } - pub fn new_neuron_block( - bg: &mut graph::BlockGraph, - mi: &model_instance::ModelInstance, - input: graph::BlockPtrOutput, - ntype: NeuronType, - init_type: InitType, - ) -> Result> { + bg: &mut graph::BlockGraph, + mi: &model_instance::ModelInstance, + input: graph::BlockPtrOutput, + ntype: NeuronType, + init_type: InitType, +) -> Result> { match ntype { - NeuronType::Sum => block_misc::new_sum_block( bg, input), - _ => new_neuronlayer_block( bg, - mi, - input, - ntype, - 1, // a single neuron - init_type, - 0.0, // dropout - 0.0, // maxnorm - false, // layer norm - ), + NeuronType::Sum => block_misc::new_sum_block(bg, input), + _ => new_neuronlayer_block( + bg, mi, input, ntype, 1, // a single neuron + init_type, 0.0, // dropout + 0.0, // maxnorm + false, // layer norm + ), } } - - - -impl BlockTrait for BlockNeuronLayer - - { +impl BlockTrait for BlockNeuronLayer { fn as_any(&mut self) -> &mut dyn Any { self } @@ -180,122 +197,147 @@ impl BlockTrait for BlockNeuronLayer debug_assert!(self.output_offset != usize::MAX); debug_assert!(self.input_offset != usize::MAX); - assert!(self.weights_len != 0, "allocate_and_init_weights(): Have you forgotten to call set_num_inputs()?"); - self.weights = vec![Weight{weight:1.0}; self.weights_len as usize]; - self.weights_optimizer = vec![OptimizerData::{optimizer_data: self.optimizer.initial_data()}; self.weights_len as usize]; + assert!( + self.weights_len != 0, + "allocate_and_init_weights(): Have you forgotten to call set_num_inputs()?" + ); + self.weights = vec![Weight { weight: 1.0 }; self.weights_len as usize]; + self.weights_optimizer = vec![ + OptimizerData:: { + optimizer_data: self.optimizer.initial_data() + }; + self.weights_len as usize + ]; self.rng_scratchpad = vec![0; self.num_neurons]; // We need to seed each layer with a separate seed... how? // by the time we call this function input_offset and output_offset are set and are unique. L - self.rng = Xoshiro256PlusPlus::seed_from_u64((self.input_offset * self.output_offset + self.num_inputs + self.weights_len as usize) as u64); + self.rng = Xoshiro256PlusPlus::seed_from_u64( + (self.input_offset * self.output_offset + self.num_inputs + self.weights_len as usize) + as u64, + ); match self.init_type { - InitType::Xavier => { - let bound = 6.0_f64.sqrt() / ((self.num_inputs+self.num_neurons) as f64).sqrt(); + let bound = 6.0_f64.sqrt() / ((self.num_inputs + self.num_neurons) as f64).sqrt(); let normal = Uniform::new(-bound, bound); for i in 0..self.num_neurons * self.num_inputs { - self.weights[i as usize].weight = normal.sample(&mut self.rng) as f32; - } - }, + self.weights[i as usize].weight = normal.sample(&mut self.rng) as f32; + } + } InitType::Hu => { - let normal = Normal::new(0.0, (2.0/self.num_inputs as f64).sqrt() as f64).unwrap(); + let normal = + Normal::new(0.0, (2.0 / self.num_inputs as f64).sqrt() as f64).unwrap(); for i in 0..self.num_neurons * self.num_inputs { - self.weights[i as usize].weight = normal.sample(&mut self.rng) as f32; - } + self.weights[i as usize].weight = normal.sample(&mut self.rng) as f32; + } + } + // InitType::RandomFirst1 => { for i in 0..self.num_inputs { self.weights[i as usize].weight = 1.0}}, + // InitType::RandomFirst10 => { for i in 0..self.num_inputs { self.weights[i as usize].weight = 0.0}; self.weights[0].weight = 1.0;}, + InitType::One => { + for i in 0..self.weights_len { + self.weights[i as usize].weight = 1.0 + } + } + InitType::Zero => { + for i in 0..self.weights_len { + self.weights[i as usize].weight = 0.0 + } } -// InitType::RandomFirst1 => { for i in 0..self.num_inputs { self.weights[i as usize].weight = 1.0}}, -// InitType::RandomFirst10 => { for i in 0..self.num_inputs { self.weights[i as usize].weight = 0.0}; self.weights[0].weight = 1.0;}, - InitType::One => { for i in 0..self.weights_len { self.weights[i as usize].weight = 1.0}}, - InitType::Zero => { for i in 0..self.weights_len { self.weights[i as usize].weight = 0.0}}, } - - - // Bias terms are always initialized to zero + + // Bias terms are always initialized to zero for i in 0..self.num_neurons { self.weights[(self.num_neurons * self.num_inputs + i) as usize].weight = 0.0 } - } - fn get_num_output_slots(&self) -> usize {1} - + fn get_num_output_slots(&self) -> usize { + 1 + } fn get_num_output_values(&self, output: graph::OutputSlot) -> usize { assert!(output.get_output_index() == 0); self.num_neurons } - - fn set_input_offset(&mut self, input: graph::InputSlot, offset: usize) { + fn set_input_offset(&mut self, input: graph::InputSlot, offset: usize) { assert!(input.get_input_index() == 0); self.input_offset = offset; } - fn set_output_offset(&mut self, output: graph::OutputSlot, offset: usize) { + fn set_output_offset(&mut self, output: graph::OutputSlot, offset: usize) { assert!(output.get_output_index() == 0); self.output_offset = offset; } - - - #[inline(always)] - fn forward_backward(&mut self, - further_blocks: &mut [Box], - fb: &feature_buffer::FeatureBuffer, - pb: &mut port_buffer::PortBuffer, - update:bool) { + fn forward_backward( + &mut self, + further_blocks: &mut [Box], + fb: &feature_buffer::FeatureBuffer, + pb: &mut port_buffer::PortBuffer, + update: bool, + ) { debug_assert!(self.num_inputs > 0); debug_assert!(self.output_offset != usize::MAX); debug_assert!(self.input_offset != usize::MAX); - + unsafe { let bias_offset = self.num_inputs * self.num_neurons; { - let (input_tape, output_tape) = block_helpers::get_input_output_borrows(&mut pb.tape, - self.input_offset, self.num_inputs, - self.output_offset, self.num_neurons); - + let (input_tape, output_tape) = block_helpers::get_input_output_borrows( + &mut pb.tape, + self.input_offset, + self.num_inputs, + self.output_offset, + self.num_neurons, + ); + // If we are in pure prediction mode ( let dropout_inv = match update { true => self.dropout_inv, - false => 1.0 + false => 1.0, }; if !USE_BLAS { - let mut j_offset:usize = 0; + let mut j_offset: usize = 0; for j in 0..self.num_neurons { - let mut wsum:f32 = self.weights.get_unchecked(bias_offset + j).weight; // bias term - for i in 0..self.num_inputs { - wsum += input_tape.get_unchecked(i) * self.weights.get_unchecked(i + j_offset as usize).weight; + let mut wsum: f32 = self.weights.get_unchecked(bias_offset + j).weight; // bias term + for i in 0..self.num_inputs { + wsum += input_tape.get_unchecked(i) + * self.weights.get_unchecked(i + j_offset as usize).weight; } j_offset += self.num_inputs; *output_tape.get_unchecked_mut(j) = wsum * self.dropout_inv; } } else { - // This is actually speed things up considerably. - output_tape.copy_from_slice(std::mem::transmute::<&[Weight], &[f32]>(self.weights.get_unchecked(bias_offset..))); + // This is actually speed things up considerably. + output_tape.copy_from_slice(std::mem::transmute::<&[Weight], &[f32]>( + self.weights.get_unchecked(bias_offset..), + )); sgemv( - b'T', // trans: u8, - self.num_inputs as i32, // m: i32, - self.num_neurons as i32, // n: i32, - dropout_inv, // alpha: f32, - std::mem::transmute::<&[Weight], &[f32]>(self.weights.get_unchecked(0..)), // a: &[f32], - self.num_inputs as i32, // lda: i32, - &input_tape.get_unchecked(0..),// x: &[f32], - 1, // incx: i32, - 1.0, // beta: f32, - output_tape.get_unchecked_mut(0..), //y: &mut [f32], - 1, // incy: i32 - ) + b'T', // trans: u8, + self.num_inputs as i32, // m: i32, + self.num_neurons as i32, // n: i32, + dropout_inv, // alpha: f32, + std::mem::transmute::<&[Weight], &[f32]>(self.weights.get_unchecked(0..)), // a: &[f32], + self.num_inputs as i32, // lda: i32, + &input_tape.get_unchecked(0..), // x: &[f32], + 1, // incx: i32, + 1.0, // beta: f32, + output_tape.get_unchecked_mut(0..), //y: &mut [f32], + 1, // incy: i32 + ) } if update { // In case we are doing doing learning, and we have a dropout - + if false && self.dropout != 0.0 { - let mut fill_view:&mut [u8] = slice::from_raw_parts_mut(self.rng_scratchpad.as_mut_ptr() as *mut u8, - self.num_neurons * mem::size_of::()); - + let mut fill_view: &mut [u8] = slice::from_raw_parts_mut( + self.rng_scratchpad.as_mut_ptr() as *mut u8, + self.num_neurons * mem::size_of::(), + ); + //let mut fill_view: [u8; 100] = [0; 100]; // This one is from the "You won't believe it till you measure it 10 times and even then you will suspect yourself not the compiler" // For some reason call to this function in a block that NEVER executes, slows down the code by 50% @@ -303,19 +345,16 @@ impl BlockTrait for BlockNeuronLayer // I thought It might be related to from_raw_parts above, but it isn't // Attempted fix of going through a #[inline(never)] function did not work - self.rng.fill_bytes(&mut fill_view); + self.rng.fill_bytes(&mut fill_view); //fill_bytes(&mut self.rng, &mut fill_view); for j in 0..self.num_neurons { if *self.rng_scratchpad.get_unchecked(j) < self.dropout_threshold { *output_tape.get_unchecked_mut(j) = 0.0; } - } + } } - } } - - block_helpers::forward_backward(further_blocks, fb, pb, update); @@ -323,53 +362,72 @@ impl BlockTrait for BlockNeuronLayer if self.neuron_type == NeuronType::WeightedSum { // first we need to initialize inputs to zero // TODO - what to think about this buffer - let mut output_errors: [f32; MAX_NUM_INPUTS] = MaybeUninit::uninit().assume_init(); - output_errors.get_unchecked_mut(0..self.num_inputs).fill(0.0); + let mut output_errors: [f32; MAX_NUM_INPUTS] = + MaybeUninit::uninit().assume_init(); + output_errors + .get_unchecked_mut(0..self.num_inputs) + .fill(0.0); + + let (input_tape, output_tape) = block_helpers::get_input_output_borrows( + &mut pb.tape, + self.input_offset, + self.num_inputs, + self.output_offset, + self.num_neurons, + ); - let (input_tape, output_tape) = block_helpers::get_input_output_borrows(&mut pb.tape, - self.input_offset, self.num_inputs, - self.output_offset, self.num_neurons); - for j in 0..self.num_neurons as usize { - if self.dropout != 0.0 && *self.rng_scratchpad.get_unchecked(j) < self.dropout_threshold { - // println!("B:j {}", j); - continue - } + if self.dropout != 0.0 + && *self.rng_scratchpad.get_unchecked(j) < self.dropout_threshold + { + // println!("B:j {}", j); + continue; + } + + let general_gradient = output_tape.get_unchecked(j) * self.dropout_inv; + + let j_offset = j * self.num_inputs as usize; + for i in 0..self.num_inputs as usize { + let feature_value = input_tape.get_unchecked(i); + let gradient = general_gradient * feature_value; + let update = self.optimizer.calculate_update( + gradient, + &mut self + .weights_optimizer + .get_unchecked_mut(i + j_offset) + .optimizer_data, + ); + *output_errors.get_unchecked_mut(i) += + self.weights.get_unchecked(i + j_offset).weight * general_gradient; + self.weights.get_unchecked_mut(i + j_offset).weight -= update; + } + { + // Updating bias term: + let gradient = general_gradient * 1.0; + let update = self.optimizer.calculate_update( + gradient, + &mut self + .weights_optimizer + .get_unchecked_mut(bias_offset + j) + .optimizer_data, + ); + self.weights.get_unchecked_mut(bias_offset + j).weight -= update; + } - let general_gradient = output_tape.get_unchecked(j) * self.dropout_inv; - - let j_offset = j * self.num_inputs as usize; + if self.max_norm != 0.0 && fb.example_number % 10 == 0 { + let mut wsquaredsum = 0.000001; // Epsilon for i in 0..self.num_inputs as usize { - let feature_value = input_tape.get_unchecked(i); - let gradient = general_gradient * feature_value; - let update = self.optimizer.calculate_update(gradient, - &mut self.weights_optimizer.get_unchecked_mut(i + j_offset).optimizer_data); - *output_errors.get_unchecked_mut(i) += self.weights.get_unchecked(i + j_offset).weight * general_gradient; - self.weights.get_unchecked_mut(i + j_offset).weight -= update; + let w = self.weights.get_unchecked_mut(i + j_offset).weight; + wsquaredsum += w * w; } - { - // Updating bias term: - let gradient = general_gradient * 1.0; - let update = self.optimizer.calculate_update(gradient, - &mut self.weights_optimizer.get_unchecked_mut(bias_offset + j).optimizer_data); - self.weights.get_unchecked_mut(bias_offset + j).weight -= update; - } - - if self.max_norm != 0.0 && fb.example_number % 10 == 0 { - let mut wsquaredsum = 0.000001; // Epsilon + let norm = wsquaredsum.sqrt(); + if norm > self.max_norm { + let scaling = self.max_norm / norm; for i in 0..self.num_inputs as usize { - let w = self.weights.get_unchecked_mut(i + j_offset).weight; - wsquaredsum += w * w; + self.weights.get_unchecked_mut(i + j_offset).weight *= scaling; } - let norm = wsquaredsum.sqrt(); - if norm > self.max_norm { - let scaling = self.max_norm / norm; - for i in 0..self.num_inputs as usize { - self.weights.get_unchecked_mut(i + j_offset).weight *= scaling; - } - } } - + } } if self.layer_norm && fb.example_number % 10 == 0 { let mut sum: f32 = 0.0; @@ -380,311 +438,314 @@ impl BlockTrait for BlockNeuronLayer sum += w; sumsqr += w * w; } - let var1 = (sumsqr - sum*sum/bias_offset as f32) / - bias_offset as f32; + let var1 = (sumsqr - sum * sum / bias_offset as f32) / bias_offset as f32; let var2 = var1.sqrt(); for i in 0..bias_offset { - self.weights.get_unchecked_mut(i).weight /=var2; + self.weights.get_unchecked_mut(i).weight /= var2; } - } - input_tape.copy_from_slice(output_errors.get_unchecked(0..self.num_inputs)); - - } - } - } // unsafe end } - - fn forward(&self, further_blocks: &[Box], - fb: &feature_buffer::FeatureBuffer, - pb: &mut port_buffer::PortBuffer, - ) { - unsafe { + fn forward( + &self, + further_blocks: &[Box], + fb: &feature_buffer::FeatureBuffer, + pb: &mut port_buffer::PortBuffer, + ) { + unsafe { let frandseed = fb.example_number * fb.example_number; let bias_offset = self.num_inputs * self.num_neurons; - let (input_tape, output_tape) = block_helpers::get_input_output_borrows(&mut pb.tape, - self.input_offset, self.num_inputs, - self.output_offset, self.num_neurons); - + let (input_tape, output_tape) = block_helpers::get_input_output_borrows( + &mut pb.tape, + self.input_offset, + self.num_inputs, + self.output_offset, + self.num_neurons, + ); + if !USE_BLAS { - let mut j_offset:usize = 0; + let mut j_offset: usize = 0; for j in 0..self.num_neurons { - let mut wsum:f32 = self.weights.get_unchecked(bias_offset + j).weight; // bias term - for i in 0..self.num_inputs { - wsum += input_tape.get_unchecked(i) * self.weights.get_unchecked(i + j_offset as usize).weight; + let mut wsum: f32 = self.weights.get_unchecked(bias_offset + j).weight; // bias term + for i in 0..self.num_inputs { + wsum += input_tape.get_unchecked(i) + * self.weights.get_unchecked(i + j_offset as usize).weight; } j_offset += self.num_inputs; *output_tape.get_unchecked_mut(j) = wsum; } } else { - // This is actually speed things up considerably. - output_tape.copy_from_slice(std::mem::transmute::<&[Weight], &[f32]>(self.weights.get_unchecked(bias_offset..))); + // This is actually speed things up considerably. + output_tape.copy_from_slice(std::mem::transmute::<&[Weight], &[f32]>( + self.weights.get_unchecked(bias_offset..), + )); sgemv( - b'T',// trans: u8, - self.num_inputs as i32, // m: i32, - self.num_neurons as i32, // n: i32, - 1.0, // alpha: f32, - std::mem::transmute::<&[Weight], &[f32]>(self.weights.get_unchecked(0..)), // a: &[f32], - self.num_inputs as i32, //lda: i32, - &input_tape.get_unchecked(0..),// x: &[f32], - 1, //incx: i32, - 1.0, // beta: f32, - output_tape.get_unchecked_mut(0..), //y: &mut [f32], - 1,//incy: i32 - ) - } + b'T', // trans: u8, + self.num_inputs as i32, // m: i32, + self.num_neurons as i32, // n: i32, + 1.0, // alpha: f32, + std::mem::transmute::<&[Weight], &[f32]>(self.weights.get_unchecked(0..)), // a: &[f32], + self.num_inputs as i32, //lda: i32, + &input_tape.get_unchecked(0..), // x: &[f32], + 1, //incx: i32, + 1.0, // beta: f32, + output_tape.get_unchecked_mut(0..), //y: &mut [f32], + 1, //incy: i32 + ) + } block_helpers::forward(further_blocks, fb, pb); - - - } // unsafe end - } - + fn get_serialized_len(&self) -> usize { return self.weights_len as usize; } - - fn read_weights_from_buf(&mut self, input_bufreader: &mut dyn io::Read) -> Result<(), Box> { + + fn read_weights_from_buf( + &mut self, + input_bufreader: &mut dyn io::Read, + ) -> Result<(), Box> { block_helpers::read_weights_from_buf(&mut self.weights, input_bufreader)?; block_helpers::read_weights_from_buf(&mut self.weights_optimizer, input_bufreader)?; Ok(()) } - fn write_weights_to_buf(&self, output_bufwriter: &mut dyn io::Write) -> Result<(), Box> { + fn write_weights_to_buf( + &self, + output_bufwriter: &mut dyn io::Write, + ) -> Result<(), Box> { block_helpers::write_weights_to_buf(&self.weights, output_bufwriter)?; block_helpers::write_weights_to_buf(&self.weights_optimizer, output_bufwriter)?; Ok(()) - } - fn read_weights_from_buf_into_forward_only(&self, input_bufreader: &mut dyn io::Read, forward: &mut Box) -> Result<(), Box> { - let mut forward = forward.as_any().downcast_mut::>().unwrap(); + fn read_weights_from_buf_into_forward_only( + &self, + input_bufreader: &mut dyn io::Read, + forward: &mut Box, + ) -> Result<(), Box> { + let mut forward = forward + .as_any() + .downcast_mut::>() + .unwrap(); block_helpers::read_weights_from_buf(&mut forward.weights, input_bufreader)?; - block_helpers::skip_weights_from_buf(self.weights_len as usize, &self.weights_optimizer, input_bufreader)?; + block_helpers::skip_weights_from_buf( + self.weights_len as usize, + &self.weights_optimizer, + input_bufreader, + )?; Ok(()) } /// Sets internal state of weights based on some completely object-dependent parameters - fn testing_set_weights(&mut self, aa: i32, bb: i32, index: usize, w: &[f32]) -> Result<(), Box> { + fn testing_set_weights( + &mut self, + aa: i32, + bb: i32, + index: usize, + w: &[f32], + ) -> Result<(), Box> { self.weights[index].weight = w[0]; self.weights_optimizer[index].optimizer_data = self.optimizer.initial_data(); Ok(()) } } - - - - - - - - - mod tests { // Note this useful idiom: importing names from outer (for mod tests) scope. use super::*; + use crate::assert_epsilon; use crate::block_misc; - use crate::model_instance::Optimizer; + use crate::block_misc::Observe; use crate::feature_buffer; use crate::graph::BlockGraph; - use block_helpers::{slearn2}; - use crate::block_misc::Observe; - use crate::assert_epsilon; + use crate::model_instance::Optimizer; + use block_helpers::slearn2; fn fb_vec() -> feature_buffer::FeatureBuffer { feature_buffer::FeatureBuffer { - label: 0.0, - example_importance: 1.0, - example_number: 0, - lr_buffer: Vec::new(), - ffm_buffer: Vec::new(), - ffm_fields_count: 0, + label: 0.0, + example_importance: 1.0, + example_number: 0, + lr_buffer: Vec::new(), + ffm_buffer: Vec::new(), + ffm_fields_count: 0, } } - #[test] fn test_simple() { - let mut mi = model_instance::ModelInstance::new_empty().unwrap(); + let mut mi = model_instance::ModelInstance::new_empty().unwrap(); mi.nn_learning_rate = 0.1; mi.nn_power_t = 0.0; mi.optimizer = Optimizer::SGD; - + let mut bg = BlockGraph::new(); let input_block = block_misc::new_const_block(&mut bg, vec![2.0]).unwrap(); - let neuron_block = new_neuronlayer_block(&mut bg, - &mi, - input_block, - NeuronType::WeightedSum, - 1, - InitType::One, - 0.0, // dropout - 0.0, // max norm - false, - ).unwrap(); - let observe_block = block_misc::new_observe_block(&mut bg, neuron_block, Observe::Forward, Some(1.0)).unwrap(); + let neuron_block = new_neuronlayer_block( + &mut bg, + &mi, + input_block, + NeuronType::WeightedSum, + 1, + InitType::One, + 0.0, // dropout + 0.0, // max norm + false, + ) + .unwrap(); + let observe_block = + block_misc::new_observe_block(&mut bg, neuron_block, Observe::Forward, Some(1.0)) + .unwrap(); bg.finalize(); bg.allocate_and_init_weights(&mi); - + let mut pb = bg.new_port_buffer(); - + let fb = fb_vec(); - assert_epsilon!(slearn2 (&mut bg, &fb, &mut pb, true), 2.0); - assert_epsilon!(slearn2 (&mut bg, &fb, &mut pb, true), 1.5); + assert_epsilon!(slearn2(&mut bg, &fb, &mut pb, true), 2.0); + assert_epsilon!(slearn2(&mut bg, &fb, &mut pb, true), 1.5); } #[test] fn test_two_neurons() { - let mut mi = model_instance::ModelInstance::new_empty().unwrap(); + let mut mi = model_instance::ModelInstance::new_empty().unwrap(); mi.nn_learning_rate = 0.1; mi.nn_power_t = 0.0; mi.optimizer = Optimizer::SGD; - - + let NUM_NEURONS = 2; let mut bg = BlockGraph::new(); let input_block = block_misc::new_const_block(&mut bg, vec![2.0]).unwrap(); - let neuron_block = new_neuronlayer_block(&mut bg, - &mi, - input_block, - NeuronType::WeightedSum, - NUM_NEURONS, - InitType::One, - 0.0, // dropout - 0.0, // max norm - false, // layer norm - ).unwrap(); - let observe_block = block_misc::new_observe_block(&mut bg, neuron_block, Observe::Forward, Some(1.0)).unwrap(); + let neuron_block = new_neuronlayer_block( + &mut bg, + &mi, + input_block, + NeuronType::WeightedSum, + NUM_NEURONS, + InitType::One, + 0.0, // dropout + 0.0, // max norm + false, // layer norm + ) + .unwrap(); + let observe_block = + block_misc::new_observe_block(&mut bg, neuron_block, Observe::Forward, Some(1.0)) + .unwrap(); bg.finalize(); bg.allocate_and_init_weights(&mi); - + let mut pb = bg.new_port_buffer(); - + let fb = fb_vec(); - assert_epsilon!(slearn2 (&mut bg, &fb, &mut pb, true), 2.0); + assert_epsilon!(slearn2(&mut bg, &fb, &mut pb, true), 2.0); // what do we expect: // on tape 0 input of 2.0 will be replaced with the gradient of 2.0 // on tape 1 input has been consumed by returning function // on tape 2 the output was consumed by slearn - assert_eq!(pb.observations.len(), NUM_NEURONS as usize); + assert_eq!(pb.observations.len(), NUM_NEURONS as usize); assert_eq!(pb.observations[0], 2.0); // since we are using identity loss function, only one was consumed by slearn assert_eq!(pb.observations[1], 2.0); // since we are using identity loss function, only one was consumed by slearn - assert_epsilon!(slearn2 (&mut bg, &fb, &mut pb, false), 1.5); - - + assert_epsilon!(slearn2(&mut bg, &fb, &mut pb, false), 1.5); } // #[test] // fn test_neuron() { - // let mut mi = model_instance::ModelInstance::new_empty().unwrap(); + // let mut mi = model_instance::ModelInstance::new_empty().unwrap(); // mi.nn_learning_rate = 0.1; // mi.nn_power_t = 0.0; // mi.nn_init_acc_gradient = mi.init_acc_gradient; // mi.optimizer = Optimizer::SGD; - - + // let mut bg = BlockGraph::new(); // let input_block = block_misc::new_const_block(&mut bg, vec![2.0]).unwrap(); // let neuron_block = new_neuron_block(&mut bg, &mi, input_block, NeuronType::WeightedSum, InitType::One).unwrap(); // let observe_block = block_misc::new_observe_block(&mut bg, neuron_block, Observe::Forward, Some(1.0)).unwrap(); // bg.finalize(); // bg.allocate_and_init_weights(&mi); - + // let mut pb = bg.new_port_buffer(); // let fb = fb_vec(); // assert_epsilon!(slearn2 (&mut bg, &fb, &mut pb, true), 2.0); // assert_eq!(pb.observations.len(), 1); // assert_epsilon!(slearn2 (&mut bg, &fb, &mut pb, true), 1.5); // } -/* - #[test] - fn test_dropout() { - let mut mi = model_instance::ModelInstance::new_empty().unwrap(); - - let NUM_NEURONS = 6; - let mut bg = BlockGraph::new(); - let input_block = block_misc::new_const_block(&mut bg, vec![3.0]).unwrap(); - let observe_block_backward = block_misc::new_observe_block(&mut bg, input_block, Observe::Backward, None).unwrap(); - let neuron_block = new_neuronlayer_block(&mut bg, - &mi, - observe_block_backward, - NeuronType::WeightedSum, - NUM_NEURONS, - InitType::One, - 0.5, // dropout - 0.0, // max norm - false, - ).unwrap(); - let observe_block = block_misc::new_observe_block(&mut bg, neuron_block, Observe::Forward, Some(3.0)).unwrap(); - bg.finalize(); - bg.allocate_and_init_weights(&mi); - - let mut pb = bg.new_port_buffer(); - - let fb = fb_vec(); - - spredict2 (&mut bg, &fb, &mut pb, false); - assert_eq!(pb.observations, vec![3.0, 3.0, 3.0, 3.0, 3.0, 3.0, // mostly deterministic due to specific PRNG use, - 3.0 // Untouched output when forward-only - ]); - - slearn2 (&mut bg, &fb, &mut pb, true); - assert_eq!(pb.observations, vec![6.0, 0.0, 0.0, 0.0, 6.0, 6.0, // mostly deterministic due to specific PRNG use - 18.0 // scaled backprop - ]); -// println!("O: {:?}", pb.observations); - - - - } - -*/ + /* + #[test] + fn test_dropout() { + let mut mi = model_instance::ModelInstance::new_empty().unwrap(); + + let NUM_NEURONS = 6; + let mut bg = BlockGraph::new(); + let input_block = block_misc::new_const_block(&mut bg, vec![3.0]).unwrap(); + let observe_block_backward = block_misc::new_observe_block(&mut bg, input_block, Observe::Backward, None).unwrap(); + let neuron_block = new_neuronlayer_block(&mut bg, + &mi, + observe_block_backward, + NeuronType::WeightedSum, + NUM_NEURONS, + InitType::One, + 0.5, // dropout + 0.0, // max norm + false, + ).unwrap(); + let observe_block = block_misc::new_observe_block(&mut bg, neuron_block, Observe::Forward, Some(3.0)).unwrap(); + bg.finalize(); + bg.allocate_and_init_weights(&mi); + + let mut pb = bg.new_port_buffer(); + + let fb = fb_vec(); + + spredict2 (&mut bg, &fb, &mut pb, false); + assert_eq!(pb.observations, vec![3.0, 3.0, 3.0, 3.0, 3.0, 3.0, // mostly deterministic due to specific PRNG use, + 3.0 // Untouched output when forward-only + ]); + + slearn2 (&mut bg, &fb, &mut pb, true); + assert_eq!(pb.observations, vec![6.0, 0.0, 0.0, 0.0, 6.0, 6.0, // mostly deterministic due to specific PRNG use + 18.0 // scaled backprop + ]); + // println!("O: {:?}", pb.observations); + } + */ + + /* #[test] + fn test_segm() { + unsafe { + let input_matrix:Vec = vec![1.0, 2.0, + 3.0, 4.0, + 5.0, 6.0]; + let input_vec: Vec = vec![2.0, 1.0]; + let mut output_vec : Vec = vec![0.0, 0.0, 0.0]; + + sgemv( + b'T',// trans: u8, + 2, // m: i32, // num_neurons + 3, // n: i32, // num inputs + 1.0, // alpha: f32, + &input_matrix, // a: &[f32], + 2, //lda: i32, + &input_vec,// x: &[f32], + 1, //incx: i32, + 1.0, // beta: f32, + &mut output_vec, //y: &mut [f32], + 1,//incy: i32 + ); + + println!("Output : {:?}", output_vec); - - - -/* #[test] - fn test_segm() { - unsafe { - let input_matrix:Vec = vec![1.0, 2.0, - 3.0, 4.0, - 5.0, 6.0]; - let input_vec: Vec = vec![2.0, 1.0]; - let mut output_vec : Vec = vec![0.0, 0.0, 0.0]; - - sgemv( - b'T',// trans: u8, - 2, // m: i32, // num_neurons - 3, // n: i32, // num inputs - 1.0, // alpha: f32, - &input_matrix, // a: &[f32], - 2, //lda: i32, - &input_vec,// x: &[f32], - 1, //incx: i32, - 1.0, // beta: f32, - &mut output_vec, //y: &mut [f32], - 1,//incy: i32 - ); - - println!("Output : {:?}", output_vec); - - } - } -*/ + } + } + */ } - - - diff --git a/src/block_normalize.rs b/src/block_normalize.rs index 1033252d..5b642a28 100644 --- a/src/block_normalize.rs +++ b/src/block_normalize.rs @@ -1,31 +1,31 @@ use std::any::Any; use std::error::Error; -use crate::regressor; -use crate::model_instance; +use crate::block_helpers; use crate::feature_buffer; +use crate::graph; +use crate::model_instance; use crate::port_buffer; +use crate::regressor; use regressor::BlockTrait; -use crate::block_helpers; -use crate::graph; -const EPS:f32 = 1e-2; +const EPS: f32 = 1e-2; -pub struct BlockNormalize { +pub struct BlockNormalize { pub num_inputs: usize, pub input_offset: usize, pub output_offset: usize, } - -// This is purely variance normalization as described in +// This is purely variance normalization as described in // https://arxiv.org/pdf/2006.12753.pdf // Early results show no improvements for normalization od neural layers -pub fn new_normalize_layer_block( bg: &mut graph::BlockGraph, - mi: &model_instance::ModelInstance, - input: graph::BlockPtrOutput - ) -> Result> { +pub fn new_normalize_layer_block( + bg: &mut graph::BlockGraph, + mi: &model_instance::ModelInstance, + input: graph::BlockPtrOutput, +) -> Result> { let num_inputs = bg.get_num_output_values(vec![&input]); assert!(num_inputs != 0); let mut block = Box::new(BlockNormalize { @@ -38,147 +38,144 @@ pub fn new_normalize_layer_block( bg: &mut graph::BlockGraph, Ok(block_outputs.pop().unwrap()) } - - - - - -impl BlockTrait for BlockNormalize - - { +impl BlockTrait for BlockNormalize { fn as_any(&mut self) -> &mut dyn Any { self } - fn allocate_and_init_weights(&mut self, mi: &model_instance::ModelInstance) { - } - - fn get_num_output_slots(&self) -> usize {1} + fn allocate_and_init_weights(&mut self, mi: &model_instance::ModelInstance) {} + fn get_num_output_slots(&self) -> usize { + 1 + } fn get_num_output_values(&self, output: graph::OutputSlot) -> usize { assert!(output.get_output_index() == 0); - return self.num_inputs + return self.num_inputs; } - - fn set_input_offset(&mut self, input: graph::InputSlot, offset: usize) { + + fn set_input_offset(&mut self, input: graph::InputSlot, offset: usize) { assert!(input.get_input_index() == 0); self.input_offset = offset; } - fn set_output_offset(&mut self, output: graph::OutputSlot, offset: usize) { + fn set_output_offset(&mut self, output: graph::OutputSlot, offset: usize) { assert!(output.get_output_index() == 0); self.output_offset = offset; } - #[inline(always)] - fn forward_backward(&mut self, - further_blocks: &mut [Box], - fb: &feature_buffer::FeatureBuffer, - pb: &mut port_buffer::PortBuffer, - update:bool) { + fn forward_backward( + &mut self, + further_blocks: &mut [Box], + fb: &feature_buffer::FeatureBuffer, + pb: &mut port_buffer::PortBuffer, + update: bool, + ) { debug_assert!(self.output_offset != usize::MAX); debug_assert!(self.input_offset != usize::MAX); debug_assert!(self.num_inputs > 0); - + unsafe { - let mut mean:f32 = 0.0; - for i in 0..self.num_inputs as usize { - mean+= *pb.tape.get_unchecked_mut(self.input_offset + i); + let mut mean: f32 = 0.0; + for i in 0..self.num_inputs as usize { + mean += *pb.tape.get_unchecked_mut(self.input_offset + i); } mean /= self.num_inputs as f32; let meansq = mean * mean; - let mut variance:f32 = 0.0; + let mut variance: f32 = 0.0; for i in 0..self.num_inputs as usize { let w = meansq - *pb.tape.get_unchecked_mut(self.input_offset + i); - variance += w*w; + variance += w * w; } variance += EPS; variance /= self.num_inputs as f32; variance = variance.sqrt(); - -// println!("var1: {}, var2: {}, sum: {}, sumsqr: {}", var1, var2, sum, sumsqr); - let variance_inv = 1.0/ variance; -// let avg = sum / self.num_inputs as f32; + + // println!("var1: {}, var2: {}, sum: {}, sumsqr: {}", var1, var2, sum, sumsqr); + let variance_inv = 1.0 / variance; + // let avg = sum / self.num_inputs as f32; for i in 0..self.num_inputs { - *pb.tape.get_unchecked_mut(self.output_offset + i) = (*pb.tape.get_unchecked(self.input_offset + i) - mean) * variance_inv; + *pb.tape.get_unchecked_mut(self.output_offset + i) = + (*pb.tape.get_unchecked(self.input_offset + i) - mean) * variance_inv; } block_helpers::forward_backward(further_blocks, fb, pb, update); - + if update { for i in 0..self.num_inputs { - *pb.tape.get_unchecked_mut (self.input_offset + i) = *pb.tape.get_unchecked_mut(self.output_offset + i) * variance_inv; + *pb.tape.get_unchecked_mut(self.input_offset + i) = + *pb.tape.get_unchecked_mut(self.output_offset + i) * variance_inv; } - } } // unsafe end } - - fn forward(&self, further_blocks: &[Box], - fb: &feature_buffer::FeatureBuffer, - pb: &mut port_buffer::PortBuffer, - ) { + + fn forward( + &self, + further_blocks: &[Box], + fb: &feature_buffer::FeatureBuffer, + pb: &mut port_buffer::PortBuffer, + ) { debug_assert!(self.output_offset != usize::MAX); debug_assert!(self.input_offset != usize::MAX); debug_assert!(self.num_inputs > 0); - + unsafe { -/* let mut sum: f32 = 0.0; - let mut sumsqr: f32 = 0.0; - let K = 1.0; - for i in 0..self.num_inputs as usize { - let w = *pb.tape.get_unchecked_mut(self.input_offset + i) - K; - sum += w; - sumsqr += w * w; - } - let var1 = (sumsqr - sum*sum/self.num_inputs as f32) / - self.num_inputs as f32 + EPS; - let var2 = var1.sqrt(); -// println!("var1: {}, var2: {}, sum: {}, sumsqr: {}", var1, var2, sum, sumsqr); -*/ - let mut mean:f32 = 0.0; - for i in 0..self.num_inputs as usize { - mean+= *pb.tape.get_unchecked_mut(self.input_offset + i); + /* let mut sum: f32 = 0.0; + let mut sumsqr: f32 = 0.0; + let K = 1.0; + for i in 0..self.num_inputs as usize { + let w = *pb.tape.get_unchecked_mut(self.input_offset + i) - K; + sum += w; + sumsqr += w * w; + } + let var1 = (sumsqr - sum*sum/self.num_inputs as f32) / + self.num_inputs as f32 + EPS; + let var2 = var1.sqrt(); + // println!("var1: {}, var2: {}, sum: {}, sumsqr: {}", var1, var2, sum, sumsqr); + */ + let mut mean: f32 = 0.0; + for i in 0..self.num_inputs as usize { + mean += *pb.tape.get_unchecked_mut(self.input_offset + i); } mean /= self.num_inputs as f32; let meansq = mean * mean; - let mut variance:f32 = 0.0; + let mut variance: f32 = 0.0; for i in 0..self.num_inputs as usize { let w = meansq - *pb.tape.get_unchecked_mut(self.input_offset + i); - variance += w*w; + variance += w * w; } variance += EPS; variance /= self.num_inputs as f32; variance = variance.sqrt(); - let var3 = 1.0/ variance; + let var3 = 1.0 / variance; for i in 0..self.num_inputs { - *pb.tape.get_unchecked_mut(self.output_offset + i) = *pb.tape.get_unchecked(self.input_offset + i) * var3; + *pb.tape.get_unchecked_mut(self.output_offset + i) = + *pb.tape.get_unchecked(self.input_offset + i) * var3; } block_helpers::forward(further_blocks, fb, pb); } // unsafe end } - } -pub struct BlockStopBackward { +pub struct BlockStopBackward { pub num_inputs: usize, pub input_offset: usize, pub output_offset: usize, } - -// This is purely variance normalization as described in +// This is purely variance normalization as described in // https://arxiv.org/pdf/2006.12753.pdf // Early results show no improvements for normalization od neural layers -pub fn new_stop_block( bg: &mut graph::BlockGraph, - mi: &model_instance::ModelInstance, - input: graph::BlockPtrOutput - ) -> Result> { +pub fn new_stop_block( + bg: &mut graph::BlockGraph, + mi: &model_instance::ModelInstance, + input: graph::BlockPtrOutput, +) -> Result> { let num_inputs = bg.get_num_output_values(vec![&input]); debug_assert!(num_inputs != 0); let mut block = Box::new(BlockStopBackward { @@ -191,81 +188,74 @@ pub fn new_stop_block( bg: &mut graph::BlockGraph, Ok(block_outputs.pop().unwrap()) } - - - - - -impl BlockTrait for BlockStopBackward - - { +impl BlockTrait for BlockStopBackward { fn as_any(&mut self) -> &mut dyn Any { self } - fn allocate_and_init_weights(&mut self, mi: &model_instance::ModelInstance) { - } - - fn get_num_output_slots(&self) -> usize {1} + fn allocate_and_init_weights(&mut self, mi: &model_instance::ModelInstance) {} + fn get_num_output_slots(&self) -> usize { + 1 + } fn get_num_output_values(&self, output: graph::OutputSlot) -> usize { assert!(output.get_output_index() == 0); - return self.num_inputs + return self.num_inputs; } - - fn set_input_offset(&mut self, input: graph::InputSlot, offset: usize) { + + fn set_input_offset(&mut self, input: graph::InputSlot, offset: usize) { assert!(input.get_input_index() == 0); self.input_offset = offset; } - fn set_output_offset(&mut self, output: graph::OutputSlot, offset: usize) { + fn set_output_offset(&mut self, output: graph::OutputSlot, offset: usize) { assert!(output.get_output_index() == 0); self.output_offset = offset; } - #[inline(always)] - fn forward_backward(&mut self, - further_blocks: &mut [Box], - fb: &feature_buffer::FeatureBuffer, - pb: &mut port_buffer::PortBuffer, - update:bool) { + fn forward_backward( + &mut self, + further_blocks: &mut [Box], + fb: &feature_buffer::FeatureBuffer, + pb: &mut port_buffer::PortBuffer, + update: bool, + ) { debug_assert!(self.output_offset != usize::MAX); debug_assert!(self.input_offset != usize::MAX); debug_assert!(self.num_inputs > 0); - - pb.tape.copy_within(self.input_offset .. (self.input_offset + self.num_inputs), self.output_offset); + + pb.tape.copy_within( + self.input_offset..(self.input_offset + self.num_inputs), + self.output_offset, + ); block_helpers::forward_backward(further_blocks, fb, pb, update); - + if update { pb.tape[self.input_offset..(self.input_offset + self.num_inputs)].fill(0.0); } } - - fn forward(&self, further_blocks: &[Box], - fb: &feature_buffer::FeatureBuffer, - pb: &mut port_buffer::PortBuffer, - ) { + + fn forward( + &self, + further_blocks: &[Box], + fb: &feature_buffer::FeatureBuffer, + pb: &mut port_buffer::PortBuffer, + ) { debug_assert!(self.output_offset != usize::MAX); debug_assert!(self.input_offset != usize::MAX); debug_assert!(self.num_inputs > 0); - pb.tape.copy_within(self.input_offset .. (self.input_offset + self.num_inputs), self.output_offset); - - block_helpers::forward(further_blocks, fb, pb); + pb.tape.copy_within( + self.input_offset..(self.input_offset + self.num_inputs), + self.output_offset, + ); + + block_helpers::forward(further_blocks, fb, pb); } - } - - - - - - - - /* mod tests { // Note this useful idiom: importing names from outer (for mod tests) scope. @@ -292,31 +282,31 @@ mod tests { #[test] fn test_simple_positive() { - let mut mi = model_instance::ModelInstance::new_empty().unwrap(); + let mut mi = model_instance::ModelInstance::new_empty().unwrap(); let mut bg = BlockGraph::new(); let input_block = block_misc::new_const_block(&mut bg, vec![2.0]).unwrap(); let relu_block = new_relu_block(&mut bg, &mi, input_block).unwrap(); let observe_block = block_misc::new_observe_block(&mut bg, relu_block, Observe::Forward, Some(1.0)).unwrap(); bg.finalize(); bg.allocate_and_init_weights(&mi); - + let mut pb = bg.new_port_buffer(); - + let fb = fb_vec(); assert_epsilon!(slearn2 (&mut bg, &fb, &mut pb, true), 2.0); assert_epsilon!(slearn2 (&mut bg, &fb, &mut pb, true), 2.0); // relu desnt learn } fn test_simple_negative() { - let mut mi = model_instance::ModelInstance::new_empty().unwrap(); + let mut mi = model_instance::ModelInstance::new_empty().unwrap(); let mut bg = BlockGraph::new(); let input_block = block_misc::new_const_block(&mut bg, vec![-2.0]).unwrap(); let relu_block = new_relu_block(&mut bg, &mi, input_block).unwrap(); let observe_block = block_misc::new_observe_block(&mut bg, relu_block, Observe::Forward, Some(1.0)).unwrap(); bg.finalize(); bg.allocate_and_init_weights(&mi); - + let mut pb = bg.new_port_buffer(); - + let fb = fb_vec(); assert_epsilon!(slearn2 (&mut bg, &fb, &mut pb, true), 0.0); assert_epsilon!(slearn2 (&mut bg, &fb, &mut pb, true), 0.0); // relu desnt learn @@ -327,4 +317,4 @@ mod tests { -*/ \ No newline at end of file +*/ diff --git a/src/block_relu.rs b/src/block_relu.rs index 251aa8eb..9ead178c 100644 --- a/src/block_relu.rs +++ b/src/block_relu.rs @@ -1,27 +1,26 @@ use std::any::Any; use std::error::Error; -use crate::regressor; -use crate::model_instance; -use crate::feature_buffer; -use crate::port_buffer; use crate::block_helpers; +use crate::feature_buffer; use crate::graph; +use crate::graph::BlockGraph; +use crate::model_instance; +use crate::port_buffer; +use crate::regressor; use regressor::BlockTrait; -use crate::graph::{BlockGraph}; - -pub struct BlockRELU { +pub struct BlockRELU { pub num_inputs: usize, pub input_offset: usize, pub output_offset: usize, } - -pub fn new_relu_block( bg: &mut graph::BlockGraph, - mi: &model_instance::ModelInstance, - input: graph::BlockPtrOutput - ) -> Result> { +pub fn new_relu_block( + bg: &mut graph::BlockGraph, + mi: &model_instance::ModelInstance, + input: graph::BlockPtrOutput, +) -> Result> { let num_inputs = bg.get_num_output_values(vec![&input]); assert!(num_inputs != 0); let mut block = Box::new(BlockRELU { @@ -34,56 +33,50 @@ pub fn new_relu_block( bg: &mut graph::BlockGraph, Ok(block_outputs.pop().unwrap()) } - - - - - -impl BlockTrait for BlockRELU - - { +impl BlockTrait for BlockRELU { fn as_any(&mut self) -> &mut dyn Any { self } - fn allocate_and_init_weights(&mut self, mi: &model_instance::ModelInstance) { - } - - fn get_num_output_slots(&self) -> usize {1} + fn allocate_and_init_weights(&mut self, mi: &model_instance::ModelInstance) {} + fn get_num_output_slots(&self) -> usize { + 1 + } fn get_num_output_values(&self, output: graph::OutputSlot) -> usize { assert!(output.get_output_index() == 0); - return self.num_inputs + return self.num_inputs; } - - fn set_input_offset(&mut self, input: graph::InputSlot, offset: usize) { + + fn set_input_offset(&mut self, input: graph::InputSlot, offset: usize) { assert!(input.get_input_index() == 0); self.input_offset = offset; } - fn set_output_offset(&mut self, output: graph::OutputSlot, offset: usize) { + fn set_output_offset(&mut self, output: graph::OutputSlot, offset: usize) { assert!(output.get_output_index() == 0); self.output_offset = offset; } - #[inline(always)] - fn forward_backward(&mut self, - further_blocks: &mut [Box], - fb: &feature_buffer::FeatureBuffer, - pb: &mut port_buffer::PortBuffer, - update:bool) { + fn forward_backward( + &mut self, + further_blocks: &mut [Box], + fb: &feature_buffer::FeatureBuffer, + pb: &mut port_buffer::PortBuffer, + update: bool, + ) { debug_assert!(self.output_offset != usize::MAX); debug_assert!(self.input_offset != usize::MAX); debug_assert!(self.num_inputs > 0); - + unsafe { - for i in 0..self.num_inputs as usize { + for i in 0..self.num_inputs as usize { let w = *pb.tape.get_unchecked_mut(self.input_offset + i); if w < 0.0 { *pb.tape.get_unchecked_mut(self.output_offset + i) = 0.0; - *pb.tape.get_unchecked_mut(self.input_offset + i) = 0.0; + *pb.tape.get_unchecked_mut(self.input_offset + i) = 0.0; } else { *pb.tape.get_unchecked_mut(self.output_offset + i) = w; *pb.tape.get_unchecked_mut(self.input_offset + i) = 1.0; @@ -97,21 +90,22 @@ impl BlockTrait for BlockRELU let gradient = *pb.tape.get_unchecked(self.output_offset + i); *pb.tape.get_unchecked_mut(self.input_offset + i) *= gradient; } - } } // unsafe end } - - fn forward(&self, further_blocks: &[Box], - fb: &feature_buffer::FeatureBuffer, - pb: &mut port_buffer::PortBuffer, - ) { + + fn forward( + &self, + further_blocks: &[Box], + fb: &feature_buffer::FeatureBuffer, + pb: &mut port_buffer::PortBuffer, + ) { debug_assert!(self.output_offset != usize::MAX); debug_assert!(self.input_offset != usize::MAX); debug_assert!(self.num_inputs > 0); - + unsafe { - for i in 0..self.num_inputs as usize { + for i in 0..self.num_inputs as usize { let w = *pb.tape.get_unchecked_mut(self.input_offset + i); if w < 0.0 { *pb.tape.get_unchecked_mut(self.output_offset + i) = 0.0; @@ -122,73 +116,61 @@ impl BlockTrait for BlockRELU block_helpers::forward(further_blocks, fb, pb); } // unsafe end } - } - - - - - - - - - mod tests { // Note this useful idiom: importing names from outer (for mod tests) scope. use super::*; + use crate::assert_epsilon; use crate::block_misc; use crate::feature_buffer; - use block_helpers::{slearn2}; - use block_misc::{Observe}; - use crate::assert_epsilon; + use block_helpers::slearn2; + use block_misc::Observe; fn fb_vec() -> feature_buffer::FeatureBuffer { feature_buffer::FeatureBuffer { - label: 0.0, - example_importance: 1.0, - example_number: 0, - lr_buffer: Vec::new(), - ffm_buffer: Vec::new(), - ffm_fields_count: 0, + label: 0.0, + example_importance: 1.0, + example_number: 0, + lr_buffer: Vec::new(), + ffm_buffer: Vec::new(), + ffm_fields_count: 0, } } - #[test] fn test_simple_positive() { - let mut mi = model_instance::ModelInstance::new_empty().unwrap(); + let mut mi = model_instance::ModelInstance::new_empty().unwrap(); let mut bg = BlockGraph::new(); let input_block = block_misc::new_const_block(&mut bg, vec![2.0]).unwrap(); let relu_block = new_relu_block(&mut bg, &mi, input_block).unwrap(); - let observe_block = block_misc::new_observe_block(&mut bg, relu_block, Observe::Forward, Some(1.0)).unwrap(); + let observe_block = + block_misc::new_observe_block(&mut bg, relu_block, Observe::Forward, Some(1.0)) + .unwrap(); bg.finalize(); bg.allocate_and_init_weights(&mi); - + let mut pb = bg.new_port_buffer(); - + let fb = fb_vec(); - assert_epsilon!(slearn2 (&mut bg, &fb, &mut pb, true), 2.0); - assert_epsilon!(slearn2 (&mut bg, &fb, &mut pb, true), 2.0); // relu desnt learn + assert_epsilon!(slearn2(&mut bg, &fb, &mut pb, true), 2.0); + assert_epsilon!(slearn2(&mut bg, &fb, &mut pb, true), 2.0); // relu desnt learn } fn test_simple_negative() { - let mut mi = model_instance::ModelInstance::new_empty().unwrap(); + let mut mi = model_instance::ModelInstance::new_empty().unwrap(); let mut bg = BlockGraph::new(); let input_block = block_misc::new_const_block(&mut bg, vec![-2.0]).unwrap(); let relu_block = new_relu_block(&mut bg, &mi, input_block).unwrap(); - let observe_block = block_misc::new_observe_block(&mut bg, relu_block, Observe::Forward, Some(1.0)).unwrap(); + let observe_block = + block_misc::new_observe_block(&mut bg, relu_block, Observe::Forward, Some(1.0)) + .unwrap(); bg.finalize(); bg.allocate_and_init_weights(&mi); - + let mut pb = bg.new_port_buffer(); - + let fb = fb_vec(); - assert_epsilon!(slearn2 (&mut bg, &fb, &mut pb, true), 0.0); - assert_epsilon!(slearn2 (&mut bg, &fb, &mut pb, true), 0.0); // relu desnt learn + assert_epsilon!(slearn2(&mut bg, &fb, &mut pb, true), 0.0); + assert_epsilon!(slearn2(&mut bg, &fb, &mut pb, true), 0.0); // relu desnt learn } - - } - - - diff --git a/src/cache.rs b/src/cache.rs index 13ea74eb..d6cbf821 100644 --- a/src/cache.rs +++ b/src/cache.rs @@ -1,11 +1,10 @@ - -use std::{mem,slice}; +use std::error::Error; +use std::fs; use std::io; use std::io::Read; use std::io::Write; -use std::fs; -use std::error::Error; use std::path; +use std::{mem, slice}; //use flate2::write::DeflateEncoder; //use flate2::Compression; //use flate2::read::DeflateDecoder; @@ -15,8 +14,8 @@ use byteorder::{LittleEndian, ReadBytesExt, WriteBytesExt}; use crate::vwmap; -const CACHE_HEADER_MAGIC_STRING: &[u8; 4] = b"FWCA"; // Fwumious Wabbit CAche -const CACHE_HEADER_VERSION:u32 = 11; +const CACHE_HEADER_MAGIC_STRING: &[u8; 4] = b"FWCA"; // Fwumious Wabbit CAche +const CACHE_HEADER_VERSION: u32 = 11; /* Version incompatibilites: 10->11: float namespaces cannot have a weight attached @@ -31,8 +30,7 @@ Version incompatibilites: // u_size + blob: json encoding of vw_source // ...cached examples - -const READBUF_LEN:usize = 1024*100; +const READBUF_LEN: usize = 1024 * 100; // This is super ugly hack around the fact that we need to call finish() before closing the lz4 stream // Effectively lz4 implementation we're using is kind of bad @@ -52,7 +50,9 @@ impl Write for Wrapper { impl Drop for Wrapper { fn drop(&mut self) { match self.s.take() { - Some(s) => {let a = s.finish();} + Some(s) => { + let a = s.finish(); + } None => {} } } @@ -65,8 +65,8 @@ pub struct RecordCache { final_filename: String, pub writing: bool, pub reading: bool, -// pub output_buffer: Vec, - pub byte_buffer: Vec,//[u8; READBUF_LEN], + // pub output_buffer: Vec, + pub byte_buffer: Vec, //[u8; READBUF_LEN], start_pointer: usize, end_pointer: usize, total_read: usize, @@ -81,11 +81,10 @@ impl RecordCache { final_filename = format!("{}.fwcache", input_filename); if !input_filename.ends_with("gz") { gz = false; - } else - { + } else { gz = true; } - + let mut rc = RecordCache { output_bufwriter: Box::new(io::BufWriter::new(io::sink())), input_bufreader: Box::new(io::empty()), @@ -98,7 +97,7 @@ impl RecordCache { end_pointer: 0, total_read: 0, }; - + if enabled { if path::Path::new(&final_filename).exists() { rc.reading = true; @@ -106,58 +105,70 @@ impl RecordCache { // we buffer ourselves, otherwise i would be wise to use bufreader rc.input_bufreader = Box::new(fs::File::open(&final_filename).unwrap()); } else { -// rc.input_bufreader = Box::new(zstd::stream::Decoder::new(fs::File::open(&final_filename).unwrap()).unwrap()); - rc.input_bufreader = Box::new(lz4::Decoder::new(fs::File::open(&final_filename).unwrap()).unwrap()); + // rc.input_bufreader = Box::new(zstd::stream::Decoder::new(fs::File::open(&final_filename).unwrap()).unwrap()); + rc.input_bufreader = Box::new( + lz4::Decoder::new(fs::File::open(&final_filename).unwrap()).unwrap(), + ); } - println!("using cache_file = {}", final_filename ); + println!("using cache_file = {}", final_filename); println!("ignoring text input in favor of cache input"); match rc.verify_header(vw_map) { - Ok(()) => {}, + Ok(()) => {} Err(e) => { println!("Couldn't use the existing cache file: {:?}", e); rc.reading = false; } } rc.byte_buffer.resize(READBUF_LEN, 0); - } - + if !rc.reading { rc.writing = true; - println!("creating cache file = {}", final_filename ); + println!("creating cache file = {}", final_filename); if !gz { - rc.output_bufwriter = Box::new(io::BufWriter::new(fs::File::create(temporary_filename).unwrap())); + rc.output_bufwriter = Box::new(io::BufWriter::new( + fs::File::create(temporary_filename).unwrap(), + )); } else { -// rc.output_bufwriter = Box::new(io::BufWriter::new(DeflateEncoder::new(fs::File::create(temporary_filename).unwrap(), -// Compression::fast()))); - -// rc.output_bufwriter = Box::new(io::BufWriter::new(zstd::stream::Encoder::new(fs::File::create(temporary_filename).unwrap(), -// -5).unwrap().auto_finish())); -// rc.output_bufwriter = Box::new(io::BufWriter::new(lz4::EncoderBuilder::new() -// .level(3).build(fs::File::create(temporary_filename).unwrap() -// ).unwrap())); - let w = Wrapper{s:Some(lz4::EncoderBuilder::new().level(3).build(fs::File::create(temporary_filename).unwrap()).unwrap())}; - rc.output_bufwriter = Box::new(io::BufWriter::new(w)); + // rc.output_bufwriter = Box::new(io::BufWriter::new(DeflateEncoder::new(fs::File::create(temporary_filename).unwrap(), + // Compression::fast()))); + + // rc.output_bufwriter = Box::new(io::BufWriter::new(zstd::stream::Encoder::new(fs::File::create(temporary_filename).unwrap(), + // -5).unwrap().auto_finish())); + // rc.output_bufwriter = Box::new(io::BufWriter::new(lz4::EncoderBuilder::new() + // .level(3).build(fs::File::create(temporary_filename).unwrap() + // ).unwrap())); + let w = Wrapper { + s: Some( + lz4::EncoderBuilder::new() + .level(3) + .build(fs::File::create(temporary_filename).unwrap()) + .unwrap(), + ), + }; + rc.output_bufwriter = Box::new(io::BufWriter::new(w)); } rc.write_header(vw_map).unwrap(); } - } + } rc } - + pub fn push_record(&mut self, record_buf: &[u32]) -> Result<(), Box> { if self.writing { let element_size = mem::size_of::(); - unsafe { - let vv:&[u8] = slice::from_raw_parts(record_buf.as_ptr() as *const u8, - record_buf.len() * element_size) ; + unsafe { + let vv: &[u8] = slice::from_raw_parts( + record_buf.as_ptr() as *const u8, + record_buf.len() * element_size, + ); self.output_bufwriter.write_all(&vv)?; } } Ok(()) } - - pub fn write_finish(&mut self) -> Result<(), Box> { + + pub fn write_finish(&mut self) -> Result<(), Box> { if self.writing { self.output_bufwriter.flush()?; fs::rename(&self.temporary_filename, &self.final_filename)?; @@ -167,68 +178,80 @@ impl RecordCache { pub fn write_header(&mut self, vw_map: &vwmap::VwNamespaceMap) -> Result<(), Box> { self.output_bufwriter.write_all(CACHE_HEADER_MAGIC_STRING)?; - self.output_bufwriter.write_u32::(CACHE_HEADER_VERSION)?; + self.output_bufwriter + .write_u32::(CACHE_HEADER_VERSION)?; vw_map.save_to_buf(&mut self.output_bufwriter)?; Ok(()) } pub fn verify_header(&mut self, vwmap: &vwmap::VwNamespaceMap) -> Result<(), Box> { - let mut magic_string: [u8; 4] = [0;4]; + let mut magic_string: [u8; 4] = [0; 4]; self.input_bufreader.read(&mut magic_string)?; if &magic_string != CACHE_HEADER_MAGIC_STRING { return Err("Cache header does not begin with magic bytes FWFW")?; } - + let version = self.input_bufreader.read_u32::()?; if CACHE_HEADER_VERSION != version { - return Err(format!("Cache file version of this binary: {}, version of the cache file: {}", CACHE_HEADER_VERSION, version))?; + return Err(format!( + "Cache file version of this binary: {}, version of the cache file: {}", + CACHE_HEADER_VERSION, version + ))?; } - + // Compare vwmap in cache and the one we've been given. If they differ, rebuild cache let vwmap_from_cache = vwmap::VwNamespaceMap::new_from_buf(&mut self.input_bufreader)?; if vwmap_from_cache.vw_source != vwmap.vw_source { return Err("vw_namespace_map.csv and the one from cache file differ")?; } - + Ok(()) } - pub fn get_next_record(&mut self) -> Result<&[u32], Box> { if !self.reading { return Err("next_recrod() called on reading cache, when not opened in reading mode")?; } - unsafe { + unsafe { // We're going to cast another view over the data, so we can read it as u32 // This requires that the allocator we're using gives us sufficiently-aligned bytes, // but that's not guaranteed, so blow up to avoid UB if the allocator uses that freedom. - assert_eq!(self.byte_buffer.as_ptr() as usize % mem::align_of::(), 0); - let buf_view:&[u32] = slice::from_raw_parts(self.byte_buffer.as_ptr() as *const u32, READBUF_LEN/4); + assert_eq!( + self.byte_buffer.as_ptr() as usize % mem::align_of::(), + 0 + ); + let buf_view: &[u32] = + slice::from_raw_parts(self.byte_buffer.as_ptr() as *const u32, READBUF_LEN / 4); loop { // Classical buffer strategy: // Return if you have full record in buffer, // Otherwise shift the buffer and backfill it if self.end_pointer - self.start_pointer >= 4 { - let record_len = buf_view[self.start_pointer /4 ] as usize; + let record_len = buf_view[self.start_pointer / 4] as usize; if self.start_pointer + record_len * 4 <= self.end_pointer { - let ret_buf = &buf_view[self.start_pointer/4..self.start_pointer/4 + record_len]; + let ret_buf = + &buf_view[self.start_pointer / 4..self.start_pointer / 4 + record_len]; self.start_pointer += record_len * 4; return Ok(ret_buf); } - } - self.byte_buffer.copy_within(self.start_pointer..self.end_pointer, 0); + } + self.byte_buffer + .copy_within(self.start_pointer..self.end_pointer, 0); self.end_pointer -= self.start_pointer; self.start_pointer = 0; - let read_len = match self.input_bufreader.read(&mut self.byte_buffer[self.end_pointer..READBUF_LEN]) { + let read_len = match self + .input_bufreader + .read(&mut self.byte_buffer[self.end_pointer..READBUF_LEN]) + { Ok(0) => return Ok(&[]), Ok(n) => n, - Err(e) => Err(e)? + Err(e) => Err(e)?, }; self.end_pointer += read_len; self.total_read += read_len; - } + } } } -} \ No newline at end of file +} diff --git a/src/cmdline.rs b/src/cmdline.rs index ec128d33..3b3873cb 100644 --- a/src/cmdline.rs +++ b/src/cmdline.rs @@ -1,13 +1,13 @@ -use clap::{App, Arg, AppSettings}; use crate::version; +use clap::{App, AppSettings, Arg}; pub fn parse<'a>() -> clap::ArgMatches<'a> { - let matches = create_expected_args().get_matches(); - matches + let matches = create_expected_args().get_matches(); + matches } pub fn create_expected_args<'a>() -> App<'a, 'a> { - App::new("fwumious wabbit") + App::new("fwumious wabbit") .version(version::LATEST) .author("Andraz Tori ") .about("Superfast Logistic Regression & Field Aware Factorization Machines") diff --git a/src/consts.rs b/src/consts.rs index 131d3854..279c8f6c 100644 --- a/src/consts.rs +++ b/src/consts.rs @@ -1,7 +1,4 @@ +// Maximum supported FFM embedding size +pub const FFM_MAX_K: usize = 128; - -// Maximum supported FFM embedding size -pub const FFM_MAX_K:usize = 128; - -pub const NUM_TAPES:usize = 8; - +pub const NUM_TAPES: usize = 8; diff --git a/src/feature_buffer.rs b/src/feature_buffer.rs index c57a9954..10b947d6 100644 --- a/src/feature_buffer.rs +++ b/src/feature_buffer.rs @@ -1,11 +1,11 @@ +use crate::feature_transform_executor; use crate::model_instance; use crate::parser; -use crate::feature_transform_executor; -use crate::vwmap::{NamespaceType, NamespaceFormat}; +use crate::vwmap::{NamespaceFormat, NamespaceType}; -const VOWPAL_FNV_PRIME:u32 = 16777619; // vowpal magic number -//const CONSTANT_NAMESPACE:usize = 128; -const CONSTANT_HASH:u32 = 11650396; +const VOWPAL_FNV_PRIME: u32 = 16777619; // vowpal magic number + //const CONSTANT_NAMESPACE:usize = 128; +const CONSTANT_HASH: u32 = 11650396; #[derive(Clone, Debug, PartialEq)] pub struct HashAndValue { @@ -21,7 +21,6 @@ pub struct HashAndValueAndSeq { pub contra_field_index: u32, } - #[derive(Clone, Debug)] pub struct FeatureBuffer { pub label: f32, @@ -32,9 +31,6 @@ pub struct FeatureBuffer { pub ffm_fields_count: u32, } - - - #[derive(Clone)] pub struct FeatureBufferTranslator { model_instance: model_instance::ModelInstance, @@ -59,13 +55,21 @@ macro_rules! feature_reader { $bl:block ) => { if $namespace_descriptor.namespace_type == NamespaceType::Transformed { // This is super-unoptimized - let executor = unsafe {$transform_executors.executors.get_unchecked($namespace_descriptor.namespace_index as usize)}; - + let executor = unsafe { + $transform_executors + .executors + .get_unchecked($namespace_descriptor.namespace_index as usize) + }; + // If we have a cyclic defintion (which is a bug), this will panic! let mut namespace_to = executor.namespace_to.borrow_mut(); namespace_to.tmp_data.truncate(0); - - executor.function_executor.execute_function($record_buffer, &mut namespace_to, &$transform_executors); + + executor.function_executor.execute_function( + $record_buffer, + &mut namespace_to, + &$transform_executors, + ); for (hash_index1, hash_value1) in &namespace_to.tmp_data { let $hash_index = *hash_index1; @@ -74,28 +78,32 @@ macro_rules! feature_reader { } } else { let namespace_index = $namespace_descriptor.namespace_index as usize; - let first_token = unsafe {*$record_buffer.get_unchecked(namespace_index + parser::HEADER_LEN as usize)}; + let first_token = unsafe { + *$record_buffer.get_unchecked(namespace_index + parser::HEADER_LEN as usize) + }; if (first_token & parser::IS_NOT_SINGLE_MASK) == 0 { let $hash_index = first_token; let $hash_value: f32 = 1.0; $bl } else { - let start = ((first_token >> 16) & 0x3fff) as usize; + let start = ((first_token >> 16) & 0x3fff) as usize; let end = (first_token & 0xffff) as usize; if $namespace_descriptor.namespace_format != NamespaceFormat::F32 { for hash_offset in (start..end).step_by(2) { - let $hash_index = unsafe {*$record_buffer.get_unchecked(hash_offset)}; - let $hash_value = unsafe {f32::from_bits(*$record_buffer.get_unchecked(hash_offset+1))}; + let $hash_index = unsafe { *$record_buffer.get_unchecked(hash_offset) }; + let $hash_value = unsafe { + f32::from_bits(*$record_buffer.get_unchecked(hash_offset + 1)) + }; $bl } } else { for hash_offset in (start..end).step_by(2) { - let $hash_index = unsafe {*$record_buffer.get_unchecked(hash_offset)}; + let $hash_index = unsafe { *$record_buffer.get_unchecked(hash_offset) }; let $hash_value: f32 = 1.0; $bl } } - } + } } }; } @@ -109,14 +117,16 @@ macro_rules! feature_reader_float_namespace { $float_value:ident, $bl:block ) => { let namespace_index = $namespace_descriptor.namespace_index as usize; - let first_token = unsafe {*$record_buffer.get_unchecked(namespace_index + parser::HEADER_LEN as usize)}; + let first_token = + unsafe { *$record_buffer.get_unchecked(namespace_index + parser::HEADER_LEN as usize) }; if $namespace_descriptor.namespace_format == NamespaceFormat::F32 { - let start = ((first_token >> 16) & 0x3fff) as usize; + let start = ((first_token >> 16) & 0x3fff) as usize; let end = (first_token & 0xffff) as usize; for hash_offset in (start..end).step_by(2) { - let $hash_index = unsafe {*$record_buffer.get_unchecked(hash_offset)}; - let $hash_value:f32 = 1.0; - let $float_value = unsafe {f32::from_bits(*$record_buffer.get_unchecked(hash_offset+1))}; + let $hash_index = unsafe { *$record_buffer.get_unchecked(hash_offset) }; + let $hash_value: f32 = 1.0; + let $float_value = + unsafe { f32::from_bits(*$record_buffer.get_unchecked(hash_offset + 1)) }; $bl } } else { @@ -125,14 +135,10 @@ macro_rules! feature_reader_float_namespace { }; } - - - impl FeatureBufferTranslator { pub fn new(mi: &model_instance::ModelInstance) -> FeatureBufferTranslator { - // Calculate lr_hash_mask - let lr_hash_mask = (1 << mi.bit_precision) -1; + let lr_hash_mask = (1 << mi.bit_precision) - 1; // Calculate ffm_hash_mask let mut ffm_bits_for_dimensions = 0; while mi.ffm_k > (1 << (ffm_bits_for_dimensions)) { @@ -140,8 +146,7 @@ impl FeatureBufferTranslator { } let dimensions_mask = (1 << ffm_bits_for_dimensions) - 1; // in ffm we will simply mask the lower bits, so we spare them for k - let ffm_hash_mask = ((1 << mi.ffm_bit_precision) -1) ^ dimensions_mask; - + let ffm_hash_mask = ((1 << mi.ffm_bit_precision) - 1) ^ dimensions_mask; let mut fb = FeatureBuffer { label: 0.0, @@ -150,184 +155,278 @@ impl FeatureBufferTranslator { lr_buffer: Vec::new(), ffm_buffer: Vec::new(), ffm_fields_count: 0, - }; - + }; // avoid doing any allocations in translate - let fbt = FeatureBufferTranslator{ - model_instance: mi.clone(), // not the nicest option - hashes_vec_in : Vec::with_capacity(100), - hashes_vec_out : Vec::with_capacity(100), - feature_buffer: fb, - lr_hash_mask: lr_hash_mask, - ffm_hash_mask: ffm_hash_mask, - transform_executors: feature_transform_executor::TransformExecutors::from_namespace_transforms(&mi.transform_namespaces), + let fbt = FeatureBufferTranslator { + model_instance: mi.clone(), // not the nicest option + hashes_vec_in: Vec::with_capacity(100), + hashes_vec_out: Vec::with_capacity(100), + feature_buffer: fb, + lr_hash_mask: lr_hash_mask, + ffm_hash_mask: ffm_hash_mask, + transform_executors: + feature_transform_executor::TransformExecutors::from_namespace_transforms( + &mi.transform_namespaces, + ), }; fbt } - + pub fn print(&self) -> () { println!("item out {:?}", self.feature_buffer.lr_buffer); } - - + pub fn translate(&mut self, record_buffer: &[u32], example_number: u64) -> () { { let lr_buffer = &mut self.feature_buffer.lr_buffer; lr_buffer.truncate(0); - self.feature_buffer.label = record_buffer[parser::LABEL_OFFSET] as f32; // copy label - self.feature_buffer.example_importance = f32::from_bits(record_buffer[parser::EXAMPLE_IMPORTANCE_OFFSET]); + self.feature_buffer.label = record_buffer[parser::LABEL_OFFSET] as f32; // copy label + self.feature_buffer.example_importance = + f32::from_bits(record_buffer[parser::EXAMPLE_IMPORTANCE_OFFSET]); self.feature_buffer.example_number = example_number; - let mut output_len:usize = 0; - let mut hashes_vec_in : &mut Vec = &mut self.hashes_vec_in; - let mut hashes_vec_out : &mut Vec = &mut self.hashes_vec_out; - for (combo_index, feature_combo_desc) in self.model_instance.feature_combo_descs.iter().enumerate() { + let mut output_len: usize = 0; + let mut hashes_vec_in: &mut Vec = &mut self.hashes_vec_in; + let mut hashes_vec_out: &mut Vec = &mut self.hashes_vec_out; + for (combo_index, feature_combo_desc) in + self.model_instance.feature_combo_descs.iter().enumerate() + { let combo_index = combo_index as u32; let feature_combo_weight = feature_combo_desc.weight; // we unroll first iteration of the loop and optimize - let num_namespaces:usize = feature_combo_desc.namespace_descriptors.len() ; - let namespace_descriptor = unsafe{*feature_combo_desc.namespace_descriptors.get_unchecked(0)}; + let num_namespaces: usize = feature_combo_desc.namespace_descriptors.len(); + let namespace_descriptor = + unsafe { *feature_combo_desc.namespace_descriptors.get_unchecked(0) }; // We special case a single feature (common occurance) if num_namespaces == 1 { - feature_reader!(record_buffer, self.transform_executors, namespace_descriptor, hash_index, hash_value, { - lr_buffer.push(HashAndValue {hash: hash_index & self.lr_hash_mask, - value: hash_value * feature_combo_weight, - combo_index: combo_index}); - }); + feature_reader!( + record_buffer, + self.transform_executors, + namespace_descriptor, + hash_index, + hash_value, + { + lr_buffer.push(HashAndValue { + hash: hash_index & self.lr_hash_mask, + value: hash_value * feature_combo_weight, + combo_index: combo_index, + }); + } + ); } else { hashes_vec_in.truncate(0); - feature_reader!(record_buffer, self.transform_executors, namespace_descriptor, hash_index, hash_value, { - hashes_vec_in.push(HashAndValue {hash: hash_index, value: hash_value, combo_index: combo_index}); - }); - for namespace_descriptor in unsafe{feature_combo_desc.namespace_descriptors.get_unchecked(1 as usize .. num_namespaces)} { + feature_reader!( + record_buffer, + self.transform_executors, + namespace_descriptor, + hash_index, + hash_value, + { + hashes_vec_in.push(HashAndValue { + hash: hash_index, + value: hash_value, + combo_index: combo_index, + }); + } + ); + for namespace_descriptor in unsafe { + feature_combo_desc + .namespace_descriptors + .get_unchecked(1 as usize..num_namespaces) + } { hashes_vec_out.truncate(0); for handv in &(*hashes_vec_in) { let half_hash = handv.hash.overflowing_mul(VOWPAL_FNV_PRIME).0; - feature_reader!(record_buffer, self.transform_executors, *namespace_descriptor, hash_index, hash_value, { - hashes_vec_out.push(HashAndValue{ hash: hash_index ^ half_hash, - value: handv.value * hash_value, - combo_index: combo_index}); - }); + feature_reader!( + record_buffer, + self.transform_executors, + *namespace_descriptor, + hash_index, + hash_value, + { + hashes_vec_out.push(HashAndValue { + hash: hash_index ^ half_hash, + value: handv.value * hash_value, + combo_index: combo_index, + }); + } + ); } std::mem::swap(&mut hashes_vec_in, &mut hashes_vec_out); } for handv in &(*hashes_vec_in) { - lr_buffer.push(HashAndValue{hash: handv.hash & self.lr_hash_mask, - value: handv.value * feature_combo_weight, - combo_index: combo_index}); + lr_buffer.push(HashAndValue { + hash: handv.hash & self.lr_hash_mask, + value: handv.value * feature_combo_weight, + combo_index: combo_index, + }); } } } // add the constant if self.model_instance.add_constant_feature { - lr_buffer.push(HashAndValue{hash: CONSTANT_HASH & self.lr_hash_mask, - value: 1.0, - combo_index: self.model_instance.feature_combo_descs.len() as u32}); // we treat bias as a separate output + lr_buffer.push(HashAndValue { + hash: CONSTANT_HASH & self.lr_hash_mask, + value: 1.0, + combo_index: self.model_instance.feature_combo_descs.len() as u32, + }); // we treat bias as a separate output } // FFM loops have not been optimized yet - if self.model_instance.ffm_k > 0 { + if self.model_instance.ffm_k > 0 { // currently we only support primitive features as namespaces, (from --lrqfa command) // this is for compatibility with vowpal // but in theory we could support also combo features let ffm_buffer = &mut self.feature_buffer.ffm_buffer; ffm_buffer.truncate(0); - self.feature_buffer.ffm_fields_count = self.model_instance.ffm_fields.len() as u32; + self.feature_buffer.ffm_fields_count = self.model_instance.ffm_fields.len() as u32; //let feature_len = self.feature_buffer.ffm_fields_count * self.model_instance.ffm_k; - for (contra_field_index, ffm_field) in self.model_instance.ffm_fields.iter().enumerate() { + for (contra_field_index, ffm_field) in + self.model_instance.ffm_fields.iter().enumerate() + { for namespace_descriptor in ffm_field { - feature_reader!(record_buffer, self.transform_executors, *namespace_descriptor, hash_index, hash_value, { - ffm_buffer.push(HashAndValueAndSeq {hash: hash_index & self.ffm_hash_mask, - value: hash_value, - contra_field_index: contra_field_index as u32 * self.model_instance.ffm_k as u32}); - }); + feature_reader!( + record_buffer, + self.transform_executors, + *namespace_descriptor, + hash_index, + hash_value, + { + ffm_buffer.push(HashAndValueAndSeq { + hash: hash_index & self.ffm_hash_mask, + value: hash_value, + contra_field_index: contra_field_index as u32 + * self.model_instance.ffm_k as u32, + }); + } + ); } } } - } - } } - mod tests { // Note this useful idiom: importing names from outer (for mod tests) scope. use super::*; - use crate::parser::{NO_FEATURES, IS_NOT_SINGLE_MASK, MASK31}; - use crate::vwmap::{NamespaceType, NamespaceDescriptor, NamespaceFormat}; + use crate::parser::{IS_NOT_SINGLE_MASK, MASK31, NO_FEATURES}; + use crate::vwmap::{NamespaceDescriptor, NamespaceFormat, NamespaceType}; fn add_header(v2: Vec) -> Vec { let mut rr: Vec = vec![100, 1, 1.0f32.to_bits()]; rr.extend(v2); rr } - + fn nd(start: u32, end: u32) -> u32 { return (start << 16) + end; } fn ns_desc(i: u16) -> NamespaceDescriptor { - NamespaceDescriptor {namespace_index: i, - namespace_type: NamespaceType::Primitive, - namespace_format: NamespaceFormat::Categorical} + NamespaceDescriptor { + namespace_index: i, + namespace_type: NamespaceType::Primitive, + namespace_format: NamespaceFormat::Categorical, + } } fn ns_desc_f32(i: u16) -> NamespaceDescriptor { - NamespaceDescriptor {namespace_index: i, - namespace_type: NamespaceType::Primitive, - namespace_format: NamespaceFormat::F32} + NamespaceDescriptor { + namespace_index: i, + namespace_type: NamespaceType::Primitive, + namespace_format: NamespaceFormat::F32, + } } - #[test] fn test_constant() { let mut mi = model_instance::ModelInstance::new_empty().unwrap(); mi.add_constant_feature = true; - mi.feature_combo_descs.push(model_instance::FeatureComboDesc { - namespace_descriptors: vec![ns_desc(0)], - weight: 1.0}); - + mi.feature_combo_descs + .push(model_instance::FeatureComboDesc { + namespace_descriptors: vec![ns_desc(0)], + weight: 1.0, + }); + let mut fbt = FeatureBufferTranslator::new(&mi); let rb = add_header(vec![parser::NO_FEATURES]); // no feature fbt.translate(&rb, 0); - assert_eq!(fbt.feature_buffer.lr_buffer, vec![HashAndValue {hash:116060, value:1.0, combo_index: 1}]); // vw compatibility - no feature is no feature + assert_eq!( + fbt.feature_buffer.lr_buffer, + vec![HashAndValue { + hash: 116060, + value: 1.0, + combo_index: 1 + }] + ); // vw compatibility - no feature is no feature } - - + #[test] fn test_single_once() { let mut mi = model_instance::ModelInstance::new_empty().unwrap(); mi.add_constant_feature = false; - mi.feature_combo_descs.push(model_instance::FeatureComboDesc { - namespace_descriptors: vec![ns_desc(0)], - weight: 1.0}); - + mi.feature_combo_descs + .push(model_instance::FeatureComboDesc { + namespace_descriptors: vec![ns_desc(0)], + weight: 1.0, + }); + let mut fbt = FeatureBufferTranslator::new(&mi); let rb = add_header(vec![parser::NO_FEATURES]); // no feature - fbt.translate(&rb, 0 ); + fbt.translate(&rb, 0); assert_eq!(fbt.feature_buffer.lr_buffer, vec![]); // vw compatibility - no feature is no feature - let rb = add_header(vec![0xfea]); fbt.translate(&rb, 0); - assert_eq!(fbt.feature_buffer.lr_buffer, vec![HashAndValue {hash:0xfea, value:1.0, combo_index: 0}]); - - let rb = add_header(vec![parser::IS_NOT_SINGLE_MASK | nd(4,8), 0xfea, 1.0f32.to_bits(), 0xfeb, 1.0f32.to_bits()]); + assert_eq!( + fbt.feature_buffer.lr_buffer, + vec![HashAndValue { + hash: 0xfea, + value: 1.0, + combo_index: 0 + }] + ); + + let rb = add_header(vec![ + parser::IS_NOT_SINGLE_MASK | nd(4, 8), + 0xfea, + 1.0f32.to_bits(), + 0xfeb, + 1.0f32.to_bits(), + ]); fbt.translate(&rb, 0); - assert_eq!(fbt.feature_buffer.lr_buffer, vec![HashAndValue {hash:0xfea, value:1.0, combo_index: 0}, HashAndValue {hash:0xfeb, value:1.0, combo_index: 0}]); + assert_eq!( + fbt.feature_buffer.lr_buffer, + vec![ + HashAndValue { + hash: 0xfea, + value: 1.0, + combo_index: 0 + }, + HashAndValue { + hash: 0xfeb, + value: 1.0, + combo_index: 0 + } + ] + ); } #[test] fn test_single_twice() { - let mut mi = model_instance::ModelInstance::new_empty().unwrap(); + let mut mi = model_instance::ModelInstance::new_empty().unwrap(); mi.add_constant_feature = false; - mi.feature_combo_descs.push(model_instance::FeatureComboDesc { - namespace_descriptors: vec![ns_desc(0)], - weight: 1.0}); - mi.feature_combo_descs.push(model_instance::FeatureComboDesc { - namespace_descriptors: vec![ns_desc(1)], - weight: 1.0}); + mi.feature_combo_descs + .push(model_instance::FeatureComboDesc { + namespace_descriptors: vec![ns_desc(0)], + weight: 1.0, + }); + mi.feature_combo_descs + .push(model_instance::FeatureComboDesc { + namespace_descriptors: vec![ns_desc(1)], + weight: 1.0, + }); let mut fbt = FeatureBufferTranslator::new(&mi); @@ -337,24 +436,46 @@ mod tests { let rb = add_header(vec![0xfea, parser::NO_FEATURES]); fbt.translate(&rb, 0); - assert_eq!(fbt.feature_buffer.lr_buffer, vec![HashAndValue {hash:0xfea, value:1.0, combo_index: 0}]); + assert_eq!( + fbt.feature_buffer.lr_buffer, + vec![HashAndValue { + hash: 0xfea, + value: 1.0, + combo_index: 0 + }] + ); let rb = add_header(vec![0xfea, 0xfeb]); fbt.translate(&rb, 0); - assert_eq!(fbt.feature_buffer.lr_buffer, vec![HashAndValue {hash:0xfea, value:1.0, combo_index: 0}, HashAndValue {hash:0xfeb, value:1.0, combo_index: 1}]); - + assert_eq!( + fbt.feature_buffer.lr_buffer, + vec![ + HashAndValue { + hash: 0xfea, + value: 1.0, + combo_index: 0 + }, + HashAndValue { + hash: 0xfeb, + value: 1.0, + combo_index: 1 + } + ] + ); } // for singles, vowpal and fwumious are the same // however for doubles theya are not #[test] fn test_double_vowpal() { - let mut mi = model_instance::ModelInstance::new_empty().unwrap(); + let mut mi = model_instance::ModelInstance::new_empty().unwrap(); mi.add_constant_feature = false; - mi.feature_combo_descs.push(model_instance::FeatureComboDesc { - namespace_descriptors: vec![ns_desc(0), ns_desc(1)], - weight: 1.0}); - + mi.feature_combo_descs + .push(model_instance::FeatureComboDesc { + namespace_descriptors: vec![ns_desc(0), ns_desc(1)], + weight: 1.0, + }); + let mut fbt = FeatureBufferTranslator::new(&mi); let rb = add_header(vec![parser::NO_FEATURES]); fbt.translate(&rb, 0); @@ -362,34 +483,53 @@ mod tests { let rb = add_header(vec![123456789, parser::NO_FEATURES]); fbt.translate(&rb, 0); - assert_eq!(fbt.feature_buffer.lr_buffer, vec![]); // since the other feature is missing - VW compatibility says no feature is here + assert_eq!(fbt.feature_buffer.lr_buffer, vec![]); // since the other feature is missing - VW compatibility says no feature is here - let rb = add_header(vec![2988156968 & parser::MASK31, 2422381320 & parser::MASK31, parser::NO_FEATURES]); + let rb = add_header(vec![ + 2988156968 & parser::MASK31, + 2422381320 & parser::MASK31, + parser::NO_FEATURES, + ]); fbt.translate(&rb, 0); -// println!("out {}, out mod 2^24 {}", fbt.feature_buffer.lr_buffer[1], fbt.feature_buffer.lr_buffer[1] & ((1<<24)-1)); - assert_eq!(fbt.feature_buffer.lr_buffer, vec![HashAndValue {hash: 208368, value:1.0, combo_index: 0}]); - + // println!("out {}, out mod 2^24 {}", fbt.feature_buffer.lr_buffer[1], fbt.feature_buffer.lr_buffer[1] & ((1<<24)-1)); + assert_eq!( + fbt.feature_buffer.lr_buffer, + vec![HashAndValue { + hash: 208368, + value: 1.0, + combo_index: 0 + }] + ); } - + #[test] fn test_single_with_weight_vowpal() { - let mut mi = model_instance::ModelInstance::new_empty().unwrap(); + let mut mi = model_instance::ModelInstance::new_empty().unwrap(); mi.add_constant_feature = false; - mi.feature_combo_descs.push(model_instance::FeatureComboDesc { - namespace_descriptors: vec![ns_desc(0)], - weight: 2.0}); - + mi.feature_combo_descs + .push(model_instance::FeatureComboDesc { + namespace_descriptors: vec![ns_desc(0)], + weight: 2.0, + }); + let mut fbt = FeatureBufferTranslator::new(&mi); let rb = add_header(vec![0xfea]); fbt.translate(&rb, 0); - assert_eq!(fbt.feature_buffer.lr_buffer, vec![HashAndValue {hash: 0xfea, value:2.0, combo_index: 0}]); + assert_eq!( + fbt.feature_buffer.lr_buffer, + vec![HashAndValue { + hash: 0xfea, + value: 2.0, + combo_index: 0 + }] + ); } - + #[test] fn test_ffm_empty() { - let mut mi = model_instance::ModelInstance::new_empty().unwrap(); + let mut mi = model_instance::ModelInstance::new_empty().unwrap(); mi.add_constant_feature = false; - mi.ffm_fields.push(vec![]); // single field, empty + mi.ffm_fields.push(vec![]); // single field, empty mi.ffm_k = 1; let mut fbt = FeatureBufferTranslator::new(&mi); let rb = add_header(vec![0xfea]); @@ -399,64 +539,173 @@ mod tests { #[test] fn test_ffm_one() { - let mut mi = model_instance::ModelInstance::new_empty().unwrap(); + let mut mi = model_instance::ModelInstance::new_empty().unwrap(); mi.add_constant_feature = false; - mi.ffm_fields.push(vec![ns_desc(0)]); // single feature in a single fields + mi.ffm_fields.push(vec![ns_desc(0)]); // single feature in a single fields mi.ffm_k = 1; let mut fbt = FeatureBufferTranslator::new(&mi); let rb = add_header(vec![0xfea]); fbt.translate(&rb, 0); - assert_eq!(fbt.feature_buffer.ffm_buffer, vec![HashAndValueAndSeq{hash: 0xfea, value: 1.0, contra_field_index:0}]); + assert_eq!( + fbt.feature_buffer.ffm_buffer, + vec![HashAndValueAndSeq { + hash: 0xfea, + value: 1.0, + contra_field_index: 0 + }] + ); } #[test] fn test_ffm_two_fields() { - let mut mi = model_instance::ModelInstance::new_empty().unwrap(); + let mut mi = model_instance::ModelInstance::new_empty().unwrap(); mi.add_constant_feature = false; - mi.ffm_fields.push(vec![ns_desc(0)]); // single namespace in a field - mi.ffm_fields.push(vec![ns_desc(0),ns_desc(1)]); // two namespaces in a field + mi.ffm_fields.push(vec![ns_desc(0)]); // single namespace in a field + mi.ffm_fields.push(vec![ns_desc(0), ns_desc(1)]); // two namespaces in a field mi.ffm_k = 1; let mut fbt = FeatureBufferTranslator::new(&mi); - let rb = add_header(vec![parser::IS_NOT_SINGLE_MASK | nd(5,9), 0xfec, 0xfea, 2.0f32.to_bits(), 0xfeb, 3.0f32.to_bits()]); + let rb = add_header(vec![ + parser::IS_NOT_SINGLE_MASK | nd(5, 9), + 0xfec, + 0xfea, + 2.0f32.to_bits(), + 0xfeb, + 3.0f32.to_bits(), + ]); fbt.translate(&rb, 0); - assert_eq!(fbt.feature_buffer.ffm_buffer, vec![ HashAndValueAndSeq{hash: 0xfea, value: 2.0, contra_field_index:0}, - HashAndValueAndSeq{hash: 0xfeb, value: 3.0, contra_field_index:0}, - HashAndValueAndSeq{hash: 0xfea, value: 2.0, contra_field_index:1}, - HashAndValueAndSeq{hash: 0xfeb, value: 3.0, contra_field_index:1}, - HashAndValueAndSeq{hash: 0xfec, value: 1.0, contra_field_index:1}]); + assert_eq!( + fbt.feature_buffer.ffm_buffer, + vec![ + HashAndValueAndSeq { + hash: 0xfea, + value: 2.0, + contra_field_index: 0 + }, + HashAndValueAndSeq { + hash: 0xfeb, + value: 3.0, + contra_field_index: 0 + }, + HashAndValueAndSeq { + hash: 0xfea, + value: 2.0, + contra_field_index: 1 + }, + HashAndValueAndSeq { + hash: 0xfeb, + value: 3.0, + contra_field_index: 1 + }, + HashAndValueAndSeq { + hash: 0xfec, + value: 1.0, + contra_field_index: 1 + } + ] + ); } - + #[test] fn test_ffm_three_fields() { - let mut mi = model_instance::ModelInstance::new_empty().unwrap(); + let mut mi = model_instance::ModelInstance::new_empty().unwrap(); mi.add_constant_feature = false; - mi.ffm_fields.push(vec![ns_desc(0)]); // single namespace in a field 0xfea, 0xfeb - mi.ffm_fields.push(vec![ns_desc(0), ns_desc(1)]); // two namespaces in a field 0xfea, 0xfeb, 0xfec - mi.ffm_fields.push(vec![ns_desc(1)]); // single namespace in a field 0xfec + mi.ffm_fields.push(vec![ns_desc(0)]); // single namespace in a field 0xfea, 0xfeb + mi.ffm_fields.push(vec![ns_desc(0), ns_desc(1)]); // two namespaces in a field 0xfea, 0xfeb, 0xfec + mi.ffm_fields.push(vec![ns_desc(1)]); // single namespace in a field 0xfec mi.ffm_k = 1; let mut fbt = FeatureBufferTranslator::new(&mi); - let rb = add_header(vec![parser::IS_NOT_SINGLE_MASK | nd(5,9), 0x1, 0xfff, 2.0f32.to_bits(), 0xfeb, 3.0f32.to_bits()]); + let rb = add_header(vec![ + parser::IS_NOT_SINGLE_MASK | nd(5, 9), + 0x1, + 0xfff, + 2.0f32.to_bits(), + 0xfeb, + 3.0f32.to_bits(), + ]); fbt.translate(&rb, 0); // Hashes get changed, because k = 3 means we'll be aligning hashes - assert_eq!(fbt.feature_buffer.ffm_buffer, vec![ HashAndValueAndSeq{hash: 0xfff, value: 2.0, contra_field_index: 0}, - HashAndValueAndSeq{hash: 0xfeb, value: 3.0, contra_field_index: 0}, - HashAndValueAndSeq{hash: 0xfff, value: 2.0, contra_field_index: 1}, - HashAndValueAndSeq{hash: 0xfeb, value: 3.0, contra_field_index: 1}, - HashAndValueAndSeq{hash: 0x1, value: 1.0, contra_field_index: 1}, - HashAndValueAndSeq{hash: 0x1, value: 1.0, contra_field_index: 2}, - ]); + assert_eq!( + fbt.feature_buffer.ffm_buffer, + vec![ + HashAndValueAndSeq { + hash: 0xfff, + value: 2.0, + contra_field_index: 0 + }, + HashAndValueAndSeq { + hash: 0xfeb, + value: 3.0, + contra_field_index: 0 + }, + HashAndValueAndSeq { + hash: 0xfff, + value: 2.0, + contra_field_index: 1 + }, + HashAndValueAndSeq { + hash: 0xfeb, + value: 3.0, + contra_field_index: 1 + }, + HashAndValueAndSeq { + hash: 0x1, + value: 1.0, + contra_field_index: 1 + }, + HashAndValueAndSeq { + hash: 0x1, + value: 1.0, + contra_field_index: 2 + }, + ] + ); // Now hashes get changed, because k = 3 means we'll be aligning hashes mi.ffm_k = 3; let mut fbt = FeatureBufferTranslator::new(&mi); - let rb = add_header(vec![parser::IS_NOT_SINGLE_MASK | nd(5,9), 0x1, 0xfff, 2.0f32.to_bits(), 0xfeb, 3.0f32.to_bits()]); + let rb = add_header(vec![ + parser::IS_NOT_SINGLE_MASK | nd(5, 9), + 0x1, + 0xfff, + 2.0f32.to_bits(), + 0xfeb, + 3.0f32.to_bits(), + ]); fbt.translate(&rb, 0); - assert_eq!(fbt.feature_buffer.ffm_buffer, vec![ HashAndValueAndSeq{hash: 0xffc, value: 2.0, contra_field_index: 0}, - HashAndValueAndSeq{hash: 0xfe8, value: 3.0, contra_field_index: 0}, - HashAndValueAndSeq{hash: 0xffc, value: 2.0, contra_field_index: 3}, - HashAndValueAndSeq{hash: 0xfe8, value: 3.0, contra_field_index: 3}, - HashAndValueAndSeq{hash: 0x0, value: 1.0, contra_field_index: 3}, - HashAndValueAndSeq{hash: 0x0, value: 1.0, contra_field_index: 6}, - ]); + assert_eq!( + fbt.feature_buffer.ffm_buffer, + vec![ + HashAndValueAndSeq { + hash: 0xffc, + value: 2.0, + contra_field_index: 0 + }, + HashAndValueAndSeq { + hash: 0xfe8, + value: 3.0, + contra_field_index: 0 + }, + HashAndValueAndSeq { + hash: 0xffc, + value: 2.0, + contra_field_index: 3 + }, + HashAndValueAndSeq { + hash: 0xfe8, + value: 3.0, + contra_field_index: 3 + }, + HashAndValueAndSeq { + hash: 0x0, + value: 1.0, + contra_field_index: 3 + }, + HashAndValueAndSeq { + hash: 0x0, + value: 1.0, + contra_field_index: 6 + }, + ] + ); // one more which we dont test } @@ -464,10 +713,12 @@ mod tests { fn test_example_importance() { let mut mi = model_instance::ModelInstance::new_empty().unwrap(); mi.add_constant_feature = false; - mi.feature_combo_descs.push(model_instance::FeatureComboDesc { - namespace_descriptors: vec![ns_desc(0)], - weight: 1.0}); - + mi.feature_combo_descs + .push(model_instance::FeatureComboDesc { + namespace_descriptors: vec![ns_desc(0)], + weight: 1.0, + }); + let mut fbt = FeatureBufferTranslator::new(&mi); let rb = add_header(vec![parser::NO_FEATURES]); // no feature fbt.translate(&rb, 0); @@ -478,19 +729,37 @@ mod tests { fn test_single_namespace_float() { let mut mi = model_instance::ModelInstance::new_empty().unwrap(); mi.add_constant_feature = false; - mi.feature_combo_descs.push(model_instance::FeatureComboDesc { - namespace_descriptors: vec![ns_desc_f32(1)], - weight: 1.0}); - + mi.feature_combo_descs + .push(model_instance::FeatureComboDesc { + namespace_descriptors: vec![ns_desc_f32(1)], + weight: 1.0, + }); + let mut fbt = FeatureBufferTranslator::new(&mi); - let rb = add_header(vec![ NO_FEATURES, - nd(6, 10) | IS_NOT_SINGLE_MASK, - NO_FEATURES, - 0xffc & MASK31, 3.0f32.to_bits(), - 0xffa & MASK31, 4.0f32.to_bits(), - ]); + let rb = add_header(vec![ + NO_FEATURES, + nd(6, 10) | IS_NOT_SINGLE_MASK, + NO_FEATURES, + 0xffc & MASK31, + 3.0f32.to_bits(), + 0xffa & MASK31, + 4.0f32.to_bits(), + ]); fbt.translate(&rb, 0); - assert_eq!(fbt.feature_buffer.lr_buffer, vec![HashAndValue {hash:0xffc, value:1.0, combo_index: 0}, HashAndValue {hash:0xffa, value:1.0, combo_index: 0}]); + assert_eq!( + fbt.feature_buffer.lr_buffer, + vec![ + HashAndValue { + hash: 0xffc, + value: 1.0, + combo_index: 0 + }, + HashAndValue { + hash: 0xffa, + value: 1.0, + combo_index: 0 + } + ] + ); } - -} \ No newline at end of file +} diff --git a/src/feature_transform_executor.rs b/src/feature_transform_executor.rs index 33e67fee..3ae4b8ef 100644 --- a/src/feature_transform_executor.rs +++ b/src/feature_transform_executor.rs @@ -1,29 +1,29 @@ -use crate::vwmap; use crate::parser; +use crate::vwmap; use std::error::Error; use std::io::Error as IOError; use std::io::ErrorKind; use std::cell::RefCell; -use fasthash::murmur3; use dyn_clone::{clone_trait_object, DynClone}; +use fasthash::murmur3; +use crate::feature_transform_implementations::{ + TransformerBinner, TransformerCombine, TransformerLogRatioBinner, TransformerWeight, +}; use crate::feature_transform_parser; -use crate::feature_transform_implementations::{TransformerBinner, TransformerLogRatioBinner, TransformerCombine, TransformerWeight}; - - pub fn default_seeds(to_namespace_index: u32) -> [u32; 5] { - let to_namespace_index = to_namespace_index ^ 1u32 << 31; // compatibility with earlier version - [ - // These are random numbers, i threw a dice! - murmur3::hash32_with_seed(vec![214, 231, 1, 55], to_namespace_index), - murmur3::hash32_with_seed(vec![255, 6, 14, 69], to_namespace_index), - murmur3::hash32_with_seed(vec![50, 6, 71, 123], to_namespace_index), - murmur3::hash32_with_seed(vec![10, 3, 0,43] , to_namespace_index), - murmur3::hash32_with_seed(vec![0, 53, 10, 201] , to_namespace_index), - ] + let to_namespace_index = to_namespace_index ^ 1u32 << 31; // compatibility with earlier version + [ + // These are random numbers, i threw a dice! + murmur3::hash32_with_seed(vec![214, 231, 1, 55], to_namespace_index), + murmur3::hash32_with_seed(vec![255, 6, 14, 69], to_namespace_index), + murmur3::hash32_with_seed(vec![50, 6, 71, 123], to_namespace_index), + murmur3::hash32_with_seed(vec![10, 3, 0, 43], to_namespace_index), + murmur3::hash32_with_seed(vec![0, 53, 10, 201], to_namespace_index), + ] } #[derive(Clone, Copy)] @@ -35,13 +35,10 @@ pub enum SeedNumber { Four = 4, } - - - #[derive(Clone)] pub struct ExecutorToNamespace { pub namespace_descriptor: vwmap::NamespaceDescriptor, - pub namespace_seeds: [u32; 5], // These are precomputed namespace seeds + pub namespace_seeds: [u32; 5], // These are precomputed namespace seeds pub tmp_data: Vec<(u32, f32)>, } @@ -50,18 +47,20 @@ pub struct ExecutorFromNamespace { pub namespace_descriptor: vwmap::NamespaceDescriptor, } - impl ExecutorToNamespace { // We use const generics here as an experiment to see if they would be useful elsewhere to specialize functions #[inline(always)] - pub fn emit_i32(&mut self, to_data:i32, hash_value:f32) { - let hash_index = murmur3::hash32_with_seed(to_data.to_le_bytes(), *unsafe{self.namespace_seeds.get_unchecked(SEED_ID)}) & parser::MASK31; + pub fn emit_i32(&mut self, to_data: i32, hash_value: f32) { + let hash_index = murmur3::hash32_with_seed(to_data.to_le_bytes(), *unsafe { + self.namespace_seeds.get_unchecked(SEED_ID) + }) & parser::MASK31; self.tmp_data.push((hash_index, hash_value)); - } + } #[inline(always)] - pub fn emit_f32(&mut self, f:f32, hash_value:f32, interpolated: bool) { - if !f.is_finite() { // these handle INF, -INF and NAN + pub fn emit_f32(&mut self, f: f32, hash_value: f32, interpolated: bool) { + if !f.is_finite() { + // these handle INF, -INF and NAN self.emit_i32::(f.to_bits() as i32, hash_value); } else { if interpolated { @@ -79,14 +78,22 @@ impl ExecutorToNamespace { self.emit_i32::(f as i32, hash_value); } } - } + } #[inline(always)] - pub fn emit_i32_i32(&mut self, to_data1:i32, to_data2:i32, hash_value:f32) { - let hash_index = murmur3::hash32_with_seed(to_data1.to_le_bytes(), unsafe{*self.namespace_seeds.get_unchecked(SEED_ID)}); - let hash_index = murmur3::hash32_with_seed(to_data2.to_le_bytes(), hash_index) & parser::MASK31; + pub fn emit_i32_i32( + &mut self, + to_data1: i32, + to_data2: i32, + hash_value: f32, + ) { + let hash_index = murmur3::hash32_with_seed(to_data1.to_le_bytes(), unsafe { + *self.namespace_seeds.get_unchecked(SEED_ID) + }); + let hash_index = + murmur3::hash32_with_seed(to_data2.to_le_bytes(), hash_index) & parser::MASK31; self.tmp_data.push((hash_index, hash_value)); - } + } } #[derive(Clone)] @@ -96,130 +103,191 @@ pub struct TransformExecutor { } impl TransformExecutor { - pub fn from_namespace_transform(namespace_transform: &feature_transform_parser::NamespaceTransform) -> Result> { - + pub fn from_namespace_transform( + namespace_transform: &feature_transform_parser::NamespaceTransform, + ) -> Result> { let namespace_to = ExecutorToNamespace { namespace_descriptor: namespace_transform.to_namespace.namespace_descriptor, - namespace_seeds: default_seeds(namespace_transform.to_namespace.namespace_descriptor.namespace_index as u32), + namespace_seeds: default_seeds( + namespace_transform + .to_namespace + .namespace_descriptor + .namespace_index as u32, + ), tmp_data: Vec::new(), }; - + let te = TransformExecutor { namespace_to: RefCell::new(namespace_to), - function_executor: Self::create_executor(&namespace_transform.function_name, - &namespace_transform.from_namespaces, - &namespace_transform.function_parameters)?, + function_executor: Self::create_executor( + &namespace_transform.function_name, + &namespace_transform.from_namespaces, + &namespace_transform.function_parameters, + )?, }; Ok(te) } - pub fn create_executor(function_name: &str, namespaces_from: &Vec, function_params: &Vec) - -> Result, Box> { -/* let mut executor_namespaces_from: Vec = Vec::new(); - for namespace in namespaces_from { - executor_namespaces_from.push(ExecutorFromNamespace{namespace_descriptor: namespace.namespace_descriptor, - }); - }*/ - if function_name == "BinnerSqrtPlain" { - TransformerBinner::create_function(&(|x, resolution| x.sqrt() * resolution), function_name, namespaces_from, function_params, false) + pub fn create_executor( + function_name: &str, + namespaces_from: &Vec, + function_params: &Vec, + ) -> Result, Box> { + /* let mut executor_namespaces_from: Vec = Vec::new(); + for namespace in namespaces_from { + executor_namespaces_from.push(ExecutorFromNamespace{namespace_descriptor: namespace.namespace_descriptor, + }); + }*/ + if function_name == "BinnerSqrtPlain" { + TransformerBinner::create_function( + &(|x, resolution| x.sqrt() * resolution), + function_name, + namespaces_from, + function_params, + false, + ) } else if function_name == "BinnerSqrt" { - TransformerBinner::create_function(&(|x, resolution| x.sqrt() * resolution), function_name, namespaces_from, function_params, true) + TransformerBinner::create_function( + &(|x, resolution| x.sqrt() * resolution), + function_name, + namespaces_from, + function_params, + true, + ) } else if function_name == "BinnerLogPlain" { - TransformerBinner::create_function(&(|x, resolution| x.ln() * resolution), function_name, namespaces_from, function_params, false) + TransformerBinner::create_function( + &(|x, resolution| x.ln() * resolution), + function_name, + namespaces_from, + function_params, + false, + ) } else if function_name == "BinnerLog" { - TransformerBinner::create_function(&(|x, resolution| x.ln() * resolution), function_name, namespaces_from, function_params, true) + TransformerBinner::create_function( + &(|x, resolution| x.ln() * resolution), + function_name, + namespaces_from, + function_params, + true, + ) } else if function_name == "BinnerLogRatioPlain" { - TransformerLogRatioBinner::create_function(function_name, namespaces_from, function_params, false) + TransformerLogRatioBinner::create_function( + function_name, + namespaces_from, + function_params, + false, + ) } else if function_name == "BinnerLogRatio" { - TransformerLogRatioBinner::create_function(function_name, namespaces_from, function_params, true) + TransformerLogRatioBinner::create_function( + function_name, + namespaces_from, + function_params, + true, + ) } else if function_name == "Combine" { TransformerCombine::create_function(function_name, namespaces_from, function_params) } else if function_name == "Weight" { TransformerWeight::create_function(function_name, namespaces_from, function_params) } else { - return Err(Box::new(IOError::new(ErrorKind::Other, format!("Unknown transformer function: {}", function_name)))); - + return Err(Box::new(IOError::new( + ErrorKind::Other, + format!("Unknown transformer function: {}", function_name), + ))); } } } - - #[derive(Clone)] pub struct TransformExecutors { pub executors: Vec, - } impl TransformExecutors { - pub fn from_namespace_transforms(namespace_transforms: &feature_transform_parser::NamespaceTransforms) -> TransformExecutors{ - let mut executors:Vec = Vec::new(); + pub fn from_namespace_transforms( + namespace_transforms: &feature_transform_parser::NamespaceTransforms, + ) -> TransformExecutors { + let mut executors: Vec = Vec::new(); let mut namespaces_to: Vec = Vec::new(); for transformed_namespace in &namespace_transforms.v { - let transformed_namespace_executor = TransformExecutor::from_namespace_transform(&transformed_namespace).unwrap(); + let transformed_namespace_executor = + TransformExecutor::from_namespace_transform(&transformed_namespace).unwrap(); executors.push(transformed_namespace_executor); - } - TransformExecutors {executors: executors} + TransformExecutors { + executors: executors, + } } -/* -// We don't use this function as we have put it into feature_reader! macro - #[inline(always)] - pub fn get_transformations<'a>(&self, record_buffer: &[u32], feature_index_offset: u32) -> &TransformExecutor { - let executor_index = feature_index_offset & !feature_transform_parser::TRANSFORM_NAMESPACE_MARK; // remove transform namespace mark - let executor = unsafe {&self.executors.get_unchecked(executor_index as usize)}; - - // If we have a cyclic defintion (which is a bug), this will panic! - let mut namespace_to = executor.namespace_to.borrow_mut(); - namespace_to.tmp_data.truncate(0); - - executor.function_executor.execute_function(record_buffer, &mut namespace_to, &self); - executor - } -*/ + /* + // We don't use this function as we have put it into feature_reader! macro + #[inline(always)] + pub fn get_transformations<'a>(&self, record_buffer: &[u32], feature_index_offset: u32) -> &TransformExecutor { + let executor_index = feature_index_offset & !feature_transform_parser::TRANSFORM_NAMESPACE_MARK; // remove transform namespace mark + let executor = unsafe {&self.executors.get_unchecked(executor_index as usize)}; -} + // If we have a cyclic defintion (which is a bug), this will panic! + let mut namespace_to = executor.namespace_to.borrow_mut(); + namespace_to.tmp_data.truncate(0); + executor.function_executor.execute_function(record_buffer, &mut namespace_to, &self); + executor + } + */ +} // Some black magic from: https://stackoverflow.com/questions/30353462/how-to-clone-a-struct-storing-a-boxed-trait-object // We need clone() because of serving. There is also an option of doing FeatureBufferTransform from scratch in each thread pub trait FunctionExecutorTrait: DynClone + Send { - fn execute_function(&self, record_buffer: &[u32], to_namespace: &mut ExecutorToNamespace, transform_executors: &TransformExecutors); + fn execute_function( + &self, + record_buffer: &[u32], + to_namespace: &mut ExecutorToNamespace, + transform_executors: &TransformExecutors, + ); } clone_trait_object!(FunctionExecutorTrait); - - mod tests { // Note this useful idiom: importing names from outer (for mod tests) scope. use super::*; - use crate::parser; use crate::feature_transform_executor::default_seeds; + use crate::parser; fn ns_desc(i: u16) -> vwmap::NamespaceDescriptor { - vwmap::NamespaceDescriptor {namespace_index: i, - namespace_type: vwmap::NamespaceType::Primitive, - namespace_format: vwmap::NamespaceFormat::Categorical} + vwmap::NamespaceDescriptor { + namespace_index: i, + namespace_type: vwmap::NamespaceType::Primitive, + namespace_format: vwmap::NamespaceFormat::Categorical, + } } - #[test] fn test_interpolation() { let to_namespace_empty = ExecutorToNamespace { - namespace_descriptor: ns_desc(1), - namespace_seeds: default_seeds(1), // These are precomputed namespace seeds - tmp_data: Vec::new(), - }; + namespace_descriptor: ns_desc(1), + namespace_seeds: default_seeds(1), // These are precomputed namespace seeds + tmp_data: Vec::new(), + }; let mut to_namespace = to_namespace_empty.clone(); - to_namespace.emit_f32::<{SeedNumber::Default as usize}>(5.4, 20.0, true); - let to_data_1:i32 = 6; + to_namespace.emit_f32::<{ SeedNumber::Default as usize }>(5.4, 20.0, true); + let to_data_1: i32 = 6; let to_data_1_value = 20.0 * (5.4 - 5.0); - let hash_index_1 = murmur3::hash32_with_seed(to_data_1.to_le_bytes(), to_namespace.namespace_seeds[SeedNumber::Default as usize]) & parser::MASK31; - let to_data_2:i32 = 5; + let hash_index_1 = murmur3::hash32_with_seed( + to_data_1.to_le_bytes(), + to_namespace.namespace_seeds[SeedNumber::Default as usize], + ) & parser::MASK31; + let to_data_2: i32 = 5; let to_data_2_value = 20.0 * (6.0 - 5.4); - let hash_index_2 = murmur3::hash32_with_seed(to_data_2.to_le_bytes(), to_namespace.namespace_seeds[SeedNumber::Default as usize]) & parser::MASK31; - assert_eq!(to_namespace.tmp_data, vec![(hash_index_1, to_data_1_value), (hash_index_2, to_data_2_value)]); - } + let hash_index_2 = murmur3::hash32_with_seed( + to_data_2.to_le_bytes(), + to_namespace.namespace_seeds[SeedNumber::Default as usize], + ) & parser::MASK31; + assert_eq!( + to_namespace.tmp_data, + vec![ + (hash_index_1, to_data_1_value), + (hash_index_2, to_data_2_value) + ] + ); + } } - diff --git a/src/feature_transform_implementations.rs b/src/feature_transform_implementations.rs index a9e278b7..ba0ee5c6 100644 --- a/src/feature_transform_implementations.rs +++ b/src/feature_transform_implementations.rs @@ -1,18 +1,17 @@ - use std::error::Error; use std::io::Error as IOError; use std::io::ErrorKind; - -use crate::parser; use crate::feature_reader; use crate::feature_reader_float_namespace; +use crate::parser; -use crate::feature_transform_executor::{SeedNumber, ExecutorFromNamespace, ExecutorToNamespace, FunctionExecutorTrait, TransformExecutors}; +use crate::feature_transform_executor::{ + ExecutorFromNamespace, ExecutorToNamespace, FunctionExecutorTrait, SeedNumber, + TransformExecutors, +}; use crate::feature_transform_parser; -use crate::vwmap::{NamespaceType, NamespaceFormat, NamespaceDescriptor}; - - +use crate::vwmap::{NamespaceDescriptor, NamespaceFormat, NamespaceType}; // Basic example of a "full blown" simple FunctionExecutorTrait #[derive(Clone)] @@ -21,31 +20,52 @@ struct FunctionExampleSqrt { } impl FunctionExecutorTrait for FunctionExampleSqrt { - fn execute_function(&self, record_buffer: &[u32], to_namespace: &mut ExecutorToNamespace, transform_executors: &TransformExecutors) { - feature_reader_float_namespace!(record_buffer, self.from_namespace.namespace_descriptor, hash_index, hash_value, float_value, { - let transformed_float = float_value.sqrt(); - let transformed_int = transformed_float as i32; - to_namespace.emit_i32::<{SeedNumber::Default as usize}>(transformed_int, hash_value); - }); + fn execute_function( + &self, + record_buffer: &[u32], + to_namespace: &mut ExecutorToNamespace, + transform_executors: &TransformExecutors, + ) { + feature_reader_float_namespace!( + record_buffer, + self.from_namespace.namespace_descriptor, + hash_index, + hash_value, + float_value, + { + let transformed_float = float_value.sqrt(); + let transformed_int = transformed_float as i32; + to_namespace + .emit_i32::<{ SeedNumber::Default as usize }>(transformed_int, hash_value); + } + ); } } impl FunctionExampleSqrt { - fn create_function(function_name: &str, from_namespaces: &Vec, function_params: &Vec) -> Result, Box> { + fn create_function( + function_name: &str, + from_namespaces: &Vec, + function_params: &Vec, + ) -> Result, Box> { // For simplicity of example, we just assert instead of full error reporting assert!(function_params.len() == 0); assert!(from_namespaces.len() == 1); assert!(from_namespaces[0].namespace_descriptor.namespace_type == NamespaceType::Primitive); assert!(from_namespaces[0].namespace_descriptor.namespace_format == NamespaceFormat::F32); - Ok(Box::new(Self{from_namespace: ExecutorFromNamespace{namespace_descriptor: from_namespaces[0].namespace_descriptor}})) - } + Ok(Box::new(Self { + from_namespace: ExecutorFromNamespace { + namespace_descriptor: from_namespaces[0].namespace_descriptor, + }, + })) + } } // ------------------------------------------------------------------- // TransformerBinner - A basic binner -// It can take any function as a binning function f32 -> f32. Then output is rounded to integer +// It can take any function as a binning function f32 -> f32. Then output is rounded to integer -// What does greater_than do? +// What does greater_than do? // If output is smaller than the first floating parameter (greater_than), then output is rounded to integer // If the output is larger than the first floating point parameter (greater_than) then we first substract greater_than from the input and apply transform function // Example of use: you want to bin number of pageviews per user, so you generally want to do sqrt on it, but only do floor binning when pageviews <= 10 @@ -53,69 +73,97 @@ impl FunctionExampleSqrt { // What does resolution mean? // Example: BinnerSqrt(X)(10.0, 2.0) -// let's assume X is 150. sqrt(150) * 1.0 = 12.247 +// let's assume X is 150. sqrt(150) * 1.0 = 12.247 // Since reslution is 2.0, we will first mutiply 12.247 by 2 and get to 24.5. We then round that to integer = 24 // What does interpolated mean? // Example: BinnerSqrt(X)(10.0, 1.0) -// let's assume X is 150. sqrt(150) * 1.0 = 12.247 +// let's assume X is 150. sqrt(150) * 1.0 = 12.247 // You now want two values emitted - 12 at value 0.247 and 13 at value (1-0.247) - - - #[derive(Clone)] pub struct TransformerBinner { from_namespace: ExecutorFromNamespace, greater_than: f32, resolution: f32, interpolated: bool, - function_pointer: &'static (dyn Fn(f32, f32) -> f32 +'static + Sync), + function_pointer: &'static (dyn Fn(f32, f32) -> f32 + 'static + Sync), } impl FunctionExecutorTrait for TransformerBinner { - fn execute_function(&self, record_buffer: &[u32], to_namespace: &mut ExecutorToNamespace, transform_executors: &TransformExecutors) { - feature_reader_float_namespace!(record_buffer, self.from_namespace.namespace_descriptor, hash_index, hash_value, float_value, { - if float_value < self.greater_than { - to_namespace.emit_i32::<{SeedNumber::Default as usize}>(float_value as i32, hash_value); - } else { - let transformed_float = (self.function_pointer)(float_value - self.greater_than, self.resolution); - to_namespace.emit_f32::<{SeedNumber::One as usize}>(transformed_float, hash_value, self.interpolated); + fn execute_function( + &self, + record_buffer: &[u32], + to_namespace: &mut ExecutorToNamespace, + transform_executors: &TransformExecutors, + ) { + feature_reader_float_namespace!( + record_buffer, + self.from_namespace.namespace_descriptor, + hash_index, + hash_value, + float_value, + { + if float_value < self.greater_than { + to_namespace.emit_i32::<{ SeedNumber::Default as usize }>( + float_value as i32, + hash_value, + ); + } else { + let transformed_float = + (self.function_pointer)(float_value - self.greater_than, self.resolution); + to_namespace.emit_f32::<{ SeedNumber::One as usize }>( + transformed_float, + hash_value, + self.interpolated, + ); + } } - }); + ); } } - impl TransformerBinner { - pub fn create_function(function_pointer: &'static (dyn Fn(f32, f32) -> f32 +'static + Sync), - function_name: &str, - from_namespaces: &Vec, - function_params: &Vec, - interpolated: bool, - ) -> Result, Box> { - + pub fn create_function( + function_pointer: &'static (dyn Fn(f32, f32) -> f32 + 'static + Sync), + function_name: &str, + from_namespaces: &Vec, + function_params: &Vec, + interpolated: bool, + ) -> Result, Box> { if function_params.len() > 2 { return Err(Box::new(IOError::new(ErrorKind::Other, format!("Function {} takes up to two float arguments, example {}(A)(2.0, 3.5). Both are optional.\nFirst parameter is the minimum parameter to apply function at (default: -MAX), second parameter is resolution (default: 1.0))", function_name, function_name)))); } - + let greater_than = match function_params.get(0) { - Some(&greater_than) => greater_than, - None => 0.0 + Some(&greater_than) => greater_than, + None => 0.0, }; if greater_than < 0.0 { - return Err(Box::new(IOError::new(ErrorKind::Other, format!("Function {} parameter greater_than cannot be negative (passed : {}))", function_name, greater_than)))); + return Err(Box::new(IOError::new( + ErrorKind::Other, + format!( + "Function {} parameter greater_than cannot be negative (passed : {}))", + function_name, greater_than + ), + ))); } let resolution = match function_params.get(1) { - Some(&resolution) => resolution, - None => 1.0 + Some(&resolution) => resolution, + None => 1.0, }; -// println!("Greater than : {}, resolution: {}", greater_than, resolution); - + // println!("Greater than : {}, resolution: {}", greater_than, resolution); + if from_namespaces.len() != 1 { - return Err(Box::new(IOError::new(ErrorKind::Other, format!("Function {} takes exactly one namespace argument, example {}(A)(2.0)", function_name, function_name)))); + return Err(Box::new(IOError::new( + ErrorKind::Other, + format!( + "Function {} takes exactly one namespace argument, example {}(A)(2.0)", + function_name, function_name + ), + ))); } for namespace in from_namespaces.iter() { @@ -124,15 +172,17 @@ impl TransformerBinner { } } - Ok(Box::new(Self{from_namespace: ExecutorFromNamespace{namespace_descriptor: from_namespaces[0].namespace_descriptor}, - resolution: resolution, - greater_than: greater_than, - function_pointer: function_pointer, - interpolated: interpolated, - })) + Ok(Box::new(Self { + from_namespace: ExecutorFromNamespace { + namespace_descriptor: from_namespaces[0].namespace_descriptor, + }, + resolution: resolution, + greater_than: greater_than, + function_pointer: function_pointer, + interpolated: interpolated, + })) } -} - +} // ------------------------------------------------------------------- // LogRatio Binner @@ -151,57 +201,102 @@ pub struct TransformerLogRatioBinner { } impl FunctionExecutorTrait for TransformerLogRatioBinner { - fn execute_function(&self, record_buffer: &[u32], to_namespace: &mut ExecutorToNamespace, transform_executors: &TransformExecutors) { - feature_reader_float_namespace!(record_buffer, self.from_namespace1.namespace_descriptor, hash_index1, hash_value1, float_value1, { - feature_reader_float_namespace!(record_buffer, self.from_namespace2.namespace_descriptor, hash_index2, hash_value2, float_value2, { - - let joint_value = hash_value1 * hash_value2; - let val1 = float_value1; - let val2 = float_value2; - if val2 + val1 < self.greater_than { - to_namespace.emit_i32_i32::<{SeedNumber::One as usize}>(val1 as i32, val2 as i32, joint_value); - } else if val1 == 0.0 { - // val2 has to be greater or equal to self.greater_than (if it wasn't we'd take the first if branch - to_namespace.emit_f32::<{SeedNumber::Two as usize}>((val2 - self.greater_than).ln(), joint_value, self.interpolated); - } else if val2 == 0.0 { - // val1 has to be greater or equal to self.greater_than (if it wasn't we'd take the first if branch - to_namespace.emit_f32::<{SeedNumber::Three as usize}>((val1 - self.greater_than).ln(), joint_value, self.interpolated); - } else { - let o = (val1/val2).ln()*self.resolution; - to_namespace.emit_f32::<{SeedNumber::Default as usize}>(o, joint_value, self.interpolated); - } - }); - }); + fn execute_function( + &self, + record_buffer: &[u32], + to_namespace: &mut ExecutorToNamespace, + transform_executors: &TransformExecutors, + ) { + feature_reader_float_namespace!( + record_buffer, + self.from_namespace1.namespace_descriptor, + hash_index1, + hash_value1, + float_value1, + { + feature_reader_float_namespace!( + record_buffer, + self.from_namespace2.namespace_descriptor, + hash_index2, + hash_value2, + float_value2, + { + let joint_value = hash_value1 * hash_value2; + let val1 = float_value1; + let val2 = float_value2; + if val2 + val1 < self.greater_than { + to_namespace.emit_i32_i32::<{ SeedNumber::One as usize }>( + val1 as i32, + val2 as i32, + joint_value, + ); + } else if val1 == 0.0 { + // val2 has to be greater or equal to self.greater_than (if it wasn't we'd take the first if branch + to_namespace.emit_f32::<{ SeedNumber::Two as usize }>( + (val2 - self.greater_than).ln(), + joint_value, + self.interpolated, + ); + } else if val2 == 0.0 { + // val1 has to be greater or equal to self.greater_than (if it wasn't we'd take the first if branch + to_namespace.emit_f32::<{ SeedNumber::Three as usize }>( + (val1 - self.greater_than).ln(), + joint_value, + self.interpolated, + ); + } else { + let o = (val1 / val2).ln() * self.resolution; + to_namespace.emit_f32::<{ SeedNumber::Default as usize }>( + o, + joint_value, + self.interpolated, + ); + } + } + ); + } + ); } } impl TransformerLogRatioBinner { pub fn create_function( - function_name: &str, - from_namespaces: &Vec, - function_params: &Vec, - interpolated: bool, - ) -> Result, Box> { - + function_name: &str, + from_namespaces: &Vec, + function_params: &Vec, + interpolated: bool, + ) -> Result, Box> { if function_params.len() > 2 { return Err(Box::new(IOError::new(ErrorKind::Other, format!("Function {} takes up to two float arguments, example {}(A)(2.0, 3.5). Both are optional.\nFirst parameter is the minimum parameter to apply function at (default: -MAX), second parameter is resolution (default: 1.0))", function_name, function_name)))); } - + let greater_than = match function_params.get(0) { - Some(&greater_than) => greater_than, - None => 0.0 + Some(&greater_than) => greater_than, + None => 0.0, }; if greater_than < 0.0 { - return Err(Box::new(IOError::new(ErrorKind::Other, format!("Function {} parameter greater_than cannot be negative (passed : {}))", function_name, greater_than)))); + return Err(Box::new(IOError::new( + ErrorKind::Other, + format!( + "Function {} parameter greater_than cannot be negative (passed : {}))", + function_name, greater_than + ), + ))); } let resolution = match function_params.get(1) { - Some(&resolution) => resolution, - None => 1.0 + Some(&resolution) => resolution, + None => 1.0, }; if from_namespaces.len() != 2 { - return Err(Box::new(IOError::new(ErrorKind::Other, format!("Function {} takes exactly two namespace arguments, example {}(A,B)(2.0)", function_name, function_name)))); + return Err(Box::new(IOError::new( + ErrorKind::Other, + format!( + "Function {} takes exactly two namespace arguments, example {}(A,B)(2.0)", + function_name, function_name + ), + ))); } for namespace in from_namespaces.iter() { if namespace.namespace_descriptor.namespace_format != NamespaceFormat::F32 { @@ -209,24 +304,26 @@ impl TransformerLogRatioBinner { } } - Ok(Box::new(Self{from_namespace1: ExecutorFromNamespace{namespace_descriptor: from_namespaces[0].namespace_descriptor}, - from_namespace2: ExecutorFromNamespace{namespace_descriptor: from_namespaces[1].namespace_descriptor}, - resolution: resolution, - greater_than: greater_than, - interpolated: interpolated, - })) + Ok(Box::new(Self { + from_namespace1: ExecutorFromNamespace { + namespace_descriptor: from_namespaces[0].namespace_descriptor, + }, + from_namespace2: ExecutorFromNamespace { + namespace_descriptor: from_namespaces[1].namespace_descriptor, + }, + resolution: resolution, + greater_than: greater_than, + interpolated: interpolated, + })) } -} - - +} // Value multiplier transformer // ------------------------------------------------------------------- // TransformerWeight - A basic weight multiplier transformer // Example of use: if you want to multiply whole namespace with certain factor and thus increase its learning rate (let's say 2.0) // In that case you would call MutliplyWeight(document_id)(2.0) -// Important - document_id does not need to be float and isnt really changed - +// Important - document_id does not need to be float and isnt really changed #[derive(Clone)] pub struct TransformerWeight { @@ -235,34 +332,62 @@ pub struct TransformerWeight { } impl FunctionExecutorTrait for TransformerWeight { - fn execute_function(&self, record_buffer: &[u32], to_namespace: &mut ExecutorToNamespace, transform_executors: &TransformExecutors) { - feature_reader!(record_buffer, transform_executors, self.from_namespace.namespace_descriptor, hash_index, hash_value, { - to_namespace.emit_i32::<{SeedNumber::Default as usize}>(hash_index as i32, hash_value * self.multiplier); - }); + fn execute_function( + &self, + record_buffer: &[u32], + to_namespace: &mut ExecutorToNamespace, + transform_executors: &TransformExecutors, + ) { + feature_reader!( + record_buffer, + transform_executors, + self.from_namespace.namespace_descriptor, + hash_index, + hash_value, + { + to_namespace.emit_i32::<{ SeedNumber::Default as usize }>( + hash_index as i32, + hash_value * self.multiplier, + ); + } + ); } } - impl TransformerWeight { - pub fn create_function( function_name: &str, - from_namespaces: &Vec, - function_params: &Vec, - ) -> Result, Box> { + pub fn create_function( + function_name: &str, + from_namespaces: &Vec, + function_params: &Vec, + ) -> Result, Box> { if function_params.len() != 1 { - return Err(Box::new(IOError::new(ErrorKind::Other, format!("Function {} takes exactly one float argument, example {}(A)(2.0)", function_name, function_name)))); + return Err(Box::new(IOError::new( + ErrorKind::Other, + format!( + "Function {} takes exactly one float argument, example {}(A)(2.0)", + function_name, function_name + ), + ))); } if from_namespaces.len() != 1 { - return Err(Box::new(IOError::new(ErrorKind::Other, format!("Function {} takes exactly one namespace argument, example {}(A)(2.0)", function_name, function_name)))); + return Err(Box::new(IOError::new( + ErrorKind::Other, + format!( + "Function {} takes exactly one namespace argument, example {}(A)(2.0)", + function_name, function_name + ), + ))); } // We do not check if input namespace is float, Weight does not require float namespace as input - - Ok(Box::new(Self{from_namespace: ExecutorFromNamespace{namespace_descriptor: from_namespaces[0].namespace_descriptor}, - multiplier: function_params[0], - })) - } -} - + Ok(Box::new(Self { + from_namespace: ExecutorFromNamespace { + namespace_descriptor: from_namespaces[0].namespace_descriptor, + }, + multiplier: function_params[0], + })) + } +} // Combine Binner // Supporting max 5 input namespaces. Because 5 ought to be enough for everybody! @@ -276,199 +401,306 @@ pub struct TransformerCombine { } impl FunctionExecutorTrait for TransformerCombine { - fn execute_function(&self, record_buffer: &[u32], to_namespace: &mut ExecutorToNamespace, transform_executors: &TransformExecutors) { + fn execute_function( + &self, + record_buffer: &[u32], + to_namespace: &mut ExecutorToNamespace, + transform_executors: &TransformExecutors, + ) { // Sure this could have been written with either using: // - Stack machine: I didn't want to introduce another dynamic layer // - Automatic code generation: Didn't have time to learn macros that well // So we are left with good old "spaghetti technique" match self.n_namespaces { - 2 => feature_reader!(record_buffer, transform_executors, self.from_namespaces[0].namespace_descriptor, hash_index0, hash_value0, { - feature_reader!(record_buffer, transform_executors, self.from_namespaces[1].namespace_descriptor, hash_index1, hash_value1, { - to_namespace.emit_i32::<{SeedNumber::Default as usize}>((hash_index0 ^ hash_index1) as i32, hash_value0 * hash_value1); - }); - }), - 3 => feature_reader!(record_buffer, transform_executors, self.from_namespaces[0].namespace_descriptor, hash_index0, hash_value0, { - feature_reader!(record_buffer, transform_executors, self.from_namespaces[1].namespace_descriptor, hash_index1, hash_value1, { - feature_reader!(record_buffer, transform_executors, self.from_namespaces[2].namespace_descriptor, hash_index2, hash_value2, { - to_namespace.emit_i32::<{SeedNumber::Default as usize}>((hash_index0 ^ hash_index1 ^ hash_index2) as i32, hash_value0 * hash_value1 * hash_value2); - }); - }); - }), - 4 => feature_reader!(record_buffer, transform_executors, self.from_namespaces[0].namespace_descriptor, hash_index0, hash_value0, { - feature_reader!(record_buffer, transform_executors, self.from_namespaces[1].namespace_descriptor, hash_index1, hash_value1, { - feature_reader!(record_buffer, transform_executors, self.from_namespaces[2].namespace_descriptor, hash_index2, hash_value2, { - feature_reader!(record_buffer, transform_executors, self.from_namespaces[3].namespace_descriptor, hash_index3, hash_value3, { - to_namespace.emit_i32::<{SeedNumber::Default as usize}>((hash_index0 ^ hash_index1 ^ hash_index2 ^ hash_index3) as i32, - hash_value0 * hash_value1 * hash_value2 * hash_value3); - }); - }); - }); - }), -/* Disabled since we have compilation time issues */ -/* 5 => feature_reader!(record_buffer, transform_executors, self.from_namespaces[0].namespace_descriptor, hash_index0, hash_value0, { - feature_reader!(record_buffer, transform_executors, self.from_namespaces[1].namespace_descriptor, hash_index1, hash_value1, { - feature_reader!(record_buffer, transform_executors, self.from_namespaces[2].namespace_descriptor, hash_index2, hash_value2, { - feature_reader!(record_buffer, transform_executors, self.from_namespaces[3].namespace_descriptor, hash_index3, hash_value3, { - feature_reader!(record_buffer, transform_executors, self.from_namespaces[4].namespace_descriptor, hash_index4, hash_value4, { - to_namespace.emit_i32::<{SeedNumber::Default as usize}>((hash_index0 ^ hash_index1 ^ hash_index2 ^ hash_index3 ^ hash_index4) as i32, - hash_value0 * hash_value1 * hash_value2 * hash_value3 * hash_value4); - }); - }); - }); - }); - }),*/ + 2 => feature_reader!( + record_buffer, + transform_executors, + self.from_namespaces[0].namespace_descriptor, + hash_index0, + hash_value0, + { + feature_reader!( + record_buffer, + transform_executors, + self.from_namespaces[1].namespace_descriptor, + hash_index1, + hash_value1, + { + to_namespace.emit_i32::<{ SeedNumber::Default as usize }>( + (hash_index0 ^ hash_index1) as i32, + hash_value0 * hash_value1, + ); + } + ); + } + ), + 3 => feature_reader!( + record_buffer, + transform_executors, + self.from_namespaces[0].namespace_descriptor, + hash_index0, + hash_value0, + { + feature_reader!( + record_buffer, + transform_executors, + self.from_namespaces[1].namespace_descriptor, + hash_index1, + hash_value1, + { + feature_reader!( + record_buffer, + transform_executors, + self.from_namespaces[2].namespace_descriptor, + hash_index2, + hash_value2, + { + to_namespace.emit_i32::<{ SeedNumber::Default as usize }>( + (hash_index0 ^ hash_index1 ^ hash_index2) as i32, + hash_value0 * hash_value1 * hash_value2, + ); + } + ); + } + ); + } + ), + 4 => feature_reader!( + record_buffer, + transform_executors, + self.from_namespaces[0].namespace_descriptor, + hash_index0, + hash_value0, + { + feature_reader!( + record_buffer, + transform_executors, + self.from_namespaces[1].namespace_descriptor, + hash_index1, + hash_value1, + { + feature_reader!( + record_buffer, + transform_executors, + self.from_namespaces[2].namespace_descriptor, + hash_index2, + hash_value2, + { + feature_reader!( + record_buffer, + transform_executors, + self.from_namespaces[3].namespace_descriptor, + hash_index3, + hash_value3, + { + to_namespace + .emit_i32::<{ SeedNumber::Default as usize }>( + (hash_index0 + ^ hash_index1 + ^ hash_index2 + ^ hash_index3) + as i32, + hash_value0 + * hash_value1 + * hash_value2 + * hash_value3, + ); + } + ); + } + ); + } + ); + } + ), + /* Disabled since we have compilation time issues */ + /* 5 => feature_reader!(record_buffer, transform_executors, self.from_namespaces[0].namespace_descriptor, hash_index0, hash_value0, { + feature_reader!(record_buffer, transform_executors, self.from_namespaces[1].namespace_descriptor, hash_index1, hash_value1, { + feature_reader!(record_buffer, transform_executors, self.from_namespaces[2].namespace_descriptor, hash_index2, hash_value2, { + feature_reader!(record_buffer, transform_executors, self.from_namespaces[3].namespace_descriptor, hash_index3, hash_value3, { + feature_reader!(record_buffer, transform_executors, self.from_namespaces[4].namespace_descriptor, hash_index4, hash_value4, { + to_namespace.emit_i32::<{SeedNumber::Default as usize}>((hash_index0 ^ hash_index1 ^ hash_index2 ^ hash_index3 ^ hash_index4) as i32, + hash_value0 * hash_value1 * hash_value2 * hash_value3 * hash_value4); + }); + }); + }); + }); + }),*/ _ => { panic!("Impossible number of from_namespaces in function TransformCombine - this should have been caught at parsing stage") - } - + } } } } impl TransformerCombine { pub fn create_function( - function_name: &str, - from_namespaces: &Vec, - function_params: &Vec, - ) -> Result, Box> { + function_name: &str, + from_namespaces: &Vec, + function_params: &Vec, + ) -> Result, Box> { if function_params.len() != 0 { - return Err(Box::new(IOError::new(ErrorKind::Other, format!("Function {} takes no float arguments {}(A)()", function_name, function_name)))); + return Err(Box::new(IOError::new( + ErrorKind::Other, + format!( + "Function {} takes no float arguments {}(A)()", + function_name, function_name + ), + ))); } if from_namespaces.len() < 2 || from_namespaces.len() > 4 { - return Err(Box::new(IOError::new(ErrorKind::Other, format!("Function {} takes between 2 and 4 namespace arguments, example {}(A,B)()", function_name, function_name)))); + return Err(Box::new(IOError::new( + ErrorKind::Other, + format!( + "Function {} takes between 2 and 4 namespace arguments, example {}(A,B)()", + function_name, function_name + ), + ))); } - // We do not need to check if the input namespace is float, Combine does not require float namespace as input + // We do not need to check if the input namespace is float, Combine does not require float namespace as input // We use fixed arrays, so we need to fill the array with defaults first - let c = ExecutorFromNamespace{namespace_descriptor: NamespaceDescriptor {namespace_index: 0, - namespace_type: NamespaceType::Primitive, - namespace_format: NamespaceFormat::Categorical}, - }; - let mut executor_from_namespaces: [ExecutorFromNamespace;4] = [c.clone(),c.clone(),c.clone(),c.clone()]; + let c = ExecutorFromNamespace { + namespace_descriptor: NamespaceDescriptor { + namespace_index: 0, + namespace_type: NamespaceType::Primitive, + namespace_format: NamespaceFormat::Categorical, + }, + }; + let mut executor_from_namespaces: [ExecutorFromNamespace; 4] = + [c.clone(), c.clone(), c.clone(), c.clone()]; for (x, namespace) in from_namespaces.iter().enumerate() { executor_from_namespaces[x].namespace_descriptor = namespace.namespace_descriptor; } - Ok(Box::new(Self{from_namespaces: executor_from_namespaces, - n_namespaces: from_namespaces.len() as u8, - })) + Ok(Box::new(Self { + from_namespaces: executor_from_namespaces, + n_namespaces: from_namespaces.len() as u8, + })) } -} - - - - - - - +} mod tests { // Note this useful idiom: importing names from outer (for mod tests) scope. use super::*; - use crate::parser::{IS_NOT_SINGLE_MASK, MASK31}; use crate::feature_transform_executor::default_seeds; + use crate::parser::{IS_NOT_SINGLE_MASK, MASK31}; fn add_header(v2: Vec) -> Vec { let mut rr: Vec = vec![100, 1, 1.0f32.to_bits()]; rr.extend(v2); rr } - + fn nd(start: u32, end: u32) -> u32 { return (start << 16) + end; } - + fn ns_desc(i: u16) -> NamespaceDescriptor { - NamespaceDescriptor {namespace_index: i, - namespace_type: NamespaceType::Primitive, - namespace_format: NamespaceFormat::Categorical, - } + NamespaceDescriptor { + namespace_index: i, + namespace_type: NamespaceType::Primitive, + namespace_format: NamespaceFormat::Categorical, + } } fn ns_desc_f32(i: u16) -> NamespaceDescriptor { - NamespaceDescriptor {namespace_index: i, - namespace_type: NamespaceType::Primitive, - namespace_format: NamespaceFormat::F32, - } - + NamespaceDescriptor { + namespace_index: i, + namespace_type: NamespaceType::Primitive, + namespace_format: NamespaceFormat::F32, + } } - #[test] fn test_transformerbinner_fail() { - // this fails because input namespace is not float namespace + // this fails because input namespace is not float namespace let from_namespace = feature_transform_parser::Namespace { namespace_verbose: "a".to_string(), namespace_descriptor: ns_desc(0), }; - + let to_namespace_empty = ExecutorToNamespace { namespace_descriptor: ns_desc(1), - namespace_seeds: default_seeds(1), // These are precomputed namespace seeds + namespace_seeds: default_seeds(1), // These are precomputed namespace seeds tmp_data: Vec::new(), }; - - let result = TransformerBinner::create_function(&(|x, y| x.sqrt() * y), "Blah", &vec![from_namespace], &vec![40., 1.4], false); - assert!(result.is_err()); + let result = TransformerBinner::create_function( + &(|x, y| x.sqrt() * y), + "Blah", + &vec![from_namespace], + &vec![40., 1.4], + false, + ); + assert!(result.is_err()); } #[test] fn test_transformerbinner() { - let from_namespace = feature_transform_parser::Namespace { namespace_descriptor: ns_desc_f32(0), namespace_verbose: "a".to_string(), }; let to_namespace_index = 1; - + let to_namespace_empty = ExecutorToNamespace { namespace_descriptor: ns_desc(to_namespace_index), - namespace_seeds: default_seeds(to_namespace_index as u32), // These are precomputed namespace seeds + namespace_seeds: default_seeds(to_namespace_index as u32), // These are precomputed namespace seeds tmp_data: Vec::new(), }; - - let transformer = TransformerBinner::create_function(&(|x, y| x.sqrt() * y), "Blah", &vec![from_namespace], &vec![40.0, 1.], false).unwrap(); - let record_buffer = [6, // length - 0, // label - (1.0_f32).to_bits(), // Example weight - nd(4, 6) | IS_NOT_SINGLE_MASK, - // Feature triple - 1775699190 & MASK31, // Hash location - 3.0f32.to_bits()]; // Float feature value - + + let transformer = TransformerBinner::create_function( + &(|x, y| x.sqrt() * y), + "Blah", + &vec![from_namespace], + &vec![40.0, 1.], + false, + ) + .unwrap(); + let record_buffer = [ + 6, // length + 0, // label + (1.0_f32).to_bits(), // Example weight + nd(4, 6) | IS_NOT_SINGLE_MASK, + // Feature triple + 1775699190 & MASK31, // Hash location + 3.0f32.to_bits(), + ]; // Float feature value + let mut to_namespace = to_namespace_empty.clone(); - let mut transform_executors = TransformExecutors {executors: vec![]}; // not used + let mut transform_executors = TransformExecutors { executors: vec![] }; // not used transformer.execute_function(&record_buffer, &mut to_namespace, &mut transform_executors); // Couldn't get mocking to work, so instead of intercepting call to emit_i32, we just repeat it and see if the results match let mut to_namespace_comparison = to_namespace_empty.clone(); - to_namespace_comparison.emit_i32::<{SeedNumber::Default as usize}>(3, 1.0f32); + to_namespace_comparison.emit_i32::<{ SeedNumber::Default as usize }>(3, 1.0f32); assert_eq!(to_namespace.tmp_data, to_namespace_comparison.tmp_data); - - + // Now let's try with value> 40.0 - let record_buffer = [6, // length - 0, // label - (1.0_f32).to_bits(), // Example weight - nd(4, 6) | IS_NOT_SINGLE_MASK, - // Feature triple - 1775699190 & MASK31, // Hash location - 300.0f32.to_bits()]; // Float feature value + let record_buffer = [ + 6, // length + 0, // label + (1.0_f32).to_bits(), // Example weight + nd(4, 6) | IS_NOT_SINGLE_MASK, + // Feature triple + 1775699190 & MASK31, // Hash location + 300.0f32.to_bits(), + ]; // Float feature value let mut to_namespace = to_namespace_empty.clone(); transformer.execute_function(&record_buffer, &mut to_namespace, &mut transform_executors); // Couldn't get mocking to work, so instead of intercepting call to emit_i32, we just repeat it and see if the results match let mut to_namespace_comparison = to_namespace_empty.clone(); - to_namespace_comparison.emit_i32::<{SeedNumber::One as usize}>((300.0_f32 - 40.0_f32).sqrt() as i32, 1.0f32); - assert_eq!(to_namespace.tmp_data, to_namespace_comparison.tmp_data); + to_namespace_comparison + .emit_i32::<{ SeedNumber::One as usize }>((300.0_f32 - 40.0_f32).sqrt() as i32, 1.0f32); + assert_eq!(to_namespace.tmp_data, to_namespace_comparison.tmp_data); } #[test] fn test_transformerlogratiobinner() { - let from_namespace_1 = feature_transform_parser::Namespace { namespace_descriptor: ns_desc_f32(0), namespace_verbose: "a".to_string(), @@ -478,255 +710,277 @@ mod tests { namespace_descriptor: ns_desc_f32(1), namespace_verbose: "c".to_string(), }; - - let to_namespace_index = 1; + + let to_namespace_index = 1; let to_namespace_empty = ExecutorToNamespace { namespace_descriptor: ns_desc(to_namespace_index), - namespace_seeds: default_seeds(to_namespace_index as u32), // These are precomputed namespace seeds + namespace_seeds: default_seeds(to_namespace_index as u32), // These are precomputed namespace seeds tmp_data: Vec::new(), }; - - let transformer = TransformerLogRatioBinner::create_function("Blah", &vec![from_namespace_1, from_namespace_2], &vec![40.0, 10.], false).unwrap(); - let record_buffer = [9, // length - 0, // label - (1.0_f32).to_bits(), // Example weight - nd(5, 7) | IS_NOT_SINGLE_MASK, - nd(7, 9) | IS_NOT_SINGLE_MASK, - // Feature triple - 1775699190 & MASK31, // Hash location - 3.0f32.to_bits(), - // Feature triple - 1775699190 & MASK31, // Hash location - 7.0f32.to_bits(), - ]; // Float feature value - + + let transformer = TransformerLogRatioBinner::create_function( + "Blah", + &vec![from_namespace_1, from_namespace_2], + &vec![40.0, 10.], + false, + ) + .unwrap(); + let record_buffer = [ + 9, // length + 0, // label + (1.0_f32).to_bits(), // Example weight + nd(5, 7) | IS_NOT_SINGLE_MASK, + nd(7, 9) | IS_NOT_SINGLE_MASK, + // Feature triple + 1775699190 & MASK31, // Hash location + 3.0f32.to_bits(), + // Feature triple + 1775699190 & MASK31, // Hash location + 7.0f32.to_bits(), + ]; // Float feature value + let mut to_namespace = to_namespace_empty.clone(); - let mut transform_executors = TransformExecutors {executors: vec![]}; // not used + let mut transform_executors = TransformExecutors { executors: vec![] }; // not used transformer.execute_function(&record_buffer, &mut to_namespace, &mut transform_executors); // Couldn't get mocking to work, so instead of intercepting call to emit_i32, we just repeat it and see if the results match let mut to_namespace_comparison = to_namespace_empty.clone(); - to_namespace_comparison.emit_i32_i32::<{SeedNumber::One as usize}>(3 as i32, 7 as i32, 1.0); + to_namespace_comparison + .emit_i32_i32::<{ SeedNumber::One as usize }>(3 as i32, 7 as i32, 1.0); assert_eq!(to_namespace.tmp_data, to_namespace_comparison.tmp_data); - - // Now let's have 30.0/60.0 - let record_buffer = [9, // length - 0, // label - (1.0_f32).to_bits(), // Example weight - nd(5, 7) | IS_NOT_SINGLE_MASK, - nd(7, 9) | IS_NOT_SINGLE_MASK, - // Feature triple - 1775699190 & MASK31, // Hash location - 30.0f32.to_bits(), - // Feature triple - 1775699190 & MASK31, // Hash location - 60.0f32.to_bits(), - - ]; // Float feature value - + let record_buffer = [ + 9, // length + 0, // label + (1.0_f32).to_bits(), // Example weight + nd(5, 7) | IS_NOT_SINGLE_MASK, + nd(7, 9) | IS_NOT_SINGLE_MASK, + // Feature triple + 1775699190 & MASK31, // Hash location + 30.0f32.to_bits(), + // Feature triple + 1775699190 & MASK31, // Hash location + 60.0f32.to_bits(), + ]; // Float feature value + let mut to_namespace = to_namespace_empty.clone(); - let mut transform_executors = TransformExecutors {executors: vec![]}; // not used + let mut transform_executors = TransformExecutors { executors: vec![] }; // not used transformer.execute_function(&record_buffer, &mut to_namespace, &mut transform_executors); // Couldn't get mocking to work, so instead of intercepting call to emit_i32, we just repeat it and see if the results match let mut to_namespace_comparison = to_namespace_empty.clone(); - to_namespace_comparison.emit_f32::<{SeedNumber::Default as usize}>((30.0/60.0_f32).ln() * 10.0, 1.0, false); + to_namespace_comparison.emit_f32::<{ SeedNumber::Default as usize }>( + (30.0 / 60.0_f32).ln() * 10.0, + 1.0, + false, + ); assert_eq!(to_namespace.tmp_data, to_namespace_comparison.tmp_data); - // Now let's have 30.0/0.0 - let record_buffer = [9, // length - 0, // label - (1.0_f32).to_bits(), // Example weight - nd(5, 7) | IS_NOT_SINGLE_MASK, - nd(7, 9) | IS_NOT_SINGLE_MASK, - // Feature triple - 1775699190 & MASK31, // Hash location - 30.0f32.to_bits(), - // Feature triple - 1775699190 & MASK31, // Hash location - 0.0f32.to_bits(), - - ]; // Float feature value - + let record_buffer = [ + 9, // length + 0, // label + (1.0_f32).to_bits(), // Example weight + nd(5, 7) | IS_NOT_SINGLE_MASK, + nd(7, 9) | IS_NOT_SINGLE_MASK, + // Feature triple + 1775699190 & MASK31, // Hash location + 30.0f32.to_bits(), + // Feature triple + 1775699190 & MASK31, // Hash location + 0.0f32.to_bits(), + ]; // Float feature value + let mut to_namespace = to_namespace_empty.clone(); - let mut transform_executors = TransformExecutors {executors: vec![]}; // not used + let mut transform_executors = TransformExecutors { executors: vec![] }; // not used transformer.execute_function(&record_buffer, &mut to_namespace, &mut transform_executors); // Couldn't get mocking to work, so instead of intercepting call to emit_i32, we just repeat it and see if the results match let mut to_namespace_comparison = to_namespace_empty.clone(); - to_namespace_comparison.emit_i32_i32::<{SeedNumber::One as usize}>(30, 0, 1.0); + to_namespace_comparison.emit_i32_i32::<{ SeedNumber::One as usize }>(30, 0, 1.0); assert_eq!(to_namespace.tmp_data, to_namespace_comparison.tmp_data); // Now let's have 0.0/50.0 - let record_buffer = [9, // length - 0, // label - (1.0_f32).to_bits(), // Example weight - nd(5, 7) | IS_NOT_SINGLE_MASK, - nd(7, 9) | IS_NOT_SINGLE_MASK, - // Feature triple - 1775699190 & MASK31, // Hash location - 0.0f32.to_bits(), - // Feature triple - 1775699190 & MASK31, // Hash location - 50.0f32.to_bits(), - - ]; // Float feature value - + let record_buffer = [ + 9, // length + 0, // label + (1.0_f32).to_bits(), // Example weight + nd(5, 7) | IS_NOT_SINGLE_MASK, + nd(7, 9) | IS_NOT_SINGLE_MASK, + // Feature triple + 1775699190 & MASK31, // Hash location + 0.0f32.to_bits(), + // Feature triple + 1775699190 & MASK31, // Hash location + 50.0f32.to_bits(), + ]; // Float feature value + let mut to_namespace = to_namespace_empty.clone(); - let mut transform_executors = TransformExecutors {executors: vec![]}; // not used + let mut transform_executors = TransformExecutors { executors: vec![] }; // not used transformer.execute_function(&record_buffer, &mut to_namespace, &mut transform_executors); // Couldn't get mocking to work, so instead of intercepting call to emit_i32, we just repeat it and see if the results match let mut to_namespace_comparison = to_namespace_empty.clone(); - to_namespace_comparison.emit_f32::<{SeedNumber::Two as usize}>((50_f32 - 40_f32).ln(), 1.0, false); + to_namespace_comparison.emit_f32::<{ SeedNumber::Two as usize }>( + (50_f32 - 40_f32).ln(), + 1.0, + false, + ); assert_eq!(to_namespace.tmp_data, to_namespace_comparison.tmp_data); - - // Now let's have 50.0/0.0 - let record_buffer = [9, // length - 0, // label - (1.0_f32).to_bits(), // Example weight - nd(5, 7) | IS_NOT_SINGLE_MASK, - nd(7, 9) | IS_NOT_SINGLE_MASK, - // Feature triple - 1775699190 & MASK31, // Hash location - 50.0f32.to_bits(), - // Feature triple - 1775699190 & MASK31, // Hash location - 0.0f32.to_bits(), - - ]; // Float feature value - + let record_buffer = [ + 9, // length + 0, // label + (1.0_f32).to_bits(), // Example weight + nd(5, 7) | IS_NOT_SINGLE_MASK, + nd(7, 9) | IS_NOT_SINGLE_MASK, + // Feature triple + 1775699190 & MASK31, // Hash location + 50.0f32.to_bits(), + // Feature triple + 1775699190 & MASK31, // Hash location + 0.0f32.to_bits(), + ]; // Float feature value + let mut to_namespace = to_namespace_empty.clone(); - let mut transform_executors = TransformExecutors {executors: vec![]}; // not used + let mut transform_executors = TransformExecutors { executors: vec![] }; // not used transformer.execute_function(&record_buffer, &mut to_namespace, &mut transform_executors); // Couldn't get mocking to work, so instead of intercepting call to emit_i32, we just repeat it and see if the results match let mut to_namespace_comparison = to_namespace_empty.clone(); - to_namespace_comparison.emit_f32::<{SeedNumber::Three as usize}>((50_f32 - 40_f32).ln(), 1.0, false); + to_namespace_comparison.emit_f32::<{ SeedNumber::Three as usize }>( + (50_f32 - 40_f32).ln(), + 1.0, + false, + ); assert_eq!(to_namespace.tmp_data, to_namespace_comparison.tmp_data); - - - } - #[test] fn test_transformerweightmutliplier() { - let from_namespace_float = feature_transform_parser::Namespace { namespace_descriptor: ns_desc_f32(0), namespace_verbose: "a".to_string(), }; let to_namespace_index = 1; - + let to_namespace_empty = ExecutorToNamespace { namespace_descriptor: ns_desc(to_namespace_index), - namespace_seeds: default_seeds(to_namespace_index as u32), // These are precomputed namespace seeds + namespace_seeds: default_seeds(to_namespace_index as u32), // These are precomputed namespace seeds tmp_data: Vec::new(), }; - - let transformer = TransformerWeight::create_function("Blah", &vec![from_namespace_float], &vec![40.]).unwrap(); - let record_buffer = [6, // length - 0, // label - (1.0_f32).to_bits(), // Example weight - nd(4, 6) | IS_NOT_SINGLE_MASK, - // Feature triple - 1775699190 & MASK31, // Hash location - 3.0f32.to_bits()]; // Float feature value - + + let transformer = + TransformerWeight::create_function("Blah", &vec![from_namespace_float], &vec![40.]) + .unwrap(); + let record_buffer = [ + 6, // length + 0, // label + (1.0_f32).to_bits(), // Example weight + nd(4, 6) | IS_NOT_SINGLE_MASK, + // Feature triple + 1775699190 & MASK31, // Hash location + 3.0f32.to_bits(), + ]; // Float feature value + let mut to_namespace = to_namespace_empty.clone(); - let mut transform_executors = TransformExecutors {executors: vec![]}; // not used + let mut transform_executors = TransformExecutors { executors: vec![] }; // not used transformer.execute_function(&record_buffer, &mut to_namespace, &mut transform_executors); // Couldn't get mocking to work, so instead of intercepting call to emit_i32, we just repeat it and see if the results match let mut to_namespace_comparison = to_namespace_empty.clone(); - to_namespace_comparison.emit_i32::<{SeedNumber::Default as usize}>((1775699190 & MASK31) as i32, 1.0f32 * 40.); + to_namespace_comparison.emit_i32::<{ SeedNumber::Default as usize }>( + (1775699190 & MASK31) as i32, + 1.0f32 * 40., + ); assert_eq!(to_namespace.tmp_data, to_namespace_comparison.tmp_data); - + // But weightmultiplier can take non-float namespaces let from_namespace_nonfloat = feature_transform_parser::Namespace { namespace_descriptor: ns_desc(0), namespace_verbose: "a".to_string(), }; - let transformer = TransformerWeight::create_function("Blah", &vec![from_namespace_nonfloat], &vec![40.]).unwrap(); - let record_buffer = [7, // length - 0, // label - (1.0_f32).to_bits(), // Example weight - nd(4, 6) | IS_NOT_SINGLE_MASK, - // Feature triple - 1775699190 & MASK31, // Hash location - 2.0f32.to_bits()]; // Feature value of the feature - + let transformer = + TransformerWeight::create_function("Blah", &vec![from_namespace_nonfloat], &vec![40.]) + .unwrap(); + let record_buffer = [ + 7, // length + 0, // label + (1.0_f32).to_bits(), // Example weight + nd(4, 6) | IS_NOT_SINGLE_MASK, + // Feature triple + 1775699190 & MASK31, // Hash location + 2.0f32.to_bits(), + ]; // Feature value of the feature + let mut to_namespace = to_namespace_empty.clone(); - let mut transform_executors = TransformExecutors {executors: vec![]}; // not used + let mut transform_executors = TransformExecutors { executors: vec![] }; // not used transformer.execute_function(&record_buffer, &mut to_namespace, &mut transform_executors); // Couldn't get mocking to work, so instead of intercepting call to emit_i32, we just repeat it and see if the results match let mut to_namespace_comparison = to_namespace_empty.clone(); - to_namespace_comparison.emit_i32::<{SeedNumber::Default as usize}>((1775699190 & MASK31) as i32, 2.0f32 * 40.); + to_namespace_comparison.emit_i32::<{ SeedNumber::Default as usize }>( + (1775699190 & MASK31) as i32, + 2.0f32 * 40., + ); assert_eq!(to_namespace.tmp_data, to_namespace_comparison.tmp_data); - - - } - #[test] fn test_transformercombine() { - let from_namespace_1 = feature_transform_parser::Namespace { namespace_descriptor: ns_desc_f32(0), namespace_verbose: "a".to_string(), }; - let from_namespace_2 = feature_transform_parser::Namespace { namespace_descriptor: ns_desc(1), namespace_verbose: "b".to_string(), }; let to_namespace_index = 2; - + let to_namespace_empty = ExecutorToNamespace { namespace_descriptor: ns_desc(to_namespace_index), - namespace_seeds: default_seeds(to_namespace_index as u32), // These are precomputed namespace seeds + namespace_seeds: default_seeds(to_namespace_index as u32), // These are precomputed namespace seeds tmp_data: Vec::new(), }; - - let transformer = TransformerCombine::create_function("Blah", &vec![from_namespace_1, from_namespace_2], &vec![]).unwrap(); - - let record_buffer = [9, // length - 0, // label - (1.0_f32).to_bits(), // Example weight - nd(5, 7) | IS_NOT_SINGLE_MASK, - nd(7, 9) | IS_NOT_SINGLE_MASK, - // Feature triple - 1775699190 & MASK31, // Hash location - 3.0f32.to_bits(), // Float value of the feature - // Feature triple - 1775699190 & MASK31, // Hash location - 3.0f32.to_bits(), // Weight of the feature - ]; + + let transformer = TransformerCombine::create_function( + "Blah", + &vec![from_namespace_1, from_namespace_2], + &vec![], + ) + .unwrap(); + + let record_buffer = [ + 9, // length + 0, // label + (1.0_f32).to_bits(), // Example weight + nd(5, 7) | IS_NOT_SINGLE_MASK, + nd(7, 9) | IS_NOT_SINGLE_MASK, + // Feature triple + 1775699190 & MASK31, // Hash location + 3.0f32.to_bits(), // Float value of the feature + // Feature triple + 1775699190 & MASK31, // Hash location + 3.0f32.to_bits(), // Weight of the feature + ]; let mut to_namespace = to_namespace_empty.clone(); - let mut transform_executors = TransformExecutors {executors: vec![]}; // not used + let mut transform_executors = TransformExecutors { executors: vec![] }; // not used transformer.execute_function(&record_buffer, &mut to_namespace, &mut transform_executors); // Couldn't get mocking to work, so instead of intercepting call to emit_i32, we just repeat it and see if the results match let mut to_namespace_comparison = to_namespace_empty.clone(); - to_namespace_comparison.emit_i32::<{SeedNumber::Default as usize}>((1775699190 ^ 1775699190) as i32, 3.0f32); + to_namespace_comparison + .emit_i32::<{ SeedNumber::Default as usize }>((1775699190 ^ 1775699190) as i32, 3.0f32); assert_eq!(to_namespace.tmp_data, to_namespace_comparison.tmp_data); } - - } - diff --git a/src/feature_transform_parser.rs b/src/feature_transform_parser.rs index 086b0b6b..6a6499ab 100644 --- a/src/feature_transform_parser.rs +++ b/src/feature_transform_parser.rs @@ -2,18 +2,16 @@ //extern crate nom; use crate::vwmap; +use serde::{Deserialize, Serialize}; +use std::cell::Cell; +use std::collections::HashMap; use std::error::Error; use std::io::Error as IOError; use std::io::ErrorKind; -use std::collections::HashMap; -use std::cell::Cell; -use serde::{Serialize,Deserialize}; - use crate::feature_transform_executor; -pub const TRANSFORM_NAMESPACE_MARK: u32 = 1<< 31; - +pub const TRANSFORM_NAMESPACE_MARK: u32 = 1 << 31; #[derive(Serialize, Deserialize, Debug, Clone, PartialEq)] pub struct Namespace { @@ -38,14 +36,12 @@ struct NSStage1Parse { name: String, definition: String, from_namespaces: Vec, - processing: Cell, + processing: Cell, done: Cell, } - pub struct NamespaceTransformsParser { - denormalized: HashMap , // to_namespace_str -> list of from_namespace_str - + denormalized: HashMap, // to_namespace_str -> list of from_namespace_str } impl NamespaceTransformsParser { @@ -55,65 +51,107 @@ impl NamespaceTransformsParser { // - checks for cyclic dependencies // - processes transformations in the right order, so from namespaces are available when to namespace is being processed pub fn new() -> NamespaceTransformsParser { - NamespaceTransformsParser {denormalized: HashMap::new()} + NamespaceTransformsParser { + denormalized: HashMap::new(), + } } - pub fn add_transform_namespace(&mut self, vw: &vwmap::VwNamespaceMap, s: &str) -> Result<(), Box> { + pub fn add_transform_namespace( + &mut self, + vw: &vwmap::VwNamespaceMap, + s: &str, + ) -> Result<(), Box> { let rr = parse_namespace_statement(s); if rr.is_err() { - return Err(Box::new(IOError::new(ErrorKind::Other, format!("Error parsing {}\n{:?}", s, rr)))); + return Err(Box::new(IOError::new( + ErrorKind::Other, + format!("Error parsing {}\n{:?}", s, rr), + ))); } - let (_, (to_namespace_verbose, function_name, from_namespaces_verbose, function_parameters)) = rr.unwrap(); + let ( + _, + (to_namespace_verbose, function_name, from_namespaces_verbose, function_parameters), + ) = rr.unwrap(); // Here we just check for clashes with namespaces from input file - let namespace_descriptor = vw.map_verbose_to_namespace_descriptor.get(&to_namespace_verbose); + let namespace_descriptor = vw + .map_verbose_to_namespace_descriptor + .get(&to_namespace_verbose); if namespace_descriptor.is_some() { - return Err(Box::new(IOError::new(ErrorKind::Other, format!("To namespace of {} already exists as primitive namespace: {:?}", s, to_namespace_verbose)))); + return Err(Box::new(IOError::new( + ErrorKind::Other, + format!( + "To namespace of {} already exists as primitive namespace: {:?}", + s, to_namespace_verbose + ), + ))); } - - self.denormalized.insert(to_namespace_verbose.to_owned(), NSStage1Parse { - name: to_namespace_verbose.to_owned(), - definition: s.to_string(), - from_namespaces: from_namespaces_verbose, - processing: Cell::new(false), - done: Cell::new(false), - }); + + self.denormalized.insert( + to_namespace_verbose.to_owned(), + NSStage1Parse { + name: to_namespace_verbose.to_owned(), + definition: s.to_string(), + from_namespaces: from_namespaces_verbose, + processing: Cell::new(false), + done: Cell::new(false), + }, + ); Ok(()) } - - pub fn resolve(&mut self, vw: &vwmap::VwNamespaceMap) -> Result> { + + pub fn resolve( + &mut self, + vw: &vwmap::VwNamespaceMap, + ) -> Result> { let mut nst = NamespaceTransforms::new(); - let mut namespaces:Vec<&String> = self.denormalized.keys().collect(); - namespaces.sort(); // ensure determinism + let mut namespaces: Vec<&String> = self.denormalized.keys().collect(); + namespaces.sort(); // ensure determinism for key in &namespaces { self.depth_first_search(vw, &mut nst, key)?; } Ok(nst) } - - pub fn depth_first_search(&self, - vw: &vwmap::VwNamespaceMap, - nst: &mut NamespaceTransforms, - verbose_name: &str) - -> Result<(), Box> { + + pub fn depth_first_search( + &self, + vw: &vwmap::VwNamespaceMap, + nst: &mut NamespaceTransforms, + verbose_name: &str, + ) -> Result<(), Box> { // If feature is primitive feature, we don't need to dive deeper - if vw.map_verbose_to_namespace_descriptor.get(verbose_name).is_some() { - return Ok(()) + if vw + .map_verbose_to_namespace_descriptor + .get(verbose_name) + .is_some() + { + return Ok(()); } let n = match self.denormalized.get(verbose_name) { Some(n) => n, - None => return Err(Box::new(IOError::new(ErrorKind::Other, format!("Could not find namespace {:?}", verbose_name)))) + None => { + return Err(Box::new(IOError::new( + ErrorKind::Other, + format!("Could not find namespace {:?}", verbose_name), + ))) + } }; if n.done.get() { - return Ok(()) + return Ok(()); } - + if n.processing.get() { - return Err(Box::new(IOError::new(ErrorKind::Other, format!("Cyclic dependency detected, one of the namespaces involved is {:?}", verbose_name)))); - } - + return Err(Box::new(IOError::new( + ErrorKind::Other, + format!( + "Cyclic dependency detected, one of the namespaces involved is {:?}", + verbose_name + ), + ))); + } + n.processing.set(true); for from_namespace in &n.from_namespaces { self.depth_first_search(vw, nst, &from_namespace)?; @@ -123,9 +161,7 @@ impl NamespaceTransformsParser { n.processing.set(false); n.done.set(true); Ok(()) - } - } // We need two-stage parsing of the transforms, so user doesn't need to specify them in order @@ -133,128 +169,157 @@ impl NamespaceTransformsParser { // then we do directed graph search and only process transformations where we have all inputs defined // and give error on circular dependencies - - impl NamespaceTransforms { pub fn new() -> NamespaceTransforms { - NamespaceTransforms { v: Vec::new(), - } + NamespaceTransforms { v: Vec::new() } } - - - + fn add_transform(&mut self, vw: &vwmap::VwNamespaceMap, s: &str) -> Result<(), Box> { let rr = parse_namespace_statement(s); if rr.is_err() { - return Err(Box::new(IOError::new(ErrorKind::Other, format!("Error parsing {}\n{:?}", s, rr)))); + return Err(Box::new(IOError::new( + ErrorKind::Other, + format!("Error parsing {}\n{:?}", s, rr), + ))); } - let (_, (to_namespace_verbose, function_name, from_namespaces_verbose, function_parameters)) = rr.unwrap(); - let to_namespace_descriptor = get_namespace_descriptor_verbose(self, vw, &to_namespace_verbose); + let ( + _, + (to_namespace_verbose, function_name, from_namespaces_verbose, function_parameters), + ) = rr.unwrap(); + let to_namespace_descriptor = + get_namespace_descriptor_verbose(self, vw, &to_namespace_verbose); if to_namespace_descriptor.is_ok() { - return Err(Box::new(IOError::new(ErrorKind::Other, format!("To namespace of {} already exists: {:?}", s, to_namespace_verbose)))); + return Err(Box::new(IOError::new( + ErrorKind::Other, + format!( + "To namespace of {} already exists: {:?}", + s, to_namespace_verbose + ), + ))); } let to_namespace_descriptor = vwmap::NamespaceDescriptor { - namespace_index: self.v.len() as u16, - namespace_type: vwmap::NamespaceType::Transformed, - namespace_format: vwmap::NamespaceFormat::Categorical, // For now all to-namespaces are categorical - }; - + namespace_index: self.v.len() as u16, + namespace_type: vwmap::NamespaceType::Transformed, + namespace_format: vwmap::NamespaceFormat::Categorical, // For now all to-namespaces are categorical + }; + let to_namespace = Namespace { namespace_descriptor: to_namespace_descriptor, namespace_verbose: to_namespace_verbose.to_owned(), }; let mut from_namespaces: Vec = Vec::new(); for from_namespace_verbose in &from_namespaces_verbose { - let from_namespace_descriptor = get_namespace_descriptor_verbose(self, vw, from_namespace_verbose)?; - from_namespaces.push(Namespace{ namespace_descriptor: from_namespace_descriptor, - namespace_verbose: from_namespace_verbose.to_string() - }); + let from_namespace_descriptor = + get_namespace_descriptor_verbose(self, vw, from_namespace_verbose)?; + from_namespaces.push(Namespace { + namespace_descriptor: from_namespace_descriptor, + namespace_verbose: from_namespace_verbose.to_string(), + }); } - - // Quadratic for loop... this never goes wrong! + + // Quadratic for loop... this never goes wrong! for (i, from_namespace_1) in from_namespaces.iter().enumerate() { - for from_namespace_2 in &from_namespaces[i+1..] { + for from_namespace_2 in &from_namespaces[i + 1..] { if from_namespace_1.namespace_descriptor == from_namespace_2.namespace_descriptor { return Err(Box::new(IOError::new(ErrorKind::Other, format!("Using the same from namespace in multiple arguments to a function is not supported: {:?}", from_namespace_1.namespace_verbose)))); } } } - let nt = NamespaceTransform { from_namespaces: from_namespaces, to_namespace: to_namespace, function_name: function_name, function_parameters: function_parameters, }; - - // Now we try to setup a function and then throw it away - for early validation + + // Now we try to setup a function and then throw it away - for early validation let _ = feature_transform_executor::TransformExecutor::from_namespace_transform(&nt)?; - + self.v.push(nt); Ok(()) - } - - - } -pub fn get_namespace_descriptor(transform_namespaces: &NamespaceTransforms, vw: &vwmap::VwNamespaceMap, namespace_char: char) - -> Result> { - // Does not support transformed names - let namespace_descriptor = match vw.map_vwname_to_namespace_descriptor.get(&vec![namespace_char as u8]) { - Some(namespace_descriptor) => return Ok(*namespace_descriptor), - None => return Err(Box::new(IOError::new(ErrorKind::Other, format!("Unknown namespace char in command line: {}", namespace_char)))) - }; +pub fn get_namespace_descriptor( + transform_namespaces: &NamespaceTransforms, + vw: &vwmap::VwNamespaceMap, + namespace_char: char, +) -> Result> { + // Does not support transformed names + let namespace_descriptor = match vw + .map_vwname_to_namespace_descriptor + .get(&vec![namespace_char as u8]) + { + Some(namespace_descriptor) => return Ok(*namespace_descriptor), + None => { + return Err(Box::new(IOError::new( + ErrorKind::Other, + format!("Unknown namespace char in command line: {}", namespace_char), + ))) + } + }; } -pub fn get_namespace_descriptor_verbose(transform_namespaces: &NamespaceTransforms, vw: &vwmap::VwNamespaceMap, namespace_verbose: &str) - -> Result> { - let namespace_descriptor = match vw.map_verbose_to_namespace_descriptor.get(namespace_verbose) { - Some(namespace_descriptor) => return Ok(*namespace_descriptor), - None => { - // Yes, we do linear search, we only call this couple of times. It's fast enough - let f:Vec<&NamespaceTransform> = transform_namespaces.v.iter().filter(|x| x.to_namespace.namespace_verbose == namespace_verbose).collect(); - if f.len() == 0 { - return Err(Box::new(IOError::new(ErrorKind::Other, format!("Unknown verbose namespace in command line: {}", namespace_verbose)))); - } else { - return Ok(f[0].to_namespace.namespace_descriptor); - } - } - }; +pub fn get_namespace_descriptor_verbose( + transform_namespaces: &NamespaceTransforms, + vw: &vwmap::VwNamespaceMap, + namespace_verbose: &str, +) -> Result> { + let namespace_descriptor = match vw + .map_verbose_to_namespace_descriptor + .get(namespace_verbose) + { + Some(namespace_descriptor) => return Ok(*namespace_descriptor), + None => { + // Yes, we do linear search, we only call this couple of times. It's fast enough + let f: Vec<&NamespaceTransform> = transform_namespaces + .v + .iter() + .filter(|x| x.to_namespace.namespace_verbose == namespace_verbose) + .collect(); + if f.len() == 0 { + return Err(Box::new(IOError::new( + ErrorKind::Other, + format!( + "Unknown verbose namespace in command line: {}", + namespace_verbose + ), + ))); + } else { + return Ok(f[0].to_namespace.namespace_descriptor); + } + } + }; } - -use nom::IResult; -use nom::character::complete; +use nom; use nom::bytes::complete::take_while; -use nom::AsChar; -use nom::sequence::tuple; -use nom::number; use nom::character; -use nom; - +use nom::character::complete; +use nom::number; +use nom::sequence::tuple; +use nom::AsChar; +use nom::IResult; -pub fn name_char(c:char) -> bool { +pub fn name_char(c: char) -> bool { if AsChar::is_alphanum(c) || c == '_' { return true; } else { return false; } - } // identifier = namespace or function name pub fn parse_identifier(input: &str) -> IResult<&str, String> { let (input, (_, first_char, rest, _)) = tuple(( - character::complete::space0, - complete::one_of("abcdefghijklmnopqrstuvwzxyABCDEFGHIJKLMNOPQRSTUVWZXY_"), - take_while(name_char), - character::complete::space0 - ))(input)?; + character::complete::space0, + complete::one_of("abcdefghijklmnopqrstuvwzxyABCDEFGHIJKLMNOPQRSTUVWZXY_"), + take_while(name_char), + character::complete::space0, + ))(input)?; let mut s = first_char.to_string(); s.push_str(rest); @@ -262,72 +327,89 @@ pub fn parse_identifier(input: &str) -> IResult<&str, String> { } pub fn parse_function_params_namespaces(input: &str) -> IResult<&str, Vec> { - let take_open = complete::char('('); - let take_close = complete::char(')'); - let take_separator = complete::char(','); - let (input, (_, namespaces_str, _)) = tuple((take_open, nom::multi::separated_list1(take_separator, parse_identifier), take_close))(input)?; + let take_open = complete::char('('); + let take_close = complete::char(')'); + let take_separator = complete::char(','); + let (input, (_, namespaces_str, _)) = tuple(( + take_open, + nom::multi::separated_list1(take_separator, parse_identifier), + take_close, + ))(input)?; Ok((input, namespaces_str)) } pub fn parse_float(input: &str) -> IResult<&str, f32> { - let (input, (_, f, _)) = tuple((character::complete::space0, - number::complete::float, - character::complete::space0 - ))(input)?; + let (input, (_, f, _)) = tuple(( + character::complete::space0, + number::complete::float, + character::complete::space0, + ))(input)?; Ok((input, f)) } pub fn parse_function_params_floats(input: &str) -> IResult<&str, Vec> { - let take_open = complete::char('('); - let take_close = complete::char(')'); - let take_separator = complete::char(','); - let (input, (_, namespaces_str, _)) = tuple((take_open, nom::multi::separated_list0(take_separator, parse_float), take_close))(input)?; + let take_open = complete::char('('); + let take_close = complete::char(')'); + let take_separator = complete::char(','); + let (input, (_, namespaces_str, _)) = tuple(( + take_open, + nom::multi::separated_list0(take_separator, parse_float), + take_close, + ))(input)?; Ok((input, namespaces_str)) } - -pub fn parse_namespace_statement(input: &str) -> IResult<&str, (String, String, Vec, Vec)> { - - let (input, (to_namespace_verbose, _, function_name, from_namespace_verbose, parameters)) = +pub fn parse_namespace_statement( + input: &str, +) -> IResult<&str, (String, String, Vec, Vec)> { + let (input, (to_namespace_verbose, _, function_name, from_namespace_verbose, parameters)) = tuple(( parse_identifier, complete::char('='), parse_identifier, parse_function_params_namespaces, - parse_function_params_floats - ))(input)?; - - Ok((input, (to_namespace_verbose, function_name, from_namespace_verbose, parameters))) + parse_function_params_floats, + ))(input)?; + + Ok(( + input, + ( + to_namespace_verbose, + function_name, + from_namespace_verbose, + parameters, + ), + )) } - - mod tests { // Note this useful idiom: importing names from outer (for mod tests) scope. use super::*; - use crate::vwmap::{NamespaceType, NamespaceFormat, NamespaceDescriptor, VwNamespaceMap}; + use crate::vwmap::{NamespaceDescriptor, NamespaceFormat, NamespaceType, VwNamespaceMap}; fn ns_desc(i: u16) -> NamespaceDescriptor { - NamespaceDescriptor {namespace_index: i, - namespace_type: NamespaceType::Primitive, - namespace_format: NamespaceFormat::Categorical, - } + NamespaceDescriptor { + namespace_index: i, + namespace_type: NamespaceType::Primitive, + namespace_format: NamespaceFormat::Categorical, + } } fn ns_desc_trans(i: u16) -> NamespaceDescriptor { - NamespaceDescriptor {namespace_index: i, - namespace_type: NamespaceType::Transformed, - namespace_format: NamespaceFormat::Categorical, - } + NamespaceDescriptor { + namespace_index: i, + namespace_type: NamespaceType::Transformed, + namespace_format: NamespaceFormat::Categorical, + } } fn ns_desc_f32(i: u16) -> NamespaceDescriptor { - NamespaceDescriptor {namespace_index: i, - namespace_type: NamespaceType::Primitive, - namespace_format: NamespaceFormat::F32, - } + NamespaceDescriptor { + namespace_index: i, + namespace_type: NamespaceType::Primitive, + namespace_format: NamespaceFormat::F32, + } } - #[test] fn test_namespace_transforms() { @@ -345,7 +427,10 @@ C,featureC,f32 let nst = nstp.resolve(&vw).unwrap(); assert_eq!(nst.v[0].to_namespace.namespace_descriptor, ns_desc_trans(0)); assert_eq!(nst.v[0].from_namespaces[0].namespace_descriptor, ns_desc(0)); - assert_eq!(nst.v[0].from_namespaces[1].namespace_descriptor, ns_desc_f32(1)); + assert_eq!( + nst.v[0].from_namespaces[1].namespace_descriptor, + ns_desc_f32(1) + ); } { @@ -354,7 +439,10 @@ C,featureC,f32 assert!(result.is_ok()); let result = nstp.resolve(&vw); assert!(result.is_err()); - assert_eq!(format!("{:?}", result), "Err(Custom { kind: Other, error: \"Unknown transformer function: unknown\" })"); + assert_eq!( + format!("{:?}", result), + "Err(Custom { kind: Other, error: \"Unknown transformer function: unknown\" })" + ); } { @@ -373,16 +461,15 @@ C,featureC,f32 assert_eq!(format!("{:?}", result), "Err(Custom { kind: Other, error: \"Using the same from namespace in multiple arguments to a function is not supported: \\\"featureA\\\"\" })"); } - { let mut nstp = NamespaceTransformsParser::new(); - nstp.add_transform_namespace(&vw, "new=unknown(nonexistent,featureB)()").unwrap(); // unknown function + nstp.add_transform_namespace(&vw, "new=unknown(nonexistent,featureB)()") + .unwrap(); // unknown function let result = nstp.resolve(&vw); assert!(result.is_err()); assert_eq!(format!("{:?}", result), "Err(Custom { kind: Other, error: \"Could not find namespace \\\"nonexistent\\\"\" })"); } - { // Now we test dependencies let mut nstp = NamespaceTransformsParser::new(); @@ -393,14 +480,23 @@ C,featureC,f32 let nst = nstp.resolve(&vw).unwrap(); assert_eq!(nst.v[0].to_namespace.namespace_descriptor, ns_desc_trans(0)); assert_eq!(nst.v[0].from_namespaces[0].namespace_descriptor, ns_desc(0)); - assert_eq!(nst.v[0].from_namespaces[1].namespace_descriptor, ns_desc_f32(1)); + assert_eq!( + nst.v[0].from_namespaces[1].namespace_descriptor, + ns_desc_f32(1) + ); assert_eq!(nst.v[1].to_namespace.namespace_descriptor, ns_desc_trans(1)); - assert_eq!(nst.v[1].from_namespaces[0].namespace_descriptor, ns_desc_trans(0)); - assert_eq!(nst.v[1].from_namespaces[1].namespace_descriptor, ns_desc_f32(1)); + assert_eq!( + nst.v[1].from_namespaces[0].namespace_descriptor, + ns_desc_trans(0) + ); + assert_eq!( + nst.v[1].from_namespaces[1].namespace_descriptor, + ns_desc_f32(1) + ); } { - // Now reverse order. We use new1 before it is declared. + // Now reverse order. We use new1 before it is declared. let mut nstp = NamespaceTransformsParser::new(); let result = nstp.add_transform_namespace(&vw, "new2=Combine(new1,featureB)()"); assert!(result.is_ok()); @@ -409,13 +505,21 @@ C,featureC,f32 let nst = nstp.resolve(&vw).unwrap(); assert_eq!(nst.v[0].to_namespace.namespace_descriptor, ns_desc_trans(0)); assert_eq!(nst.v[0].from_namespaces[0].namespace_descriptor, ns_desc(0)); - assert_eq!(nst.v[0].from_namespaces[1].namespace_descriptor, ns_desc_f32(1)); - + assert_eq!( + nst.v[0].from_namespaces[1].namespace_descriptor, + ns_desc_f32(1) + ); + assert_eq!(nst.v[1].to_namespace.namespace_descriptor, ns_desc_trans(1)); - assert_eq!(nst.v[1].from_namespaces[0].namespace_descriptor, ns_desc_trans(0)); - assert_eq!(nst.v[1].from_namespaces[1].namespace_descriptor, ns_desc_f32(1)); + assert_eq!( + nst.v[1].from_namespaces[0].namespace_descriptor, + ns_desc_trans(0) + ); + assert_eq!( + nst.v[1].from_namespaces[1].namespace_descriptor, + ns_desc_f32(1) + ); } - } #[test] fn test_namespace_transforms_cycle() { @@ -426,9 +530,8 @@ C,featureC,f32 "#; let vw = VwNamespaceMap::new(vw_map_string).unwrap(); - { - // Now create a cycle + // Now create a cycle let mut nstp = NamespaceTransformsParser::new(); let result = nstp.add_transform_namespace(&vw, "new2=Combine(new1,featureB)()"); assert!(result.is_ok()); @@ -437,27 +540,19 @@ C,featureC,f32 let nst = nstp.resolve(&vw); assert!(nst.is_err()); assert_eq!(format!("{:?}", nst), "Err(Custom { kind: Other, error: \"Cyclic dependency detected, one of the namespaces involved is \\\"new1\\\"\" })"); - - } { - // Now create a cycle + // Now create a cycle let mut nstp = NamespaceTransformsParser::new(); let result = nstp.add_transform_namespace(&vw, "new1=Combine(new1,featureB)()"); assert!(result.is_ok()); let nst = nstp.resolve(&vw); assert!(nst.is_err()); assert_eq!(format!("{:?}", nst), "Err(Custom { kind: Other, error: \"Cyclic dependency detected, one of the namespaces involved is \\\"new1\\\"\" })"); - - } - - - } - #[test] fn test_parser1() { let r = parse_identifier("a"); @@ -468,9 +563,9 @@ C,featureC,f32 assert_eq!(r.unwrap().1, "_a_b3_"); let r = parse_identifier("#"); assert_eq!(r.is_err(), true); - let r = parse_identifier("3a"); // they have to start with alphabetic character or underscore + let r = parse_identifier("3a"); // they have to start with alphabetic character or underscore assert_eq!(r.is_err(), true); - + let r = parse_function_params_namespaces("(a)"); assert_eq!(r.unwrap().1, vec!["a"]); let r = parse_function_params_namespaces("(a,b)"); @@ -482,7 +577,6 @@ C,featureC,f32 let r = parse_function_params_namespaces("()"); // empty list of namespaces is not allowed assert_eq!(r.is_err(), true); - let r = parse_float("0.2"); assert_eq!(r.unwrap().1, 0.2); let r = parse_float(" 0.2"); @@ -490,7 +584,6 @@ C,featureC,f32 let r = parse_float("(a)"); assert_eq!(r.is_err(), true); - let r = parse_function_params_floats("(0.1)"); assert_eq!(r.unwrap().1, vec![0.1]); let r = parse_function_params_floats("(0.1,0.2)"); @@ -498,7 +591,7 @@ C,featureC,f32 let r = parse_function_params_floats("( 0.1 , 0.2 )"); assert_eq!(r.unwrap().1, vec![0.1, 0.2]); let r = parse_function_params_floats("()"); // empty list of floats is allowed - let fv:Vec=Vec::new(); + let fv: Vec = Vec::new(); assert_eq!(r.unwrap().1, fv); let r = parse_namespace_statement("a=sqrt(B)(3,1,2.0)"); @@ -507,7 +600,7 @@ C,featureC,f32 assert_eq!(rw.1, "sqrt"); assert_eq!(rw.2, vec!["B"]); assert_eq!(rw.3, vec![3f32, 1f32, 2.0]); - + let r = parse_namespace_statement("abc=sqrt(BDE,CG)(3,1,2.0)"); let (o, rw) = r.unwrap(); assert_eq!(rw.0, "abc"); @@ -521,15 +614,5 @@ C,featureC,f32 assert_eq!(rw.1, "s_qrt"); assert_eq!(rw.2, vec!["_BD_E_", "C_G"]); assert_eq!(rw.3, vec![3f32, 1f32, 2.0]); - - - } } - - - - - - - diff --git a/src/graph.rs b/src/graph.rs index 91cc8f0f..3dc39acb 100644 --- a/src/graph.rs +++ b/src/graph.rs @@ -1,7 +1,6 @@ - -use crate::regressor::BlockTrait; use crate::model_instance; use crate::port_buffer; +use crate::regressor::BlockTrait; use std::error::Error; use std::mem; @@ -12,7 +11,7 @@ pub struct OutputSlot(usize); #[derive(Copy, Clone, Debug, PartialEq)] pub struct InputSlot(usize); #[derive(Copy, Clone, Debug, PartialEq)] -pub struct BlockPtr(usize); // just an id in a graph +pub struct BlockPtr(usize); // just an id in a graph #[derive(Debug, PartialEq)] pub struct BlockPtrOutput(BlockPtr, OutputSlot); // since blocks can have multiple outputs, separate between them #[derive(Copy, Clone, Debug, PartialEq)] @@ -20,8 +19,8 @@ pub struct BlockPtrInput(BlockPtr, InputSlot); // since blocks can have multiple #[derive(Debug)] pub struct BlockGraphNode { - pub edges_in: Vec, // each block can have multiple input edges - pub edges_out: Vec, // each block can have multiple output edges + pub edges_in: Vec, // each block can have multiple input edges + pub edges_out: Vec, // each block can have multiple output edges } pub struct BlockGraph { @@ -31,68 +30,90 @@ pub struct BlockGraph { tape_size: usize, } - // We need to treat join type in a special way - all inputs need to be consequtive #[derive(PartialEq, Debug)] pub enum BlockType { Regular, Join, Observe, - Copy + Copy, } impl BlockPtr { - pub fn get_node_id(&self) -> usize {self.0} + pub fn get_node_id(&self) -> usize { + self.0 + } } impl OutputSlot { - pub fn get_output_index(&self) -> usize {self.0} + pub fn get_output_index(&self) -> usize { + self.0 + } } impl InputSlot { - pub fn get_input_index(&self) -> usize {self.0} + pub fn get_input_index(&self) -> usize { + self.0 + } } impl BlockPtrOutput { - pub fn get_block_ptr(&self) -> BlockPtr {self.0} - pub fn get_node_id(&self) -> usize {self.0.get_node_id()} - pub fn get_output_index(&self) -> usize {self.1.get_output_index()} - pub fn get_output(&self) -> OutputSlot {self.1} + pub fn get_block_ptr(&self) -> BlockPtr { + self.0 + } + pub fn get_node_id(&self) -> usize { + self.0.get_node_id() + } + pub fn get_output_index(&self) -> usize { + self.1.get_output_index() + } + pub fn get_output(&self) -> OutputSlot { + self.1 + } } impl BlockPtrInput { - pub fn get_block_ptr(&self) -> BlockPtr {self.0} - pub fn get_node_id(&self) -> usize {self.0.get_node_id()} - pub fn get_input_index(&self) -> usize {self.1.get_input_index()} - pub fn get_input(&self) -> InputSlot {self.1} + pub fn get_block_ptr(&self) -> BlockPtr { + self.0 + } + pub fn get_node_id(&self) -> usize { + self.0.get_node_id() + } + pub fn get_input_index(&self) -> usize { + self.1.get_input_index() + } + pub fn get_input(&self) -> InputSlot { + self.1 + } } -const BLOCK_PTR_INPUT_DEFAULT:BlockPtrInput = BlockPtrInput(BlockPtr(usize::MAX), InputSlot(usize::MAX)); +const BLOCK_PTR_INPUT_DEFAULT: BlockPtrInput = + BlockPtrInput(BlockPtr(usize::MAX), InputSlot(usize::MAX)); impl BlockGraph { pub fn new() -> BlockGraph { - BlockGraph {nodes: Vec::new(), - blocks: Vec::new(), - blocks_final: Vec::new(), - tape_size: usize::MAX, - } + BlockGraph { + nodes: Vec::new(), + blocks: Vec::new(), + blocks_final: Vec::new(), + tape_size: usize::MAX, + } } - pub fn add_node(&mut self, - mut block: Box, - edges_in: Vec - ) -> Result, Box> { - - - // Due to how CopyBlock works (zero-copy first ouptut), it's first output cannot go to a Join block since join block needs to control its inputs + pub fn add_node( + &mut self, + mut block: Box, + edges_in: Vec, + ) -> Result, Box> { + // Due to how CopyBlock works (zero-copy first ouptut), it's first output cannot go to a Join block since join block needs to control its inputs // TODO we could just insert TrueCopy block here... - + let mut edges_in = edges_in; if block.get_block_type() == BlockType::Join { // Join a is a special block, because it does zero copy joining of outputs - let mut new_edges_in : Vec = Vec::new(); + let mut new_edges_in: Vec = Vec::new(); for edge_in in edges_in.into_iter() { - let node_id_in = edge_in.get_node_id(); + let node_id_in = edge_in.get_node_id(); if self.blocks[node_id_in].get_block_type() == BlockType::Join { // Join -> Join can always be merged into a single join // So we won't add a block here, instead, we will add input edges of the previous block @@ -108,54 +129,59 @@ impl BlockGraph { // if we connect copy to copy... we simply increase number of copies of existing input block assert!(edges_in.len() == 1); let edge_in = &edges_in[0]; - let node_id_in = edge_in.get_node_id(); + let node_id_in = edge_in.get_node_id(); if self.blocks[node_id_in].get_block_type() == BlockType::Copy { - let copy_block = self.blocks[node_id_in].as_any().downcast_mut::().unwrap(); + let copy_block = self.blocks[node_id_in] + .as_any() + .downcast_mut::() + .unwrap(); let bp = BlockPtr(node_id_in); let bo = BlockPtrOutput(bp, OutputSlot(copy_block.output_offsets.len())); copy_block.output_offsets.push(usize::MAX); - self.nodes[node_id_in].edges_out.push(BLOCK_PTR_INPUT_DEFAULT); // make empty spaceg + self.nodes[node_id_in] + .edges_out + .push(BLOCK_PTR_INPUT_DEFAULT); // make empty spaceg return Ok(vec![bo, edges_in.pop().unwrap()]); - } - + } } - - + let num_output_connectors = block.get_num_output_slots(); - let bp = BlockPtr(self.nodes.len()); // id of current node + let bp = BlockPtr(self.nodes.len()); // id of current node for (i, e) in edges_in.iter().enumerate() { let bi = InputSlot(i); let bpi = BlockPtrInput(bp, bi); self.nodes[e.get_node_id()].edges_out[e.get_output_index()] = bpi; } let mut newnode = BlockGraphNode { - edges_in: edges_in, - edges_out: Vec::new(), - }; - + edges_in: edges_in, + edges_out: Vec::new(), + }; self.nodes.push(newnode); self.blocks.push(block); - let mut vo:Vec = Vec::new(); + let mut vo: Vec = Vec::new(); for i in 0..num_output_connectors { let bo = BlockPtrOutput(bp, OutputSlot(i)); vo.push(bo); - self.nodes[bp.get_node_id()].edges_out.push(BLOCK_PTR_INPUT_DEFAULT); // make empty spaceg + self.nodes[bp.get_node_id()] + .edges_out + .push(BLOCK_PTR_INPUT_DEFAULT); // make empty spaceg } - return Ok(vo); + return Ok(vo); } - - + pub fn println(&self) { println!("Graph nodes:\n"); for n in self.nodes.iter() { - println!(" {:?}", n); + println!(" {:?}", n); } } - pub fn get_tape_size(&self) -> usize { - assert!(self.tape_size != usize::MAX, "get_tape_size() called on a graph before calling finalize()"); + assert!( + self.tape_size != usize::MAX, + "get_tape_size() called on a graph before calling finalize()" + ); self.tape_size } @@ -165,7 +191,7 @@ impl BlockGraph { pub fn get_num_input_slots(&self, bp: BlockPtr) -> usize { self.nodes[bp.get_node_id()].edges_in.len() } - + pub fn get_num_output_values(&self, outputs: Vec<&BlockPtrOutput>) -> usize { let mut t = 0; for x in outputs { @@ -174,25 +200,27 @@ impl BlockGraph { t } - fn len(&self) -> usize { self.nodes.len() } pub fn allocate_and_init_weights(&mut self, mi: &model_instance::ModelInstance) { - assert!(self.blocks_final.len() > 0, "There are no blocks in the final graph? Have you called finalize() yet?"); - for i in 0..self.blocks_final.len() { + assert!( + self.blocks_final.len() > 0, + "There are no blocks in the final graph? Have you called finalize() yet?" + ); + for i in 0..self.blocks_final.len() { self.blocks_final[i].allocate_and_init_weights(&mi); } } - + pub fn take_blocks(&mut self) -> Vec> { mem::take(&mut self.blocks_final) - /* + /* let mut blocks : Vec> = Vec::new(); for block in mem::take(&mut self.blocks).into_iter() { - // Join Block is a no-op, so it doesn't need to be executed. This is a super small optimization. + // Join Block is a no-op, so it doesn't need to be executed. This is a super small optimization. if block.get_block_type() != BlockType::Join { blocks.push(block); } @@ -202,7 +230,7 @@ impl BlockGraph { pub fn finalize(&mut self) { let mut offset: usize = 0; - + // Let's first install sinks, so the graph is without dangling parts let mut sinks: Vec = Vec::new(); for i in 0..self.len() { @@ -212,180 +240,242 @@ impl BlockGraph { sinks.push(bptro); } } - } + } // TODO we could have a single sink for all for bptro in sinks.into_iter() { // For neural nets, zeroing out the backward data is the least-surprise way of doing it block_misc::new_sink_block(self, bptro, block_misc::SinkType::Zero).unwrap(); } - // Now allocate inputs/outputs to parts of the tape for i in 0..self.len() { let current_block_type = self.blocks[i].get_block_type(); - + for (input_index, edge_in) in self.nodes[i].edges_in.iter().enumerate() { let bo = edge_in.get_output(); let bptr = edge_in.get_node_id(); let output_len = self.blocks[bptr].get_num_output_values(bo); let input_block_type = self.blocks[bptr].get_block_type(); - if (input_block_type == BlockType::Join) || - (input_block_type == BlockType::Observe) || - (input_block_type == BlockType::Copy) && (bo.get_output_index() == 0 && current_block_type != BlockType::Join && current_block_type != BlockType::Observe) { + if (input_block_type == BlockType::Join) + || (input_block_type == BlockType::Observe) + || (input_block_type == BlockType::Copy) + && (bo.get_output_index() == 0 + && current_block_type != BlockType::Join + && current_block_type != BlockType::Observe) + { // we are special casing Join block // It is zero-copy joining of inputs, which means inputs and outputs share exactly the same space let fake_offset = self.blocks[bptr].get_input_offset(InputSlot(0)).unwrap(); self.blocks[bptr].set_output_offset(bo, fake_offset); self.blocks[i].set_input_offset(InputSlot(input_index), fake_offset); - } else if (input_block_type == BlockType::Regular) || - (input_block_type == BlockType::Copy) { + } else if (input_block_type == BlockType::Regular) + || (input_block_type == BlockType::Copy) + { self.blocks[bptr].set_output_offset(bo, offset); self.blocks[i].set_input_offset(InputSlot(input_index), offset); - offset += output_len as usize; + offset += output_len as usize; } else { - panic!("Type of block not supported in scheduling: {:?}", input_block_type); + panic!( + "Type of block not supported in scheduling: {:?}", + input_block_type + ); } - } - } + } self.tape_size = offset; - + // Prepare the final list of blocks for block in mem::take(&mut self.blocks).into_iter() { - // Join Block is a no-op, so it doesn't need to be executed. This is a super small optimization. + // Join Block is a no-op, so it doesn't need to be executed. This is a super small optimization. if block.get_block_type() != BlockType::Join { self.blocks_final.push(block); } } - - } + } } mod tests { // Note this useful idiom: importing names from outer (for mod tests) scope. use super::*; + use crate::block_ffm; + use crate::block_loss_functions; + use crate::block_lr; use crate::block_misc; use crate::block_misc::Observe; use crate::model_instance; - use crate::block_loss_functions; use crate::model_instance::Optimizer; - use crate::block_lr; - use crate::block_ffm; #[test] fn graph_creation() { let mut bg = BlockGraph::new(); - + let const_block_output = block_misc::new_const_block(&mut bg, vec![1.0]).unwrap(); - assert_eq!(const_block_output, BlockPtrOutput(BlockPtr(0), OutputSlot(0))); - assert_eq!(bg.nodes[0].edges_in, vec![]); // basically [] - assert_eq!(bg.nodes[0].edges_out, vec![BLOCK_PTR_INPUT_DEFAULT]); + assert_eq!( + const_block_output, + BlockPtrOutput(BlockPtr(0), OutputSlot(0)) + ); + assert_eq!(bg.nodes[0].edges_in, vec![]); // basically [] + assert_eq!(bg.nodes[0].edges_out, vec![BLOCK_PTR_INPUT_DEFAULT]); } - + #[test] fn graph_one_sink() { let mut bg = BlockGraph::new(); - + let const_block_output = block_misc::new_const_block(&mut bg, vec![1.0]).unwrap(); - assert_eq!(const_block_output, BlockPtrOutput(BlockPtr(0), OutputSlot(0))); - assert_eq!(bg.nodes[0].edges_in.len(), 0); // basically [] - assert_eq!(bg.nodes[0].edges_out, vec![BLOCK_PTR_INPUT_DEFAULT]); - - // Let's add one result block - let output_node = block_misc::new_observe_block(&mut bg, const_block_output, Observe::Forward, Some(1.0)).unwrap(); -// assert_eq!(output_node, ()); -// println!("Output nodes: {:?}", output_nodes); - //println!("Block 0: edges_in: {:?}, edges_out: {:?}", bg.edges_in[0], bg.edges_out[0]); - assert_eq!(bg.nodes[0].edges_in, vec![]); // basically [] - assert_eq!(bg.nodes[0].edges_out, vec![BlockPtrInput(BlockPtr(1), InputSlot(0))]); + assert_eq!( + const_block_output, + BlockPtrOutput(BlockPtr(0), OutputSlot(0)) + ); + assert_eq!(bg.nodes[0].edges_in.len(), 0); // basically [] + assert_eq!(bg.nodes[0].edges_out, vec![BLOCK_PTR_INPUT_DEFAULT]); - assert_eq!(bg.nodes[1].edges_in, vec![BlockPtrOutput(BlockPtr(0), OutputSlot(0))]); - assert_eq!(bg.nodes[1].edges_out, vec![BLOCK_PTR_INPUT_DEFAULT]); + // Let's add one result block + let output_node = + block_misc::new_observe_block(&mut bg, const_block_output, Observe::Forward, Some(1.0)) + .unwrap(); + // assert_eq!(output_node, ()); + // println!("Output nodes: {:?}", output_nodes); + //println!("Block 0: edges_in: {:?}, edges_out: {:?}", bg.edges_in[0], bg.edges_out[0]); + assert_eq!(bg.nodes[0].edges_in, vec![]); // basically [] + assert_eq!( + bg.nodes[0].edges_out, + vec![BlockPtrInput(BlockPtr(1), InputSlot(0))] + ); + + assert_eq!( + bg.nodes[1].edges_in, + vec![BlockPtrOutput(BlockPtr(0), OutputSlot(0))] + ); + assert_eq!(bg.nodes[1].edges_out, vec![BLOCK_PTR_INPUT_DEFAULT]); } - + #[test] fn graph_two_sinks() { let mut bg = BlockGraph::new(); - + let const_block_output = block_misc::new_const_block(&mut bg, vec![1.0]).unwrap(); - assert_eq!(const_block_output, BlockPtrOutput(BlockPtr(0), OutputSlot(0))); + assert_eq!( + const_block_output, + BlockPtrOutput(BlockPtr(0), OutputSlot(0)) + ); assert_eq!(bg.nodes[0].edges_in, vec![]); assert_eq!(bg.nodes[0].edges_out, vec![BLOCK_PTR_INPUT_DEFAULT]); // We need to add a copy block to have two sinks let (c1, c2) = block_misc::new_copy_block_2(&mut bg, const_block_output).unwrap(); - - - // Let's add one result block - let output_node = block_misc::new_observe_block(&mut bg, c1, Observe::Forward, Some(1.0)).unwrap(); -// assert_eq!(output_node, ()); -// println!("Output nodes: {:?}", output_nodes); + + // Let's add one result block + let output_node = + block_misc::new_observe_block(&mut bg, c1, Observe::Forward, Some(1.0)).unwrap(); + // assert_eq!(output_node, ()); + // println!("Output nodes: {:?}", output_nodes); //println!("Block 0: edges_in: {:?}, edges_out: {:?}", bg.edges_in[0], bg.edges_out[0]); - assert_eq!(bg.nodes[0].edges_in.len(), 0); - assert_eq!(bg.nodes[0].edges_out, vec![BlockPtrInput(BlockPtr(1), InputSlot(0))]); + assert_eq!(bg.nodes[0].edges_in.len(), 0); + assert_eq!( + bg.nodes[0].edges_out, + vec![BlockPtrInput(BlockPtr(1), InputSlot(0))] + ); // Let's add second result block to see what happens - let output_node = block_misc::new_observe_block(&mut bg, c2, Observe::Forward, Some(1.0)).unwrap(); -// assert_eq!(output_node, ()); - assert_eq!(bg.nodes[0].edges_in, vec![]); - assert_eq!(bg.nodes[0].edges_out, vec![BlockPtrInput(BlockPtr(1), InputSlot(0))]); + let output_node = + block_misc::new_observe_block(&mut bg, c2, Observe::Forward, Some(1.0)).unwrap(); + // assert_eq!(output_node, ()); + assert_eq!(bg.nodes[0].edges_in, vec![]); + assert_eq!( + bg.nodes[0].edges_out, + vec![BlockPtrInput(BlockPtr(1), InputSlot(0))] + ); // copy block - assert_eq!(bg.nodes[1].edges_in, vec![BlockPtrOutput(BlockPtr(0), OutputSlot(0))]); - assert_eq!(bg.nodes[1].edges_out, vec![BlockPtrInput(BlockPtr(2), InputSlot(0)), BlockPtrInput(BlockPtr(3), InputSlot(0))]); + assert_eq!( + bg.nodes[1].edges_in, + vec![BlockPtrOutput(BlockPtr(0), OutputSlot(0))] + ); + assert_eq!( + bg.nodes[1].edges_out, + vec![ + BlockPtrInput(BlockPtr(2), InputSlot(0)), + BlockPtrInput(BlockPtr(3), InputSlot(0)) + ] + ); // result block 1 - assert_eq!(bg.nodes[2].edges_in, vec![BlockPtrOutput(BlockPtr(1), OutputSlot(0))]); + assert_eq!( + bg.nodes[2].edges_in, + vec![BlockPtrOutput(BlockPtr(1), OutputSlot(0))] + ); assert_eq!(bg.nodes[2].edges_out, vec![BLOCK_PTR_INPUT_DEFAULT]); // result bock 2 - assert_eq!(bg.nodes[3].edges_in, vec![BlockPtrOutput(BlockPtr(1), OutputSlot(1))]); + assert_eq!( + bg.nodes[3].edges_in, + vec![BlockPtrOutput(BlockPtr(1), OutputSlot(1))] + ); assert_eq!(bg.nodes[3].edges_out, vec![BLOCK_PTR_INPUT_DEFAULT]); } - - - + #[test] fn graph_two_sources() { let mut bg = BlockGraph::new(); - + let const_block_output1 = block_misc::new_const_block(&mut bg, vec![1.0]).unwrap(); - assert_eq!(const_block_output1, BlockPtrOutput(BlockPtr(0), OutputSlot(0))); + assert_eq!( + const_block_output1, + BlockPtrOutput(BlockPtr(0), OutputSlot(0)) + ); assert_eq!(bg.nodes[0].edges_in, vec![]); assert_eq!(bg.nodes[0].edges_out, vec![BLOCK_PTR_INPUT_DEFAULT]); let const_block_output2 = block_misc::new_const_block(&mut bg, vec![1.0, 2.0]).unwrap(); - assert_eq!(const_block_output2, BlockPtrOutput(BlockPtr(1), OutputSlot(0))); + assert_eq!( + const_block_output2, + BlockPtrOutput(BlockPtr(1), OutputSlot(0)) + ); assert_eq!(bg.nodes[1].edges_in, vec![]); assert_eq!(bg.nodes[1].edges_out, vec![BLOCK_PTR_INPUT_DEFAULT]); // Using the join block, we merge two outputs into one single output (copy-less implementation) - let mut union_output = block_misc::new_join_block(&mut bg, vec![const_block_output1, const_block_output2]).unwrap(); + let mut union_output = + block_misc::new_join_block(&mut bg, vec![const_block_output1, const_block_output2]) + .unwrap(); assert_eq!(union_output, BlockPtrOutput(BlockPtr(2), OutputSlot(0))); assert_eq!(bg.nodes[0].edges_in, vec![]); - assert_eq!(bg.nodes[0].edges_out, vec![BlockPtrInput(BlockPtr(2), InputSlot(0))]); + assert_eq!( + bg.nodes[0].edges_out, + vec![BlockPtrInput(BlockPtr(2), InputSlot(0))] + ); assert_eq!(bg.nodes[1].edges_in, vec![]); - assert_eq!(bg.nodes[1].edges_out, vec![BlockPtrInput(BlockPtr(2), InputSlot(1))]); - - // the join block - assert_eq!(bg.nodes[2].edges_in, vec![BlockPtrOutput(BlockPtr(0), OutputSlot(0)), BlockPtrOutput(BlockPtr(1), OutputSlot(0))]); + assert_eq!( + bg.nodes[1].edges_out, + vec![BlockPtrInput(BlockPtr(2), InputSlot(1))] + ); + + // the join block + assert_eq!( + bg.nodes[2].edges_in, + vec![ + BlockPtrOutput(BlockPtr(0), OutputSlot(0)), + BlockPtrOutput(BlockPtr(1), OutputSlot(0)) + ] + ); assert_eq!(bg.nodes[2].edges_out, vec![BLOCK_PTR_INPUT_DEFAULT]); - } - + } #[test] fn finalize_simple() { let mut bg = BlockGraph::new(); - + let const_block_output = block_misc::new_const_block(&mut bg, vec![1.0]).unwrap(); - let output_node = block_misc::new_observe_block(&mut bg, const_block_output, Observe::Forward, Some(1.0)).unwrap(); -// assert_eq!(output_node, ()); + let output_node = + block_misc::new_observe_block(&mut bg, const_block_output, Observe::Forward, Some(1.0)) + .unwrap(); + // assert_eq!(output_node, ()); let list = bg.finalize(); assert_eq!(bg.tape_size, 1); - } #[test] fn finalize_realistic() { - let mut mi = model_instance::ModelInstance::new_empty().unwrap(); + let mut mi = model_instance::ModelInstance::new_empty().unwrap(); mi.learning_rate = 0.1; mi.ffm_learning_rate = 0.1; mi.power_t = 0.0; @@ -394,10 +484,10 @@ mod tests { mi.ffm_k = 1; mi.ffm_bit_precision = 18; mi.ffm_fields = vec![vec![], vec![], vec![]]; // This isn't really used - + mi.optimizer = Optimizer::AdagradLUT; let mut bg = BlockGraph::new(); - + let re_lr = block_lr::new_lr_block(&mut bg, &mi).unwrap(); let re_ffm = block_ffm::new_ffm_block(&mut bg, &mi).unwrap(); let joined = block_misc::new_join_block(&mut bg, vec![re_lr, re_ffm]).unwrap(); @@ -407,78 +497,71 @@ mod tests { #[test] fn finalize_copy_to_join() { - let mut mi = model_instance::ModelInstance::new_empty().unwrap(); + let mut mi = model_instance::ModelInstance::new_empty().unwrap(); mi.ffm_k = 1; mi.ffm_fields = vec![vec![], vec![], vec![]]; // This isn't really used mi.optimizer = Optimizer::AdagradLUT; let mut bg = BlockGraph::new(); - - let const_1 = block_misc::new_const_block(&mut bg, vec![1.0]).unwrap(); // 0 - let const_2 = block_misc::new_const_block(&mut bg, vec![3.0]).unwrap(); // 1 - let const_3 = block_misc::new_const_block(&mut bg, vec![4.0]).unwrap(); // 2 - let const_4 = block_misc::new_const_block(&mut bg, vec![4.0]).unwrap(); // 3 - let (copy_output_1, copy_output_2) = block_misc::new_copy_block_2(&mut bg, const_1).unwrap(); // 4 - let (copy_output_3, copy_output_4) = block_misc::new_copy_block_2(&mut bg, const_2).unwrap(); // 5 + + let const_1 = block_misc::new_const_block(&mut bg, vec![1.0]).unwrap(); // 0 + let const_2 = block_misc::new_const_block(&mut bg, vec![3.0]).unwrap(); // 1 + let const_3 = block_misc::new_const_block(&mut bg, vec![4.0]).unwrap(); // 2 + let const_4 = block_misc::new_const_block(&mut bg, vec![4.0]).unwrap(); // 3 + let (copy_output_1, copy_output_2) = + block_misc::new_copy_block_2(&mut bg, const_1).unwrap(); // 4 + let (copy_output_3, copy_output_4) = + block_misc::new_copy_block_2(&mut bg, const_2).unwrap(); // 5 // this is not zero copy let join_1 = block_misc::new_join_block(&mut bg, vec![copy_output_1, const_3]).unwrap(); // 6 - // this is zero copy + // this is zero copy let join_2 = block_misc::new_join_block(&mut bg, vec![const_4, copy_output_2]); // 7 bg.finalize(); let mut list = bg.take_blocks(); { // first one goes to copy again, so it cannot use input as an output - let copy_block_1 = list[4].as_any().downcast_mut::().unwrap(); + let copy_block_1 = list[4] + .as_any() + .downcast_mut::() + .unwrap(); assert!(copy_block_1.input_offset != copy_block_1.output_offsets[0]); -// println!("C1: input {} outputs: {:?}", copy_block_1.input_offset, copy_block_1.output_offsets); + // println!("C1: input {} outputs: {:?}", copy_block_1.input_offset, copy_block_1.output_offsets); } { // But second one can re-use the input as output - let copy_block_2 = list[5].as_any().downcast_mut::().unwrap(); + let copy_block_2 = list[5] + .as_any() + .downcast_mut::() + .unwrap(); assert!(copy_block_2.input_offset == copy_block_2.output_offsets[0]); -// println!("C2: input {} outputs: {:?}", copy_block_2.input_offset, copy_block_2.output_offsets); + // println!("C2: input {} outputs: {:?}", copy_block_2.input_offset, copy_block_2.output_offsets); } - - } #[test] fn finalize_join_to_join() { - let mut mi = model_instance::ModelInstance::new_empty().unwrap(); + let mut mi = model_instance::ModelInstance::new_empty().unwrap(); mi.ffm_k = 1; mi.ffm_fields = vec![vec![], vec![], vec![]]; // This isn't really used mi.optimizer = Optimizer::AdagradLUT; let mut bg = BlockGraph::new(); - + let const_1 = block_misc::new_const_block(&mut bg, vec![1.0]).unwrap(); - let const_2 = block_misc::new_const_block(&mut bg, vec![1.0]).unwrap(); - let const_3 = block_misc::new_const_block(&mut bg, vec![1.0]).unwrap(); + let const_2 = block_misc::new_const_block(&mut bg, vec![1.0]).unwrap(); + let const_3 = block_misc::new_const_block(&mut bg, vec![1.0]).unwrap(); let mut join_block1 = block_misc::new_join_block(&mut bg, vec![const_1, const_2]).unwrap(); - let mut join_block2 = block_misc::new_join_block(&mut bg, vec![join_block1, const_3]).unwrap(); + let mut join_block2 = + block_misc::new_join_block(&mut bg, vec![join_block1, const_3]).unwrap(); assert_eq!(bg.nodes.len(), 5); assert_eq!(bg.nodes[3].edges_in.len(), 0); // 3 is the first join block which was removed assert_eq!(bg.nodes[3].edges_out.len(), 0); assert_eq!(bg.nodes[4].edges_in.len(), 3); // now fourth block has 3 inputs, not 2 - + bg.finalize(); let list = bg.take_blocks(); - assert_eq!(list.len(), 4); // both join blocks are no-op and thus not returned, but sink block is added automatically - + assert_eq!(list.len(), 4); // both join blocks are no-op and thus not returned, but sink block is added automatically } - - - - - - - - - - - - - } diff --git a/src/lib.rs b/src/lib.rs index 95686ff0..47542585 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -2,6 +2,10 @@ mod block_ffm; mod block_helpers; mod block_loss_functions; mod block_lr; +mod block_misc; +mod block_neural; +mod block_normalize; +mod block_relu; mod cache; mod cmdline; mod consts; @@ -9,33 +13,29 @@ mod feature_buffer; mod feature_transform_executor; mod feature_transform_implementations; mod feature_transform_parser; +mod graph; mod model_instance; mod multithread_helpers; mod optimizer; mod parser; mod persistence; +mod port_buffer; mod regressor; mod serving; mod version; mod vwmap; -mod port_buffer; -mod block_neural; -mod block_misc; -mod block_relu; -mod block_normalize; -mod graph; extern crate blas; extern crate intel_mkl_src; -use std::ffi::CStr; -use std::io::Cursor; -use std::os::raw::c_char; -use shellwords; use crate::feature_buffer::FeatureBufferTranslator; use crate::multithread_helpers::BoxedRegressorTrait; use crate::parser::VowpalParser; use crate::port_buffer::PortBuffer; +use shellwords; +use std::ffi::CStr; +use std::io::Cursor; +use std::os::raw::c_char; #[repr(C)] pub struct FfiPredictor { @@ -46,25 +46,24 @@ pub struct Predictor { feature_buffer_translator: FeatureBufferTranslator, vw_parser: VowpalParser, regressor: BoxedRegressorTrait, - pb: PortBuffer + pb: PortBuffer, } impl Predictor { - unsafe fn predict(&mut self, input_buffer: &str) -> f32 { let mut buffered_input = Cursor::new(input_buffer); let reading_result = self.vw_parser.next_vowpal(&mut buffered_input); let buffer = match reading_result { Ok([]) => return -1.0, // EOF Ok(buffer2) => buffer2, - Err(_e) => return -1.0 + Err(_e) => return -1.0, }; self.feature_buffer_translator.translate(buffer, 0); - self.regressor.predict(&self.feature_buffer_translator.feature_buffer, &mut self.pb) + self.regressor + .predict(&self.feature_buffer_translator.feature_buffer, &mut self.pb) } } - #[no_mangle] pub extern "C" fn new_fw_predictor_prototype(command: *const c_char) -> *mut FfiPredictor { // create a "prototype" predictor that loads the weights file. This predictor is expensive, and is intended @@ -74,10 +73,12 @@ pub extern "C" fn new_fw_predictor_prototype(command: *const c_char) -> *mut Ffi let words = shellwords::split(str_command).unwrap(); let cmd_matches = cmdline::create_expected_args().get_matches_from(words); let weights_filename = match cmd_matches.value_of("initial_regressor") { - Some(filename) => filename, - None => panic!("Cannot resolve input weights file name") + Some(filename) => filename, + None => panic!("Cannot resolve input weights file name"), }; - let (model_instance, vw_namespace_map, regressor) = persistence::new_regressor_from_filename(weights_filename, true, Some(&cmd_matches)).unwrap(); + let (model_instance, vw_namespace_map, regressor) = + persistence::new_regressor_from_filename(weights_filename, true, Some(&cmd_matches)) + .unwrap(); let feature_buffer_translator = FeatureBufferTranslator::new(&model_instance); let vw_parser = VowpalParser::new(&vw_namespace_map); let sharable_regressor = BoxedRegressorTrait::new(Box::new(regressor)); @@ -86,7 +87,7 @@ pub extern "C" fn new_fw_predictor_prototype(command: *const c_char) -> *mut Ffi feature_buffer_translator, vw_parser, regressor: sharable_regressor, - pb + pb, }; Box::into_raw(Box::new(predictor)).cast() } @@ -101,7 +102,7 @@ pub unsafe extern "C" fn clone_lite(prototype: *mut FfiPredictor) -> *mut FfiPre feature_buffer_translator: prototype.feature_buffer_translator.clone(), vw_parser: prototype.vw_parser.clone(), regressor: prototype.regressor.clone(), - pb : prototype.pb.clone() + pb: prototype.pb.clone(), }; Box::into_raw(Box::new(lite_predictor)).cast() } @@ -118,8 +119,7 @@ pub unsafe extern "C" fn free_predictor(ptr: *mut FfiPredictor) { drop::>(Box::from_raw(from_ptr(ptr))); } -unsafe fn from_ptr<'a>(ptr: *mut FfiPredictor) -> &'a mut Predictor -{ +unsafe fn from_ptr<'a>(ptr: *mut FfiPredictor) -> &'a mut Predictor { if ptr.is_null() { eprintln!("Fatal error, got NULL `Context` pointer"); std::process::abort(); diff --git a/src/main.rs b/src/main.rs index 140b750c..2c7ceeab 100644 --- a/src/main.rs +++ b/src/main.rs @@ -26,10 +26,10 @@ mod block_ffm; mod block_helpers; mod block_loss_functions; mod block_lr; -mod block_neural; -mod block_relu; mod block_misc; +mod block_neural; mod block_normalize; +mod block_relu; mod cache; mod cmdline; mod consts; @@ -37,17 +37,17 @@ mod feature_buffer; mod feature_transform_executor; mod feature_transform_implementations; mod feature_transform_parser; +mod graph; mod model_instance; mod multithread_helpers; mod optimizer; mod parser; mod persistence; +mod port_buffer; mod regressor; mod serving; mod version; mod vwmap; -mod port_buffer; -mod graph; fn main() { match main2() { @@ -117,7 +117,7 @@ fn main2() -> Result<(), Box> { // We'll parse once the command line into cl and then different objects will examine it let cl = cmdline::parse(); if cl.is_present("build_cache_without_training") { - return build_cache_without_training(cl) + return build_cache_without_training(cl); } // Where will we be putting perdictions (if at all) let mut predictions_file = match cl.value_of("predictions") { @@ -126,7 +126,7 @@ fn main2() -> Result<(), Box> { }; let testonly = cl.is_present("testonly"); - + let final_regressor_filename = cl.value_of("final_regressor"); match final_regressor_filename { Some(filename) => { @@ -154,15 +154,17 @@ fn main2() -> Result<(), Box> { .value_of("initial_regressor") .expect("Daemon mode only supports serving from --initial regressor"); println!("initial_regressor = {}", filename); - let (mi2, vw2, re_fixed) = persistence::new_regressor_from_filename(filename, true, Option::Some(&cl))?; - + let (mi2, vw2, re_fixed) = + persistence::new_regressor_from_filename(filename, true, Option::Some(&cl))?; + let mut se = serving::Serving::new(&cl, &vw2, Box::new(re_fixed), &mi2)?; se.serve()?; } else if cl.is_present("convert_inference_regressor") { let filename = cl .value_of("initial_regressor") .expect("Convert mode requires --initial regressor"); - let (mut mi2, vw2, re_fixed) = persistence::new_regressor_from_filename(filename, true, Option::Some(&cl))?; + let (mut mi2, vw2, re_fixed) = + persistence::new_regressor_from_filename(filename, true, Option::Some(&cl))?; mi2.optimizer = model_instance::Optimizer::SGD; match inference_regressor_filename { Some(filename1) => { @@ -171,21 +173,18 @@ fn main2() -> Result<(), Box> { None => {} } } else { - let vw: vwmap::VwNamespaceMap; let mut re: regressor::Regressor; let mi: model_instance::ModelInstance; if let Some(filename) = cl.value_of("initial_regressor") { - println!("initial_regressor = {}", filename); - (mi, vw, re) = persistence::new_regressor_from_filename(filename, testonly, Option::Some(&cl))?; - + (mi, vw, re) = + persistence::new_regressor_from_filename(filename, testonly, Option::Some(&cl))?; } else { - // We load vw_namespace_map.csv just so we know all the namespaces ahead of time // This is one of the major differences from vowpal - + let input_filename = cl.value_of("data").expect("--data expected"); let vw_namespace_map_filepath = Path::new(input_filename) .parent() diff --git a/src/model_instance.rs b/src/model_instance.rs index a7fdf431..057b4ceb 100644 --- a/src/model_instance.rs +++ b/src/model_instance.rs @@ -2,21 +2,20 @@ use std::error::Error; use std::io::Error as IOError; use std::io::ErrorKind; +use serde::{Deserialize, Serialize}; use std::collections::HashMap; -use serde::{Serialize,Deserialize}; -use crate::vwmap; use crate::consts; use crate::feature_transform_parser; -use crate::vwmap::{NamespaceDescriptor}; +use crate::vwmap; +use crate::vwmap::NamespaceDescriptor; #[derive(Serialize, Deserialize, Debug, Clone, PartialEq)] pub struct FeatureComboDesc { pub namespace_descriptors: Vec, - pub weight:f32, + pub weight: f32, } - #[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Copy)] pub enum Optimizer { SGD = 100, @@ -34,15 +33,16 @@ pub struct NNConfig { impl NNConfig { pub fn new() -> NNConfig { - NNConfig{ - layers:Vec::new(), - topology: "one".to_string()} + NNConfig { + layers: Vec::new(), + topology: "one".to_string(), + } } } #[derive(Serialize, Deserialize, Debug, Clone)] pub struct ModelInstance { - pub learning_rate: f32, + pub learning_rate: f32, #[serde(default = "default_f32_zero")] pub minimum_learning_rate: f32, pub power_t: f32, @@ -56,7 +56,7 @@ pub struct ModelInstance { pub ffm_bit_precision: u32, #[serde(default = "default_bool_false")] pub fastmath: bool, - + pub ffm_initialization_type: String, #[serde(default = "default_f32_zero")] pub ffm_k_threshold: f32, @@ -65,20 +65,20 @@ pub struct ModelInstance { #[serde(default = "default_f32_zero")] pub ffm_init_width: f32, #[serde(default = "default_f32_zero")] - pub ffm_init_zero_band: f32, // from 0.0 to 1.0, percentage of ffm_init_width + pub ffm_init_zero_band: f32, // from 0.0 to 1.0, percentage of ffm_init_width #[serde(default = "default_f32_zero")] pub ffm_init_acc_gradient: f32, #[serde(default = "default_f32_zero")] pub init_acc_gradient: f32, #[serde(default = "default_f32_zero")] - pub ffm_learning_rate: f32, + pub ffm_learning_rate: f32, #[serde(default = "default_f32_zero")] pub ffm_power_t: f32, #[serde(default = "default_f32_zero")] pub nn_init_acc_gradient: f32, #[serde(default = "default_f32_zero")] - pub nn_learning_rate: f32, + pub nn_learning_rate: f32, #[serde(default = "default_f32_zero")] pub nn_power_t: f32, @@ -86,31 +86,37 @@ pub struct ModelInstance { #[serde(default = "default_optimizer_adagrad")] pub optimizer: Optimizer, - + pub transform_namespaces: feature_transform_parser::NamespaceTransforms, - } -fn default_u32_zero() -> u32{0} -fn default_f32_zero() -> f32{0.0} -fn default_bool_false() -> bool{false} -fn default_optimizer_adagrad() -> Optimizer{Optimizer::AdagradFlex} - +fn default_u32_zero() -> u32 { + 0 +} +fn default_f32_zero() -> f32 { + 0.0 +} +fn default_bool_false() -> bool { + false +} +fn default_optimizer_adagrad() -> Optimizer { + Optimizer::AdagradFlex +} -fn parse_float(s: &str, default: f32, cl: &clap::ArgMatches) -> f32{ +fn parse_float(s: &str, default: f32, cl: &clap::ArgMatches) -> f32 { match cl.value_of(s) { Some(val) => val.parse().unwrap(), - None => default + None => default, } } impl ModelInstance { pub fn new_empty() -> Result> { let mi = ModelInstance { - learning_rate: 0.5, // vw default + learning_rate: 0.5, // vw default ffm_learning_rate: 0.5, // vw default - minimum_learning_rate: 0.0, - bit_precision: 18, // vw default + minimum_learning_rate: 0.0, + bit_precision: 18, // vw default power_t: 0.5, ffm_power_t: 0.5, add_constant_feature: true, @@ -119,9 +125,9 @@ impl ModelInstance { ffm_k: 0, ffm_bit_precision: 18, fastmath: true, - ffm_initialization_type: String::from("default"), + ffm_initialization_type: String::from("default"), ffm_k_threshold: 0.0, - ffm_init_center: 0.0, + ffm_init_center: 0.0, ffm_init_width: 0.0, ffm_init_zero_band: 0.0, ffm_init_acc_gradient: 0.0, @@ -136,13 +142,21 @@ impl ModelInstance { Ok(mi) } - - pub fn create_feature_combo_desc(&self, vw: &vwmap::VwNamespaceMap, s: &str) -> Result> { - + pub fn create_feature_combo_desc( + &self, + vw: &vwmap::VwNamespaceMap, + s: &str, + ) -> Result> { let vsplit: Vec<&str> = s.split(":").collect(); // We use : as a delimiter for weight let mut combo_weight: f32 = 1.0; if vsplit.len() > 2 { - return Err(Box::new(IOError::new(ErrorKind::Other, format!("only one value parameter allowed (denoted with \":\"): \"{:?}\"", s)))) + return Err(Box::new(IOError::new( + ErrorKind::Other, + format!( + "only one value parameter allowed (denoted with \":\"): \"{:?}\"", + s + ), + ))); } if vsplit.len() == 2 { let weight_str = vsplit[1]; @@ -152,157 +166,233 @@ impl ModelInstance { let namespaces_str = vsplit[0]; let mut namespace_descriptors: Vec = Vec::new(); for char in namespaces_str.chars() { - let namespace_descriptor = feature_transform_parser::get_namespace_descriptor(&self.transform_namespaces, vw, char)?; - namespace_descriptors.push(namespace_descriptor); + let namespace_descriptor = feature_transform_parser::get_namespace_descriptor( + &self.transform_namespaces, + vw, + char, + )?; + namespace_descriptors.push(namespace_descriptor); } Ok(FeatureComboDesc { - namespace_descriptors: namespace_descriptors, - weight: combo_weight - }) + namespace_descriptors: namespace_descriptors, + weight: combo_weight, + }) } - fn create_feature_combo_desc_from_verbose(&self, vw: &vwmap::VwNamespaceMap, s: &str) -> Result> { + fn create_feature_combo_desc_from_verbose( + &self, + vw: &vwmap::VwNamespaceMap, + s: &str, + ) -> Result> { let vsplit: Vec<&str> = s.split(":").collect(); // We use : as a delimiter for weight let mut combo_weight: f32 = 1.0; - + if vsplit.len() == 2 { let weight_str = vsplit[1]; combo_weight = match weight_str.parse() { - Ok(x) => x, - Err(y) => return Err(Box::new(IOError::new(ErrorKind::Other, format!("Could not parse the value of a feature combination: {}", weight_str)))) + Ok(x) => x, + Err(y) => { + return Err(Box::new(IOError::new( + ErrorKind::Other, + format!( + "Could not parse the value of a feature combination: {}", + weight_str + ), + ))) + } } } else if vsplit.len() > 2 { - return Err(Box::new(IOError::new(ErrorKind::Other, format!("Verbose features cannot have \":\" as part of their names: \"{:?}\"", s)))) + return Err(Box::new(IOError::new( + ErrorKind::Other, + format!( + "Verbose features cannot have \":\" as part of their names: \"{:?}\"", + s + ), + ))); } - - let namespaces_verbose: Vec<&str> = vsplit[0].split(",").collect(); // verbose names are separated by comma + + let namespaces_verbose: Vec<&str> = vsplit[0].split(",").collect(); // verbose names are separated by comma let mut namespace_descriptors: Vec = Vec::new(); for namespace_verbose in namespaces_verbose { - let namespace_descriptor = feature_transform_parser::get_namespace_descriptor_verbose(&self.transform_namespaces, vw, namespace_verbose)?; - namespace_descriptors.push(namespace_descriptor); + let namespace_descriptor = feature_transform_parser::get_namespace_descriptor_verbose( + &self.transform_namespaces, + vw, + namespace_verbose, + )?; + namespace_descriptors.push(namespace_descriptor); } Ok(FeatureComboDesc { - namespace_descriptors: namespace_descriptors, - weight: combo_weight - }) + namespace_descriptors: namespace_descriptors, + weight: combo_weight, + }) } - fn create_field_desc_from_verbose(&self, vw: &vwmap::VwNamespaceMap, s: &str) -> Result> { + fn create_field_desc_from_verbose( + &self, + vw: &vwmap::VwNamespaceMap, + s: &str, + ) -> Result> { let vsplit: Vec<&str> = s.split(":").collect(); // We use : as a delimiter for weight if vsplit.len() > 1 { - return Err(Box::new(IOError::new(ErrorKind::Other, format!("Fields currently do not support passing a value via : {:?}", s)))) + return Err(Box::new(IOError::new( + ErrorKind::Other, + format!( + "Fields currently do not support passing a value via : {:?}", + s + ), + ))); } - let namespaces_verbose: Vec<&str> = s.split(",").collect(); // verbose names are separated by comma + let namespaces_verbose: Vec<&str> = s.split(",").collect(); // verbose names are separated by comma let mut field: FieldDesc = Vec::new(); for namespace_verbose in namespaces_verbose { - let namespace_descriptor = feature_transform_parser::get_namespace_descriptor_verbose(&self.transform_namespaces, vw, namespace_verbose)?; + let namespace_descriptor = feature_transform_parser::get_namespace_descriptor_verbose( + &self.transform_namespaces, + vw, + namespace_verbose, + )?; field.push(namespace_descriptor); } Ok(field) } - fn parse_nn(&mut self, s: &str) -> Result<(), Box> { // Examples: 0:activation:relu // Examples: 4:maxnorm:5.0 // Examples: 6:width:20 - let vsplit : Vec<&str> = s.split(":").collect(); + let vsplit: Vec<&str> = s.split(":").collect(); if vsplit.len() != 3 { - return Err(Box::new(IOError::new(ErrorKind::Other, format!("--nn parameters have to be of form layer:parameter_name:parameter_value: {}", s)))); + return Err(Box::new(IOError::new( + ErrorKind::Other, + format!( + "--nn parameters have to be of form layer:parameter_name:parameter_value: {}", + s + ), + ))); } - let layer_number: usize = vsplit[0].parse().expect(&format!("--nn can not parse the layer number: {}", vsplit[0])); + let layer_number: usize = vsplit[0].parse().expect(&format!( + "--nn can not parse the layer number: {}", + vsplit[0] + )); if layer_number >= self.nn_config.layers.len() { - return Err(Box::new(IOError::new(ErrorKind::Other, format!("--nn parameter addressing layer {}, but we have only {} layers", layer_number, self.nn_config.layers.len())))); - } + return Err(Box::new(IOError::new( + ErrorKind::Other, + format!( + "--nn parameter addressing layer {}, but we have only {} layers", + layer_number, + self.nn_config.layers.len() + ), + ))); + } self.nn_config.layers[layer_number].insert(vsplit[1].to_string(), vsplit[2].to_string()); Ok(()) } - - pub fn new_from_cmdline<'a>(cl: &clap::ArgMatches<'a>, vw: &vwmap::VwNamespaceMap) -> Result> { + pub fn new_from_cmdline<'a>( + cl: &clap::ArgMatches<'a>, + vw: &vwmap::VwNamespaceMap, + ) -> Result> { let mut mi = ModelInstance::new_empty()?; let vwcompat: bool = cl.is_present("vwcompat"); - + if vwcompat { mi.fastmath = false; mi.init_acc_gradient = 0.0; if !cl.is_present("keep") { - return Err(Box::new(IOError::new(ErrorKind::Other, "--vwcompat requires at least one --keep parameter, we do not implicitly take all features available"))) + return Err(Box::new(IOError::new(ErrorKind::Other, "--vwcompat requires at least one --keep parameter, we do not implicitly take all features available"))); } // Vowpal supports a mode with "prehashed" features, where numeric strings are treated as // numeric precomputed hashes. This is even default option. - // It is generally a bad idea except if you strings really are precomputed hashes... + // It is generally a bad idea except if you strings really are precomputed hashes... if !cl.is_present("hash") { - return Err(Box::new(IOError::new(ErrorKind::Other, format!("--vwcompat requires use of --hash all")))) - } else - - if let Some(val) = cl.value_of("hash") { + return Err(Box::new(IOError::new( + ErrorKind::Other, + format!("--vwcompat requires use of --hash all"), + ))); + } else if let Some(val) = cl.value_of("hash") { if val != "all" { - return Err(Box::new(IOError::new(ErrorKind::Other, format!("--vwcompat requires use of --hash all")))) - } + return Err(Box::new(IOError::new( + ErrorKind::Other, + format!("--vwcompat requires use of --hash all"), + ))); + } } // --sgd will turn off adaptive, invariant and normalization in vowpal. You can turn adaptive back on in vw and fw with --adaptive if !cl.is_present("sgd") { - return Err(Box::new(IOError::new(ErrorKind::Other, format!("--vwcompat requires use of --sgd")))) + return Err(Box::new(IOError::new( + ErrorKind::Other, + format!("--vwcompat requires use of --sgd"), + ))); } - - } // we first need transform namespaces, before processing keep or interactions - + if let Some(in_v) = cl.values_of("transform") { let mut namespace_parser = feature_transform_parser::NamespaceTransformsParser::new(); - for value_str in in_v { + for value_str in in_v { namespace_parser.add_transform_namespace(vw, value_str)?; } mi.transform_namespaces = namespace_parser.resolve(vw)?; } - + if let Some(in_v) = cl.values_of("keep") { for value_str in in_v { - mi.feature_combo_descs.push(mi.create_feature_combo_desc(vw, value_str)?); + mi.feature_combo_descs + .push(mi.create_feature_combo_desc(vw, value_str)?); } } - + if let Some(in_v) = cl.values_of("interactions") { - for value_str in in_v { - mi.feature_combo_descs.push(mi.create_feature_combo_desc(vw, value_str)?); + for value_str in in_v { + mi.feature_combo_descs + .push(mi.create_feature_combo_desc(vw, value_str)?); } } if let Some(in_v) = cl.values_of("linear") { - for value_str in in_v { - mi.feature_combo_descs.push(mi.create_feature_combo_desc_from_verbose(vw, value_str)?); + for value_str in in_v { + mi.feature_combo_descs + .push(mi.create_feature_combo_desc_from_verbose(vw, value_str)?); } } if let Some(val) = cl.value_of("ffm_k") { mi.ffm_k = val.parse()?; - if mi.ffm_k > consts::FFM_MAX_K as u32{ - return Err(Box::new(IOError::new(ErrorKind::Other, format!("Maximum ffm_k is: {}, passed: {}", consts::FFM_MAX_K, mi.ffm_k)))) + if mi.ffm_k > consts::FFM_MAX_K as u32 { + return Err(Box::new(IOError::new( + ErrorKind::Other, + format!( + "Maximum ffm_k is: {}, passed: {}", + consts::FFM_MAX_K, + mi.ffm_k + ), + ))); } } - if let Some(val) = cl.value_of("ffm_initialization_type") { + if let Some(val) = cl.value_of("ffm_initialization_type") { mi.ffm_initialization_type = val.parse()?; } - mi.ffm_init_center = parse_float("ffm_init_center", mi.ffm_init_center, &cl); - mi.ffm_init_width = parse_float("ffm_init_width", mi.ffm_init_width, &cl); - mi.ffm_init_zero_band = parse_float("ffm_init_zero_band", mi.ffm_init_zero_band, &cl); + mi.ffm_init_center = parse_float("ffm_init_center", mi.ffm_init_center, &cl); + mi.ffm_init_width = parse_float("ffm_init_width", mi.ffm_init_width, &cl); + mi.ffm_init_zero_band = parse_float("ffm_init_zero_band", mi.ffm_init_zero_band, &cl); if let Some(in_v) = cl.values_of("ffm_field") { - for namespaces_str in in_v { - let mut field: Vec= Vec::new(); + for namespaces_str in in_v { + let mut field: Vec = Vec::new(); for char in namespaces_str.chars() { //println!("K: {}", char); - let namespace_descriptor = feature_transform_parser::get_namespace_descriptor(&mi.transform_namespaces, vw, char)?; + let namespace_descriptor = feature_transform_parser::get_namespace_descriptor( + &mi.transform_namespaces, + vw, + char, + )?; field.push(namespace_descriptor); } mi.ffm_fields.push(field); @@ -311,10 +401,11 @@ impl ModelInstance { if let Some(in_v) = cl.values_of("ffm_field_verbose") { for value_str in in_v { - mi.ffm_fields.push(mi.create_field_desc_from_verbose(vw, value_str)?); + mi.ffm_fields + .push(mi.create_field_desc_from_verbose(vw, value_str)?); } } - + if let Some(val) = cl.value_of("ffm_bit_precision") { mi.ffm_bit_precision = val.parse()?; } @@ -323,56 +414,63 @@ impl ModelInstance { mi.bit_precision = val.parse()?; } - mi.learning_rate = parse_float("learning_rate", mi.learning_rate, &cl); - mi.init_acc_gradient = parse_float("init_acc_gradient", mi.init_acc_gradient, &cl); - mi.power_t = parse_float("power_t", mi.power_t, &cl); - - mi.ffm_learning_rate = parse_float("ffm_learning_rate", mi.learning_rate, &cl); - mi.ffm_init_acc_gradient = parse_float("ffm_init_acc_gradient", mi.init_acc_gradient, &cl); - mi.ffm_power_t = parse_float("ffm_power_t", mi.power_t, &cl); + mi.learning_rate = parse_float("learning_rate", mi.learning_rate, &cl); + mi.init_acc_gradient = parse_float("init_acc_gradient", mi.init_acc_gradient, &cl); + mi.power_t = parse_float("power_t", mi.power_t, &cl); - mi.nn_learning_rate = parse_float("nn_learning_rate", mi.ffm_learning_rate, &cl); - mi.nn_init_acc_gradient = parse_float("nn_init_acc_gradient", mi.ffm_init_acc_gradient, &cl); - mi.nn_power_t = parse_float("nn_power_t", mi.ffm_power_t, &cl); + mi.ffm_learning_rate = parse_float("ffm_learning_rate", mi.learning_rate, &cl); + mi.ffm_init_acc_gradient = parse_float("ffm_init_acc_gradient", mi.init_acc_gradient, &cl); + mi.ffm_power_t = parse_float("ffm_power_t", mi.power_t, &cl); + mi.nn_learning_rate = parse_float("nn_learning_rate", mi.ffm_learning_rate, &cl); + mi.nn_init_acc_gradient = + parse_float("nn_init_acc_gradient", mi.ffm_init_acc_gradient, &cl); + mi.nn_power_t = parse_float("nn_power_t", mi.ffm_power_t, &cl); if let Some(val) = cl.value_of("nn_layers") { let nn_layers = val.parse()?; for i in 0..nn_layers { mi.nn_config.layers.push(HashMap::new()); - } + } } if let Some(val) = cl.value_of("nn_topology") { - mi.nn_config.topology = val.to_string(); + mi.nn_config.topology = val.to_string(); } if let Some(in_v) = cl.values_of("nn") { - for value_str in in_v { + for value_str in in_v { mi.parse_nn(value_str)?; } } - - if let Some(val) = cl.value_of("minimum_learning_rate") { mi.minimum_learning_rate = val.parse()?; } if let Some(val) = cl.value_of("link") { if val != "logistic" { - return Err(Box::new(IOError::new(ErrorKind::Other, format!("--link only supports 'logistic'")))) - } + return Err(Box::new(IOError::new( + ErrorKind::Other, + format!("--link only supports 'logistic'"), + ))); + } } if let Some(val) = cl.value_of("loss_function") { if val != "logistic" { - return Err(Box::new(IOError::new(ErrorKind::Other, format!("--loss_function only supports 'logistic'")))) - } + return Err(Box::new(IOError::new( + ErrorKind::Other, + format!("--loss_function only supports 'logistic'"), + ))); + } } if let Some(val) = cl.value_of("l2") { - let v2:f32 = val.parse()?; + let v2: f32 = val.parse()?; if v2.abs() > 0.00000001 { - return Err(Box::new(IOError::new(ErrorKind::Other, format!("--l2 can only be 0.0")))) + return Err(Box::new(IOError::new( + ErrorKind::Other, + format!("--l2 can only be 0.0"), + ))); } } @@ -395,65 +493,63 @@ impl ModelInstance { } } - Ok(mi) } + pub fn update_hyperparameters_from_cmd<'a>( + cmd_arguments: &clap::ArgMatches<'a>, + mi: &mut ModelInstance, + ) -> Result<(), Box> { + /*! A method that enables updating hyperparameters of an existing (pre-loaded) model. + Currently limited to the most commonly used hyperparameters: ffm_learning_rate, ffm_power_t, power_t, learning_rate. */ + + let mut replacement_hyperparam_ids: Vec<(String, String)> = vec![]; + + // Handle learning rates + if cmd_arguments.is_present("learning_rate") { + if let Some(val) = cmd_arguments.value_of("learning_rate") { + let hvalue = val.parse::()?; + mi.learning_rate = hvalue; + replacement_hyperparam_ids.push(("learning_rate".to_string(), hvalue.to_string())); + } + } + + if cmd_arguments.is_present("ffm_learning_rate") { + if let Some(val) = cmd_arguments.value_of("ffm_learning_rate") { + let hvalue = val.parse::()?; + mi.ffm_learning_rate = hvalue; + replacement_hyperparam_ids + .push(("ffm_learning_rate".to_string(), hvalue.to_string())); + } + } - pub fn update_hyperparameters_from_cmd<'a>(cmd_arguments: &clap::ArgMatches<'a>, mi: &mut ModelInstance) -> Result<(), Box> { - /*! A method that enables updating hyperparameters of an existing (pre-loaded) model. - Currently limited to the most commonly used hyperparameters: ffm_learning_rate, ffm_power_t, power_t, learning_rate. */ - - let mut replacement_hyperparam_ids: Vec<(String, String)> = vec![]; - - // Handle learning rates - if cmd_arguments.is_present("learning_rate") { - - if let Some(val) = cmd_arguments.value_of("learning_rate") { - let hvalue = val.parse::()?; - mi.learning_rate = hvalue; - replacement_hyperparam_ids.push(("learning_rate".to_string(), hvalue.to_string())); - } - } - - if cmd_arguments.is_present("ffm_learning_rate") { - - if let Some(val) = cmd_arguments.value_of("ffm_learning_rate") { - let hvalue = val.parse::()?; - mi.ffm_learning_rate = hvalue; - replacement_hyperparam_ids.push(("ffm_learning_rate".to_string(), hvalue.to_string())); - } - } - - // Handle power of t - if cmd_arguments.is_present("power_t") { - - if let Some(val) = cmd_arguments.value_of("power_t") { - let hvalue = val.parse::()?; - mi.power_t = hvalue; - replacement_hyperparam_ids.push(("power_t".to_string(), hvalue.to_string())); - } - } - - if cmd_arguments.is_present("ffm_power_t") { - if let Some(val) = cmd_arguments.value_of("ffm_power_t") { - let hvalue = val.parse::()?; - mi.ffm_power_t = hvalue; - replacement_hyperparam_ids.push(("ffm_power_t".to_string(), hvalue.to_string())); - } - } - - for (hyper_name, hyper_value) in replacement_hyperparam_ids.into_iter() { - println!("Warning! Updated hyperparameter {} to value {}", hyper_name, hyper_value); - } - - Ok(()) - - } + // Handle power of t + if cmd_arguments.is_present("power_t") { + if let Some(val) = cmd_arguments.value_of("power_t") { + let hvalue = val.parse::()?; + mi.power_t = hvalue; + replacement_hyperparam_ids.push(("power_t".to_string(), hvalue.to_string())); + } + } + if cmd_arguments.is_present("ffm_power_t") { + if let Some(val) = cmd_arguments.value_of("ffm_power_t") { + let hvalue = val.parse::()?; + mi.ffm_power_t = hvalue; + replacement_hyperparam_ids.push(("ffm_power_t".to_string(), hvalue.to_string())); + } + } -} + for (hyper_name, hyper_value) in replacement_hyperparam_ids.into_iter() { + println!( + "Warning! Updated hyperparameter {} to value {}", + hyper_name, hyper_value + ); + } + Ok(()) + } +} #[cfg(test)] mod tests { @@ -461,11 +557,12 @@ mod tests { use super::*; fn ns_desc(i: u16) -> NamespaceDescriptor { - NamespaceDescriptor {namespace_index: i, - namespace_type: vwmap::NamespaceType::Primitive, - namespace_format: vwmap::NamespaceFormat::Categorical} + NamespaceDescriptor { + namespace_index: i, + namespace_type: vwmap::NamespaceType::Primitive, + namespace_format: vwmap::NamespaceFormat::Categorical, + } } - #[test] fn test_interaction_parsing() { @@ -475,20 +572,25 @@ B,featureB C,featureC "#; let vw = vwmap::VwNamespaceMap::new(vw_map_string).unwrap(); - let mi = ModelInstance::new_empty().unwrap(); - + let mi = ModelInstance::new_empty().unwrap(); + let result = mi.create_feature_combo_desc(&vw, "A").unwrap(); - assert_eq!(result, FeatureComboDesc { - namespace_descriptors: vec![ns_desc(0)], - weight: 1.0 - }); - + assert_eq!( + result, + FeatureComboDesc { + namespace_descriptors: vec![ns_desc(0)], + weight: 1.0 + } + ); + let result = mi.create_feature_combo_desc(&vw, "BA:1.5").unwrap(); - assert_eq!(result, FeatureComboDesc { - namespace_descriptors: vec![ns_desc(1), ns_desc(0)], - weight: 1.5 - }); - + assert_eq!( + result, + FeatureComboDesc { + namespace_descriptors: vec![ns_desc(1), ns_desc(0)], + weight: 1.5 + } + ); } #[test] @@ -499,13 +601,15 @@ B,featureB:3 "#; // The main point is that weight in feature names from vw_map_str is ignored let vw = vwmap::VwNamespaceMap::new(vw_map_string).unwrap(); - let mi = ModelInstance::new_empty().unwrap(); + let mi = ModelInstance::new_empty().unwrap(); let result = mi.create_feature_combo_desc(&vw, "BA:1.5").unwrap(); - assert_eq!(result, FeatureComboDesc { - namespace_descriptors: vec![ns_desc(1), ns_desc(0)], - weight: 1.5 - }); - + assert_eq!( + result, + FeatureComboDesc { + namespace_descriptors: vec![ns_desc(1), ns_desc(0)], + weight: 1.5 + } + ); } #[test] @@ -516,24 +620,33 @@ B,featureB C,featureC "#; let vw = vwmap::VwNamespaceMap::new(vw_map_string).unwrap(); - let mi = ModelInstance::new_empty().unwrap(); - let result = mi.create_feature_combo_desc_from_verbose(&vw, "featureA").unwrap(); - assert_eq!(result, FeatureComboDesc { - namespace_descriptors: vec![ns_desc(0)], - - weight: 1.0 - }); - - let result = mi.create_feature_combo_desc_from_verbose(&vw, "featureB,featureA:1.5").unwrap(); - assert_eq!(result, FeatureComboDesc { - namespace_descriptors: vec![ns_desc(1), ns_desc(0)], - weight: 1.5 - }); + let mi = ModelInstance::new_empty().unwrap(); + let result = mi + .create_feature_combo_desc_from_verbose(&vw, "featureA") + .unwrap(); + assert_eq!( + result, + FeatureComboDesc { + namespace_descriptors: vec![ns_desc(0)], + + weight: 1.0 + } + ); + + let result = mi + .create_feature_combo_desc_from_verbose(&vw, "featureB,featureA:1.5") + .unwrap(); + assert_eq!( + result, + FeatureComboDesc { + namespace_descriptors: vec![ns_desc(1), ns_desc(0)], + weight: 1.5 + } + ); let result = mi.create_feature_combo_desc_from_verbose(&vw, "featureB:1.5,featureA"); assert!(result.is_err()); assert_eq!(format!("{:?}", result), "Err(Custom { kind: Other, error: \"Could not parse the value of a feature combination: 1.5,featureA\" })"); - } #[test] @@ -544,32 +657,32 @@ B,featureB C,featureC "#; let vw = vwmap::VwNamespaceMap::new(vw_map_string).unwrap(); - let mi = ModelInstance::new_empty().unwrap(); + let mi = ModelInstance::new_empty().unwrap(); let result = mi.create_field_desc_from_verbose(&vw, "featureA").unwrap(); assert_eq!(result, vec![ns_desc(0)]); - let result = mi.create_field_desc_from_verbose(&vw, "featureA,featureC").unwrap(); + let result = mi + .create_field_desc_from_verbose(&vw, "featureA,featureC") + .unwrap(); assert_eq!(result, vec![ns_desc(0), ns_desc(2)]); - let result = mi.create_field_desc_from_verbose(&vw, "featureA,featureC:3"); assert!(result.is_err()); assert_eq!(format!("{:?}", result), "Err(Custom { kind: Other, error: \"Fields currently do not support passing a value via : \\\"featureA,featureC:3\\\"\" })"); - - } + } #[test] fn test_nn_parsing() { - let mut mi = ModelInstance::new_empty().unwrap(); + let mut mi = ModelInstance::new_empty().unwrap(); let LAYERS = 4; for i in 0..LAYERS { - mi.nn_config.layers.push(HashMap::new()); - } + mi.nn_config.layers.push(HashMap::new()); + } assert!(mi.parse_nn("1:foo:bar").is_ok()); assert!(mi.parse_nn("0::").is_ok()); -// println!("AAA: {:?}", mi.nn_config.layers); + // println!("AAA: {:?}", mi.nn_config.layers); assert_eq!(mi.nn_config.layers[0].get("").unwrap(), ""); assert_eq!(mi.nn_config.layers[1].get("foo").unwrap(), "bar"); assert_eq!(mi.nn_config.layers[2].len(), 0); @@ -581,8 +694,5 @@ C,featureC let result = mi.parse_nn("8:a:b"); assert!(result.is_err()); assert_eq!(format!("{:?}", result), "Err(Custom { kind: Other, error: \"--nn parameter addressing layer 8, but we have only 4 layers\" })"); - - - } - + } } diff --git a/src/multithread_helpers.rs b/src/multithread_helpers.rs index ab69f460..cf7c26ac 100644 --- a/src/multithread_helpers.rs +++ b/src/multithread_helpers.rs @@ -1,30 +1,28 @@ -use std::sync::Arc; -use std::sync::Mutex; -use std::mem::ManuallyDrop; +use crate::regressor::Regressor; use core::ops::{Deref, DerefMut}; -use std::mem; use std::marker::PhantomData; -use crate::regressor::Regressor; - - +use std::mem; +use std::mem::ManuallyDrop; +use std::sync::Arc; +use std::sync::Mutex; // This is a helper for UNSAFELY sharing data between threads -pub struct UnsafelySharableTrait { +pub struct UnsafelySharableTrait { content: ManuallyDrop, reference_count: Arc>>, } -pub type BoxedRegressorTrait = UnsafelySharableTrait>; +pub type BoxedRegressorTrait = UnsafelySharableTrait>; // SUPER UNSAFE // SUPER UNSAFE // SUPER UNSAFE // This literary means we are on our own -- but it is the only way to implement HogWild performantly -unsafe impl Sync for UnsafelySharableTrait {} -unsafe impl Send for UnsafelySharableTrait {} +unsafe impl Sync for UnsafelySharableTrait {} +unsafe impl Send for UnsafelySharableTrait {} -impl Deref for UnsafelySharableTrait { +impl Deref for UnsafelySharableTrait { type Target = T; fn deref(&self) -> &T { @@ -32,14 +30,13 @@ impl Deref for UnsafelySharableTrait { } } -impl DerefMut for UnsafelySharableTrait { +impl DerefMut for UnsafelySharableTrait { fn deref_mut(&mut self) -> &mut T { &mut self.content } } - -impl Drop for UnsafelySharableTrait { +impl Drop for UnsafelySharableTrait { fn drop(&mut self) { unsafe { // we are called before reference is removed, so we need to decide if to drop it or not @@ -49,17 +46,14 @@ impl Drop for UnsafelySharableTrait { // Now this means that the content will be dropped } } - } } - -impl UnsafelySharableTrait { - pub fn new(a: T) -> UnsafelySharableTrait - { +impl UnsafelySharableTrait { + pub fn new(a: T) -> UnsafelySharableTrait { UnsafelySharableTrait:: { content: ManuallyDrop::new(a), - reference_count: Arc::new(Mutex::new(std::marker::PhantomData{})), + reference_count: Arc::new(Mutex::new(std::marker::PhantomData {})), } } } @@ -73,10 +67,10 @@ impl BoxedRegressorTrait { unsafe { // Double deref here sounds weird, but you got to know that dyn Trait and Box are the same thing, just box owns it. // And you can get dyn Trait content, but you can't get box content (directly) - let r2: Box = mem::transmute(& *self.content.deref().deref()); - let ret = BoxedRegressorTrait{ + let r2: Box = mem::transmute(&*self.content.deref().deref()); + let ret = BoxedRegressorTrait { content: ManuallyDrop::new(r2), - reference_count: self.reference_count.clone() + reference_count: self.reference_count.clone(), }; ret } diff --git a/src/optimizer.rs b/src/optimizer.rs index 75e55463..4647a559 100644 --- a/src/optimizer.rs +++ b/src/optimizer.rs @@ -1,8 +1,6 @@ - use std::marker::PhantomData; - -pub trait OptimizerTrait : std::clone::Clone { +pub trait OptimizerTrait: std::clone::Clone { type PerWeightStore: std::clone::Clone; fn new() -> Self; fn init(&mut self, learning_rate: f32, power_t: f32, initial_acc_gradient: f32); @@ -15,20 +13,20 @@ pub trait OptimizerTrait : std::clone::Clone { // This is non-adaptive fixed learning rate SGD, which is exactly the same as Vowpal when --power_t is 0.0 #[derive(Clone)] pub struct OptimizerSGD { - learning_rate: f32, + learning_rate: f32, } impl OptimizerTrait for OptimizerSGD { type PerWeightStore = PhantomData<()>; - + fn get_name() -> &'static str { "SGD" } - + fn new() -> Self { - OptimizerSGD{learning_rate: 0.0} - } - + OptimizerSGD { learning_rate: 0.0 } + } + fn init(&mut self, learning_rate: f32, _power_t: f32, _initial_acc_gradient: f32) { self.learning_rate = learning_rate; } @@ -39,11 +37,10 @@ impl OptimizerTrait for OptimizerSGD { } fn initial_data(&self) -> Self::PerWeightStore { - std::marker::PhantomData{} + std::marker::PhantomData {} } } - /******************* Adagrad with flexible power_t **************************/ /* Regular Adagrad always uses sqrt (power_t = 0.5) */ /* For power_t = 0.5, this is slower than simply using sqrt */ @@ -51,7 +48,7 @@ impl OptimizerTrait for OptimizerSGD { /* implementation is mainly used as a reference */ #[derive(Clone)] pub struct OptimizerAdagradFlex { - learning_rate: f32, + learning_rate: f32, minus_power_t: f32, initial_acc_gradient: f32, } @@ -63,12 +60,16 @@ impl OptimizerTrait for OptimizerAdagradFlex { type PerWeightStore = f32; fn new() -> Self { - OptimizerAdagradFlex{learning_rate: 0.0, minus_power_t: 0.0, initial_acc_gradient: 0.0} - } + OptimizerAdagradFlex { + learning_rate: 0.0, + minus_power_t: 0.0, + initial_acc_gradient: 0.0, + } + } fn init(&mut self, learning_rate: f32, power_t: f32, initial_acc_gradient: f32) { self.learning_rate = learning_rate; - self.minus_power_t = - power_t; + self.minus_power_t = -power_t; self.initial_acc_gradient = initial_acc_gradient; } @@ -78,32 +79,31 @@ impl OptimizerTrait for OptimizerAdagradFlex { let gradient_squared = gradient * gradient; let new_accumulated_gradient_squared = accumulated_gradient_squared + gradient_squared; *data = new_accumulated_gradient_squared; - let update = gradient * self.learning_rate * (new_accumulated_gradient_squared).powf(self.minus_power_t); + let update = gradient + * self.learning_rate + * (new_accumulated_gradient_squared).powf(self.minus_power_t); if update.is_nan() || update.is_infinite() { return 0.0; } return update; } - + fn initial_data(&self) -> Self::PerWeightStore { self.initial_acc_gradient } - } - - /***************** Adagrad using Look Up Table ******************/ // The intuition about low precision is : sqrt/powf is changing less and less as the parameter // grows. This means as parameter grows we can use lesser precision while keeping the error small. // Floating point encoding with separated exponent and mantissa is ideal for such optimization. -pub const FASTMATH_LR_LUT_BITS:u8 = 11; -pub const FASTMATH_LR_LUT_SIZE:usize = 1 << FASTMATH_LR_LUT_BITS; +pub const FASTMATH_LR_LUT_BITS: u8 = 11; +pub const FASTMATH_LR_LUT_SIZE: usize = 1 << FASTMATH_LR_LUT_BITS; #[derive(Clone, Copy)] pub struct OptimizerAdagradLUT { - pub fastmath_lr_lut: [f32; FASTMATH_LR_LUT_SIZE], + pub fastmath_lr_lut: [f32; FASTMATH_LR_LUT_SIZE], } impl OptimizerTrait for OptimizerAdagradLUT { @@ -113,9 +113,11 @@ impl OptimizerTrait for OptimizerAdagradLUT { type PerWeightStore = f32; fn new() -> Self { - OptimizerAdagradLUT{fastmath_lr_lut: [0.0;FASTMATH_LR_LUT_SIZE]} - } - + OptimizerAdagradLUT { + fastmath_lr_lut: [0.0; FASTMATH_LR_LUT_SIZE], + } + } + fn init(&mut self, learning_rate: f32, power_t: f32, initial_acc_gradient: f32) { println!("Calculating look-up tables for Adagrad learning rate calculation"); let minus_power_t = -power_t; @@ -124,18 +126,23 @@ impl OptimizerTrait for OptimizerAdagradLUT { // floating point: 1 bit of sign, 7 bits of signed expontent then floating point bits (mantissa) // we will take 7 bits of exponent + whatever most significant bits of mantissa remain // we take two consequtive such values, so we act as if had rounding - let float_x = (f32::from_bits((x as u32) << (31-FASTMATH_LR_LUT_BITS))) + initial_acc_gradient; - let float_x_plus_one = (f32::from_bits(((x+1) as u32) << (31-FASTMATH_LR_LUT_BITS))) + initial_acc_gradient; - let mut val = learning_rate * ((float_x).powf(minus_power_t) + (float_x_plus_one).powf(minus_power_t)) * 0.5; + let float_x = + (f32::from_bits((x as u32) << (31 - FASTMATH_LR_LUT_BITS))) + initial_acc_gradient; + let float_x_plus_one = + (f32::from_bits(((x + 1) as u32) << (31 - FASTMATH_LR_LUT_BITS))) + + initial_acc_gradient; + let mut val = learning_rate + * ((float_x).powf(minus_power_t) + (float_x_plus_one).powf(minus_power_t)) + * 0.5; // Safety measure if val.is_nan() || val.is_infinite() { val = learning_rate; } - + self.fastmath_lr_lut[x] = val; } } - + #[inline(always)] unsafe fn calculate_update(&self, gradient: f32, data: &mut Self::PerWeightStore) -> f32 { let accumulated_gradient_squared = *data; @@ -143,7 +150,7 @@ impl OptimizerTrait for OptimizerAdagradLUT { let gradient_squared = gradient * gradient; let new_accumulated_gradient_squared = accumulated_gradient_squared + gradient_squared; *data = new_accumulated_gradient_squared; - let key = new_accumulated_gradient_squared.to_bits() >> (31-FASTMATH_LR_LUT_BITS); + let key = new_accumulated_gradient_squared.to_bits() >> (31 - FASTMATH_LR_LUT_BITS); let update = gradient * *self.fastmath_lr_lut.get_unchecked(key as usize); return update; } @@ -152,13 +159,8 @@ impl OptimizerTrait for OptimizerAdagradLUT { // We took it into account when calcualting lookup table, so look at init() 0.0 } - } - - - - mod tests { // Note this useful idiom: importing names from outer (for mod tests) scope. use super::*; @@ -168,9 +170,9 @@ mod tests { let mut l = OptimizerSGD::new(); l.init(0.15, 0.4, 0.0); unsafe { - let mut acc: PhantomData<()> = std::marker::PhantomData{}; + let mut acc: PhantomData<()> = std::marker::PhantomData {}; let p = l.calculate_update(0.1, &mut acc); - assert_eq!(p, 0.1* 0.15); + assert_eq!(p, 0.1 * 0.15); } } @@ -183,19 +185,18 @@ mod tests { acc = 0.9; let p = l.calculate_update(0.1, &mut acc); assert_eq!(p, 0.015576674); - assert_eq!(acc, 0.9 + 0.1*0.1); + assert_eq!(acc, 0.9 + 0.1 * 0.1); acc = 0.0; let p = l.calculate_update(0.1, &mut acc); assert_eq!(p, 0.09464361); - assert_eq!(acc, 0.1*0.1); - + assert_eq!(acc, 0.1 * 0.1); + acc = 0.0; let p = l.calculate_update(0.0, &mut acc); // Here we check that we get NaN back - this is not good, but it's correct -// assert!(p.is_nan()); + // assert!(p.is_nan()); assert_eq!(acc, 0.0); - } } @@ -208,24 +209,21 @@ mod tests { acc = 0.9; let p = l.calculate_update(0.1, &mut acc); assert_eq!(p, 0.015607622); - assert_eq!(acc, 0.9 + 0.1*0.1); + assert_eq!(acc, 0.9 + 0.1 * 0.1); acc = 0.0; let p = l.calculate_update(0.1, &mut acc); assert_eq!(p, 0.09375872); - assert_eq!(acc, 0.1*0.1); + assert_eq!(acc, 0.1 * 0.1); acc = 0.0; let p = l.calculate_update(0.0, &mut acc); // Here we check that we don't get Inf back assert_eq!(p, 0.0); assert_eq!(acc, 0.0); - } } - - #[test] fn test_adagradlut_comparison() { // Here we test that our implementation of LUT has small enough relative error @@ -234,7 +232,19 @@ mod tests { l_lut.init(0.15, 0.4, 0.0); l_flex.init(0.15, 0.4, 0.0); let test_gradients = [-1.0, -0.9, -0.1, -0.00001, 0.0, 0.00001, 0.1, 0.5, 0.9, 1.0]; - let test_accumulations = [0.0000000001, 0.00001, 0.1, 0.5, 1.1, 2.0, 20.0, 200.0, 2000.0, 200000.0, 2000000.0]; + let test_accumulations = [ + 0.0000000001, + 0.00001, + 0.1, + 0.5, + 1.1, + 2.0, + 20.0, + 200.0, + 2000.0, + 200000.0, + 2000000.0, + ]; unsafe { for gradient in test_gradients.iter() { @@ -244,7 +254,7 @@ mod tests { let mut acc_lut: f32 = *accumulation; let p_lut = l_lut.calculate_update(*gradient, &mut acc_lut); let error = (p_flex - p_lut).abs(); - let relative_error:f32; + let relative_error: f32; if p_flex != 0.0 { relative_error = error / p_flex.abs(); } else { @@ -252,15 +262,9 @@ mod tests { } //println!("Relative error {}", relative_error); //println!("Err: {} - p_flex: {}, p_lut: {}, gradient: {}, accumulation {}", error, p_flex, p_lut, *gradient, *accumulation); - assert!(relative_error < 0.05); + assert!(relative_error < 0.05); } } } } - - } - - - - diff --git a/src/parser.rs b/src/parser.rs index 0cdc789d..4cd6c2dd 100644 --- a/src/parser.rs +++ b/src/parser.rs @@ -1,43 +1,40 @@ -use std::fmt; +use crate::vwmap; use fasthash::murmur3; -use std::io::BufRead; use std::error::Error; +use std::fmt; +use std::io::BufRead; use std::io::Error as IOError; use std::io::ErrorKind; use std::str; use std::string::String; -use crate::vwmap; -const RECBUF_LEN:usize = 2048; -pub const HEADER_LEN:u32 = 3; -pub const NAMESPACE_DESC_LEN:u32 = 1; -pub const LABEL_OFFSET:usize = 1; -pub const EXAMPLE_IMPORTANCE_OFFSET:usize = 2; -pub const IS_NOT_SINGLE_MASK : u32 = 1u32 << 31; +const RECBUF_LEN: usize = 2048; +pub const HEADER_LEN: u32 = 3; +pub const NAMESPACE_DESC_LEN: u32 = 1; +pub const LABEL_OFFSET: usize = 1; +pub const EXAMPLE_IMPORTANCE_OFFSET: usize = 2; +pub const IS_NOT_SINGLE_MASK: u32 = 1u32 << 31; pub const MASK31: u32 = !IS_NOT_SINGLE_MASK; -pub const NO_FEATURES: u32= IS_NOT_SINGLE_MASK; // null is just an exact IS_NOT_SINGLE_MASK +pub const NO_FEATURES: u32 = IS_NOT_SINGLE_MASK; // null is just an exact IS_NOT_SINGLE_MASK pub const NO_LABEL: u32 = 0xff; -pub const FLOAT32_ONE: u32 = 1065353216; // 1.0f32.to_bits() - - +pub const FLOAT32_ONE: u32 = 1065353216; // 1.0f32.to_bits() -#[derive (Clone)] +#[derive(Clone)] pub struct VowpalParser { vw_map: vwmap::VwNamespaceMap, tmp_read_buf: Vec, - namespace_hash_seeds: [u32; 256], // Each namespace has its hash seed + namespace_hash_seeds: [u32; 256], // Each namespace has its hash seed pub output_buffer: Vec, } - #[derive(Debug)] -pub struct FlushCommand; // Parser returns FlushCommand to signal flush message +pub struct FlushCommand; // Parser returns FlushCommand to signal flush message #[derive(Debug)] -pub struct HogwildLoadCommand { // Parser returns Hogwild Load as a command +pub struct HogwildLoadCommand { + // Parser returns Hogwild Load as a command pub filename: String, } - impl Error for FlushCommand {} impl fmt::Display for FlushCommand { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { @@ -48,13 +45,16 @@ impl fmt::Display for FlushCommand { impl Error for HogwildLoadCommand {} impl fmt::Display for HogwildLoadCommand { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - write!(f, "Not really an error: a \"hogwild_load\" command from client to load: {}", self.filename) + write!( + f, + "Not really an error: a \"hogwild_load\" command from client to load: {}", + self.filename + ) } } - -/* -organization of records buffer +/* +organization of records buffer (u32) length of the output record (u32) label (f32) Example importance (default: 1.0) @@ -74,245 +74,361 @@ organization of records buffer impl VowpalParser { pub fn new(vw: &vwmap::VwNamespaceMap) -> VowpalParser { - let mut rr = VowpalParser { - vw_map: (*vw).clone(), - tmp_read_buf: Vec::with_capacity(RECBUF_LEN), - output_buffer: Vec::with_capacity(RECBUF_LEN*2), - namespace_hash_seeds: [0; 256], - }; - rr.output_buffer.resize((vw.num_namespaces as u32 * NAMESPACE_DESC_LEN + HEADER_LEN) as usize, 0); + let mut rr = VowpalParser { + vw_map: (*vw).clone(), + tmp_read_buf: Vec::with_capacity(RECBUF_LEN), + output_buffer: Vec::with_capacity(RECBUF_LEN * 2), + namespace_hash_seeds: [0; 256], + }; + rr.output_buffer.resize( + (vw.num_namespaces as u32 * NAMESPACE_DESC_LEN + HEADER_LEN) as usize, + 0, + ); for i in 0..vw.num_namespaces { let namespace_vwname_str = &vw.vw_source.entries[i].namespace_vwname; rr.namespace_hash_seeds[i] = murmur3::hash32(namespace_vwname_str); } rr } - + pub fn print(&self) -> () { println!("item out {:?}", self.output_buffer); } #[inline(always)] - pub fn parse_float_or_error(&self, i_start: usize, i_end :usize, error_str: &str) -> Result> { + pub fn parse_float_or_error( + &self, + i_start: usize, + i_end: usize, + error_str: &str, + ) -> Result> { unsafe { -// println!("{}", str::from_utf8_unchecked(&self.tmp_read_buf[i_start..i_end])); - if i_end - i_start == 4 && - self.tmp_read_buf[i_start + 0] == 'N' as u8 && - self.tmp_read_buf[i_start + 1] == 'O' as u8 && - self.tmp_read_buf[i_start + 2] == 'N' as u8 && - self.tmp_read_buf[i_start + 3] == 'E' as u8 { - return Ok(f32::NAN) - } - + // println!("{}", str::from_utf8_unchecked(&self.tmp_read_buf[i_start..i_end])); + if i_end - i_start == 4 + && self.tmp_read_buf[i_start + 0] == 'N' as u8 + && self.tmp_read_buf[i_start + 1] == 'O' as u8 + && self.tmp_read_buf[i_start + 2] == 'N' as u8 + && self.tmp_read_buf[i_start + 3] == 'E' as u8 + { + return Ok(f32::NAN); + } + match str::from_utf8_unchecked(&self.tmp_read_buf[i_start..i_end]).parse::() { Ok(f) => return Ok(f), - Err(_e) => return Err(Box::new(IOError::new(ErrorKind::Other, format!("{}: {}", error_str, String::from_utf8_lossy(&self.tmp_read_buf[i_start..i_end]))))) + Err(_e) => { + return Err(Box::new(IOError::new( + ErrorKind::Other, + format!( + "{}: {}", + error_str, + String::from_utf8_lossy(&self.tmp_read_buf[i_start..i_end]) + ), + ))) + } }; }; } // This is a very very slow implementation, but it's ok, this is called extremely infrequently to decode a command - pub fn parse_cmd(&self, i_start: usize, rowlen :usize) -> Result>, Box> { + pub fn parse_cmd(&self, i_start: usize, rowlen: usize) -> Result>, Box> { let mut o: Vec> = Vec::new(); let mut i_end = i_start; while i_end < rowlen { - let mut out_vec : Vec = Vec::new(); + let mut out_vec: Vec = Vec::new(); while i_end < rowlen && self.tmp_read_buf[i_end] != 0x20 { out_vec.push(self.tmp_read_buf[i_end]); i_end += 1; } o.push(out_vec); - while i_end < rowlen && self.tmp_read_buf[i_end] == 0x20 {i_end += 1;} - + while i_end < rowlen && self.tmp_read_buf[i_end] == 0x20 { + i_end += 1; + } } Ok(o) } - pub fn next_vowpal(&mut self, input_bufread: &mut impl BufRead) -> Result<&[u32], Box> { - self.tmp_read_buf.truncate(0); - let rowlen1 = match input_bufread.read_until(0x0a, &mut self.tmp_read_buf) { - Ok(0) => return Ok(&[]), - Ok(n) => n, - Err(e) => Err(e)? - }; + pub fn next_vowpal( + &mut self, + input_bufread: &mut impl BufRead, + ) -> Result<&[u32], Box> { + self.tmp_read_buf.truncate(0); + let rowlen1 = match input_bufread.read_until(0x0a, &mut self.tmp_read_buf) { + Ok(0) => return Ok(&[]), + Ok(n) => n, + Err(e) => Err(e)?, + }; + + let bufpos: usize = (self.vw_map.num_namespaces + HEADER_LEN as usize) as usize; + self.output_buffer.truncate(bufpos); + for i in &mut self.output_buffer[0..bufpos] { + *i = NO_FEATURES + } + + let mut current_namespace_num_of_features = 0; - let bufpos: usize = (self.vw_map.num_namespaces + HEADER_LEN as usize) as usize; - self.output_buffer.truncate(bufpos); - for i in &mut self.output_buffer[0..bufpos] { *i = NO_FEATURES }; - - let mut current_namespace_num_of_features = 0; - - unsafe { - let p = self.tmp_read_buf.as_ptr(); - let mut i_start:usize; - let mut i_end:usize = 0; - - // first token is a label or "flush" command - match *p.add(0) { - 0x31 => self.output_buffer[LABEL_OFFSET] = 1, // 1 - 0x2d => self.output_buffer[LABEL_OFFSET] = 0, // -1 - 0x7c => self.output_buffer[LABEL_OFFSET] = NO_LABEL, // when first character is |, this means there is no label - _ => { - // "flush" ascii 66, 6C, 75, 73, 68 - if rowlen1 >= 5 && *p.add(0) == 0x66 && *p.add(1) == 0x6C && *p.add(2) == 0x75 && *p.add(3) == 0x73 && *p.add(4) == 0x68 { - return Err(Box::new(FlushCommand)) - } else if rowlen1 >= "hogwild_load ".len() { - // THIS IS SLOW, BUT IT IS CALLED VERY RARELY - // IF WE WILL AVE COMMANDS CALLED MORE FREQUENTLY, WE WILL NEED A FASTER IMPLEMENTATION - let vecs = self.parse_cmd(0, rowlen1)?; - if vecs.len() == 2 { - let command = String::from_utf8_lossy(&vecs[0]) ; - if command == "hogwild_load" { - let filename = String::from_utf8_lossy(&vecs[1]); - return Err(Box::new(HogwildLoadCommand{filename: filename.to_string()})); - } - } else { - return Err(Box::new(IOError::new(ErrorKind::Other, format!("Cannot parse an example")))) + unsafe { + let p = self.tmp_read_buf.as_ptr(); + let mut i_start: usize; + let mut i_end: usize = 0; + + // first token is a label or "flush" command + match *p.add(0) { + 0x31 => self.output_buffer[LABEL_OFFSET] = 1, // 1 + 0x2d => self.output_buffer[LABEL_OFFSET] = 0, // -1 + 0x7c => self.output_buffer[LABEL_OFFSET] = NO_LABEL, // when first character is |, this means there is no label + _ => { + // "flush" ascii 66, 6C, 75, 73, 68 + if rowlen1 >= 5 + && *p.add(0) == 0x66 + && *p.add(1) == 0x6C + && *p.add(2) == 0x75 + && *p.add(3) == 0x73 + && *p.add(4) == 0x68 + { + return Err(Box::new(FlushCommand)); + } else if rowlen1 >= "hogwild_load ".len() { + // THIS IS SLOW, BUT IT IS CALLED VERY RARELY + // IF WE WILL AVE COMMANDS CALLED MORE FREQUENTLY, WE WILL NEED A FASTER IMPLEMENTATION + let vecs = self.parse_cmd(0, rowlen1)?; + if vecs.len() == 2 { + let command = String::from_utf8_lossy(&vecs[0]); + if command == "hogwild_load" { + let filename = String::from_utf8_lossy(&vecs[1]); + return Err(Box::new(HogwildLoadCommand { + filename: filename.to_string(), + })); } } else { - return Err(Box::new(IOError::new(ErrorKind::Other, format!("Cannot parse an example")))) -// return Err(Box::new(IOError::new(ErrorKind::Other, format!("Unknown first character of the label: ascii {:?}", *p.add(0))))) + return Err(Box::new(IOError::new( + ErrorKind::Other, + format!("Cannot parse an example"), + ))); } + } else { + return Err(Box::new(IOError::new( + ErrorKind::Other, + format!("Cannot parse an example"), + ))); + // return Err(Box::new(IOError::new(ErrorKind::Other, format!("Unknown first character of the label: ascii {:?}", *p.add(0))))) } - }; - - let rowlen = rowlen1 - 1; // ignore last newline byte - if self.output_buffer[LABEL_OFFSET] != NO_LABEL { - // if we have a label, let's check if we also have label weight - while *p.add(i_end) != 0x20 && i_end < rowlen {i_end += 1;}; // find space - while *p.add(i_end) == 0x20 && i_end < rowlen {i_end += 1;}; // find first non-space - //if next character is not "|", we assume it's a example importance - //i_end +=1; - if *p.add(i_end) != 0x7c { // this token does not start with "|", so it has to be example improtance floating point - i_start = i_end; - while *p.add(i_end) != 0x20 && i_end < rowlen {i_end += 1;}; // find end of token (space) - let importance = self.parse_float_or_error(i_start, i_end, "Failed parsing example importance")?; - if importance < 0.0 { - return Err(Box::new(IOError::new(ErrorKind::Other, format!("Example importance cannot be negative: {:?}! ", importance)))); - } - self.output_buffer[EXAMPLE_IMPORTANCE_OFFSET] = importance.to_bits(); - } else { - self.output_buffer[EXAMPLE_IMPORTANCE_OFFSET] = FLOAT32_ONE; - } - } else { - self.output_buffer[EXAMPLE_IMPORTANCE_OFFSET] = FLOAT32_ONE; - } - // Then we look for first namespace - while *p.add(i_end) != 0x7c && i_end < rowlen { i_end += 1;}; - - let mut current_namespace_hash_seed:u32 = 0; - let mut current_namespace_index_offset:usize = HEADER_LEN as usize; - let mut current_namespace_format = vwmap::NamespaceFormat::Categorical; - - let mut bufpos_namespace_start = 0; - let mut current_namespace_weight:f32 = 1.0; - while i_end < rowlen { - // [:] - - // First skip spaces - while *p.add(i_end) == 0x20 && i_end < rowlen {i_end += 1;} + } + }; + + let rowlen = rowlen1 - 1; // ignore last newline byte + if self.output_buffer[LABEL_OFFSET] != NO_LABEL { + // if we have a label, let's check if we also have label weight + while *p.add(i_end) != 0x20 && i_end < rowlen { + i_end += 1; + } // find space + while *p.add(i_end) == 0x20 && i_end < rowlen { + i_end += 1; + } // find first non-space + //if next character is not "|", we assume it's a example importance + //i_end +=1; + if *p.add(i_end) != 0x7c { + // this token does not start with "|", so it has to be example improtance floating point i_start = i_end; - while *p.add(i_end) != 0x20 && *p.add(i_end) != 0x3a && i_end < rowlen {i_end += 1;} // 0x3a = ":" - let i_end_first_part = i_end; - while *p.add(i_end) != 0x20 && i_end < rowlen {i_end += 1; } - - //println!("item out {:?}", std::str::from_utf8(&rr.tmp_read_buf[i_start..i_end])); - if *p.add(i_start) == 0x7c { // "|" - // new namespace index - i_start += 1; - if i_end_first_part != i_end { - // Non-empty part after ":" is namespace weight - current_namespace_weight = self.parse_float_or_error(i_end_first_part+1, i_end, "Failed parsing namespace weight")?; - } else { - current_namespace_weight = 1.0; + while *p.add(i_end) != 0x20 && i_end < rowlen { + i_end += 1; + } // find end of token (space) + let importance = self.parse_float_or_error( + i_start, + i_end, + "Failed parsing example importance", + )?; + if importance < 0.0 { + return Err(Box::new(IOError::new( + ErrorKind::Other, + format!("Example importance cannot be negative: {:?}! ", importance), + ))); + } + self.output_buffer[EXAMPLE_IMPORTANCE_OFFSET] = importance.to_bits(); + } else { + self.output_buffer[EXAMPLE_IMPORTANCE_OFFSET] = FLOAT32_ONE; + } + } else { + self.output_buffer[EXAMPLE_IMPORTANCE_OFFSET] = FLOAT32_ONE; + } + // Then we look for first namespace + while *p.add(i_end) != 0x7c && i_end < rowlen { + i_end += 1; + } + + let mut current_namespace_hash_seed: u32 = 0; + let mut current_namespace_index_offset: usize = HEADER_LEN as usize; + let mut current_namespace_format = vwmap::NamespaceFormat::Categorical; + + let mut bufpos_namespace_start = 0; + let mut current_namespace_weight: f32 = 1.0; + while i_end < rowlen { + // [:] + + // First skip spaces + while *p.add(i_end) == 0x20 && i_end < rowlen { + i_end += 1; + } + i_start = i_end; + while *p.add(i_end) != 0x20 && *p.add(i_end) != 0x3a && i_end < rowlen { + i_end += 1; + } // 0x3a = ":" + let i_end_first_part = i_end; + while *p.add(i_end) != 0x20 && i_end < rowlen { + i_end += 1; + } + + //println!("item out {:?}", std::str::from_utf8(&rr.tmp_read_buf[i_start..i_end])); + if *p.add(i_start) == 0x7c { + // "|" + // new namespace index + i_start += 1; + if i_end_first_part != i_end { + // Non-empty part after ":" is namespace weight + current_namespace_weight = self.parse_float_or_error( + i_end_first_part + 1, + i_end, + "Failed parsing namespace weight", + )?; + } else { + current_namespace_weight = 1.0; + } + // print!("Only single letter namespaces are allowed, however namespace string is: {:?}\n", String::from_utf8_lossy(&self.tmp_read_buf[i_start..i_end_first_part])); + let current_vwname = &self.tmp_read_buf[i_start..i_end_first_part]; + // println!("Current: {:?}", current_vwname); + let current_namespace_descriptor = match self + .vw_map + .map_vwname_to_namespace_descriptor + .get(current_vwname) + { + Some(v) => v, + None => { + return Err(Box::new(IOError::new( + ErrorKind::Other, + format!( + "Feature name was not predeclared in vw_namespace_map.csv: {}", + String::from_utf8_lossy( + &self.tmp_read_buf[i_start..i_end_first_part] + ) + ), + ))) } - // print!("Only single letter namespaces are allowed, however namespace string is: {:?}\n", String::from_utf8_lossy(&self.tmp_read_buf[i_start..i_end_first_part])); - let current_vwname = &self.tmp_read_buf[i_start..i_end_first_part]; -// println!("Current: {:?}", current_vwname); - let current_namespace_descriptor = match self.vw_map.map_vwname_to_namespace_descriptor.get(current_vwname) { - Some(v) => v, - None => return Err(Box::new(IOError::new(ErrorKind::Other, format!("Feature name was not predeclared in vw_namespace_map.csv: {}", String::from_utf8_lossy(&self.tmp_read_buf[i_start..i_end_first_part]))))) - }; - let current_namespace_index = current_namespace_descriptor.namespace_index as usize; - current_namespace_hash_seed = *self.namespace_hash_seeds.get_unchecked(current_namespace_index); - current_namespace_index_offset = current_namespace_index * NAMESPACE_DESC_LEN as usize + HEADER_LEN as usize; - current_namespace_format = current_namespace_descriptor.namespace_format; - current_namespace_num_of_features = 0; - bufpos_namespace_start = self.output_buffer.len(); // this is only used if we will have multiple values - } else { - // We have a feature! Let's hash it and write it to the buffer - // println!("item out {:?}", std::str::from_utf8(&rr.tmp_read_buf[i_start..i_end])); - // print!("F {:?}\n", String::from_utf8_lossy(&self.tmp_read_buf[i_start..i_end_first_part])); - let h = murmur3::hash32_with_seed(&self.tmp_read_buf[i_start..i_end_first_part], - current_namespace_hash_seed) & MASK31; - - let feature_weight:f32 = match i_end - i_end_first_part { - 0 => 1.0, - _ => self.parse_float_or_error(i_end_first_part + 1, i_end, "Failed parsing feature weight")? - }; - - // We have three options: - // - first feature, no weights -> put it in-place - // - if it's second feature and first one was "simple", then promote it - // -- and then just add feature to the end of the buffer - if current_namespace_weight == 1.0 && - feature_weight == 1.0 && - current_namespace_num_of_features == 0 && - current_namespace_format == vwmap::NamespaceFormat::Categorical { - *self.output_buffer.get_unchecked_mut(current_namespace_index_offset) = h; - } else { - if (current_namespace_num_of_features == 1) && (*self.output_buffer.get_unchecked(current_namespace_index_offset) & IS_NOT_SINGLE_MASK) == 0 { - // We need to promote feature currently written in-place to out of place - self.output_buffer.push(*self.output_buffer.get_unchecked(current_namespace_index_offset)); - self.output_buffer.push(FLOAT32_ONE); - debug_assert_eq!(current_namespace_format, vwmap::NamespaceFormat::Categorical); - } - self.output_buffer.push(h); - if current_namespace_format == vwmap::NamespaceFormat::F32 { - // The namespace_skip_prefix allows us to parse a value A100, where A is one byte prefix which gets ignored - let float_start = i_start + self.vw_map.vw_source.namespace_skip_prefix as usize; - let float_value:f32 = match i_end_first_part - float_start { - 0 => f32::NAN, - _ => self.parse_float_or_error(float_start, i_end_first_part, "Failed parsing feature value to float (for float namespace)")? - }; - self.output_buffer.push(float_value.to_bits()); - *self.output_buffer.get_unchecked_mut(current_namespace_index_offset) = IS_NOT_SINGLE_MASK | (((bufpos_namespace_start<<16) + self.output_buffer.len()) as u32); - if current_namespace_weight * feature_weight != 1.0 { - return Err(Box::new(IOError::new(ErrorKind::Other, format!("Namespaces that are f32 can not have weight attached neither to namespace nor to a single feature (basically they can\' use :weight syntax")))) - } - } else { - self.output_buffer.push((current_namespace_weight * feature_weight).to_bits()); - *self.output_buffer.get_unchecked_mut(current_namespace_index_offset) = IS_NOT_SINGLE_MASK | (((bufpos_namespace_start<<16) + self.output_buffer.len()) as u32); + }; + let current_namespace_index = + current_namespace_descriptor.namespace_index as usize; + current_namespace_hash_seed = *self + .namespace_hash_seeds + .get_unchecked(current_namespace_index); + current_namespace_index_offset = + current_namespace_index * NAMESPACE_DESC_LEN as usize + HEADER_LEN as usize; + current_namespace_format = current_namespace_descriptor.namespace_format; + current_namespace_num_of_features = 0; + bufpos_namespace_start = self.output_buffer.len(); // this is only used if we will have multiple values + } else { + // We have a feature! Let's hash it and write it to the buffer + // println!("item out {:?}", std::str::from_utf8(&rr.tmp_read_buf[i_start..i_end])); + // print!("F {:?}\n", String::from_utf8_lossy(&self.tmp_read_buf[i_start..i_end_first_part])); + let h = murmur3::hash32_with_seed( + &self.tmp_read_buf[i_start..i_end_first_part], + current_namespace_hash_seed, + ) & MASK31; + + let feature_weight: f32 = match i_end - i_end_first_part { + 0 => 1.0, + _ => self.parse_float_or_error( + i_end_first_part + 1, + i_end, + "Failed parsing feature weight", + )?, + }; + + // We have three options: + // - first feature, no weights -> put it in-place + // - if it's second feature and first one was "simple", then promote it + // -- and then just add feature to the end of the buffer + if current_namespace_weight == 1.0 + && feature_weight == 1.0 + && current_namespace_num_of_features == 0 + && current_namespace_format == vwmap::NamespaceFormat::Categorical + { + *self + .output_buffer + .get_unchecked_mut(current_namespace_index_offset) = h; + } else { + if (current_namespace_num_of_features == 1) + && (*self + .output_buffer + .get_unchecked(current_namespace_index_offset) + & IS_NOT_SINGLE_MASK) + == 0 + { + // We need to promote feature currently written in-place to out of place + self.output_buffer.push( + *self + .output_buffer + .get_unchecked(current_namespace_index_offset), + ); + self.output_buffer.push(FLOAT32_ONE); + debug_assert_eq!( + current_namespace_format, + vwmap::NamespaceFormat::Categorical + ); + } + self.output_buffer.push(h); + if current_namespace_format == vwmap::NamespaceFormat::F32 { + // The namespace_skip_prefix allows us to parse a value A100, where A is one byte prefix which gets ignored + let float_start = + i_start + self.vw_map.vw_source.namespace_skip_prefix as usize; + let float_value: f32 = match i_end_first_part - float_start { + 0 => f32::NAN, + _ => self.parse_float_or_error( + float_start, + i_end_first_part, + "Failed parsing feature value to float (for float namespace)", + )?, + }; + self.output_buffer.push(float_value.to_bits()); + *self + .output_buffer + .get_unchecked_mut(current_namespace_index_offset) = + IS_NOT_SINGLE_MASK + | (((bufpos_namespace_start << 16) + self.output_buffer.len()) + as u32); + if current_namespace_weight * feature_weight != 1.0 { + return Err(Box::new(IOError::new(ErrorKind::Other, format!("Namespaces that are f32 can not have weight attached neither to namespace nor to a single feature (basically they can\' use :weight syntax")))); } + } else { + self.output_buffer + .push((current_namespace_weight * feature_weight).to_bits()); + *self + .output_buffer + .get_unchecked_mut(current_namespace_index_offset) = + IS_NOT_SINGLE_MASK + | (((bufpos_namespace_start << 16) + self.output_buffer.len()) + as u32); } - current_namespace_num_of_features += 1; } - i_end += 1; - + current_namespace_num_of_features += 1; } + i_end += 1; } - -// println!("item out {:?} {}", self.output_buffer, bufpos); - self.output_buffer[0] = self.output_buffer.len() as u32; - Ok(&self.output_buffer) } + // println!("item out {:?} {}", self.output_buffer, bufpos); + self.output_buffer[0] = self.output_buffer.len() as u32; + Ok(&self.output_buffer) + } } - - #[cfg(test)] mod tests { // Note this useful idiom: importing names from outer (for mod tests) scope. - use vwmap; use super::*; use std::io::Cursor; - + use vwmap; + fn nd(start: u32, end: u32) -> u32 { return (start << 16) + end; } - #[test] fn test_vowpal() { // Test for perfect vowpal-compatible hashing @@ -324,81 +440,141 @@ C,featureC let vw = vwmap::VwNamespaceMap::new(vw_map_string).unwrap(); fn str_to_cursor(s: &str) -> Cursor> { - Cursor::new(s.as_bytes().to_vec()) + Cursor::new(s.as_bytes().to_vec()) } let mut rr = VowpalParser::new(&vw); // we test a single record, single namespace let mut buf = str_to_cursor("1 |A a\n"); - assert_eq!(rr.next_vowpal(&mut buf).unwrap(), [6, 1, FLOAT32_ONE, - 2988156968 & MASK31, - NO_FEATURES, - NO_FEATURES]); - + assert_eq!( + rr.next_vowpal(&mut buf).unwrap(), + [ + 6, + 1, + FLOAT32_ONE, + 2988156968 & MASK31, + NO_FEATURES, + NO_FEATURES + ] + ); + // we test a single record, single namespace, space at the end let mut buf = str_to_cursor("1 |A a \n"); - assert_eq!(rr.next_vowpal(&mut buf).unwrap(), [6, 1, FLOAT32_ONE, - 2988156968 & MASK31, - NO_FEATURES, - NO_FEATURES]); - + assert_eq!( + rr.next_vowpal(&mut buf).unwrap(), + [ + 6, + 1, + FLOAT32_ONE, + 2988156968 & MASK31, + NO_FEATURES, + NO_FEATURES + ] + ); // we test a single record, single namespace, space after label let mut buf = str_to_cursor("1 |A a\n"); - assert_eq!(rr.next_vowpal(&mut buf).unwrap(), [6, 1, FLOAT32_ONE, - 2988156968 & MASK31, - NO_FEATURES, - NO_FEATURES]); - + assert_eq!( + rr.next_vowpal(&mut buf).unwrap(), + [ + 6, + 1, + FLOAT32_ONE, + 2988156968 & MASK31, + NO_FEATURES, + NO_FEATURES + ] + ); + // we test a single record, single namespace, space between namespace and label let mut buf = str_to_cursor("1 |A a\n"); - assert_eq!(rr.next_vowpal(&mut buf).unwrap(), [6, 1, FLOAT32_ONE, - 2988156968 & MASK31, - NO_FEATURES, - NO_FEATURES]); - - - - - + assert_eq!( + rr.next_vowpal(&mut buf).unwrap(), + [ + 6, + 1, + FLOAT32_ONE, + 2988156968 & MASK31, + NO_FEATURES, + NO_FEATURES + ] + ); + let mut buf = str_to_cursor("-1 |B b\n"); - assert_eq!(rr.next_vowpal(&mut buf).unwrap(), [6, 0, FLOAT32_ONE, - NO_FEATURES, - 2422381320 & MASK31, - NO_FEATURES]); + assert_eq!( + rr.next_vowpal(&mut buf).unwrap(), + [ + 6, + 0, + FLOAT32_ONE, + NO_FEATURES, + 2422381320 & MASK31, + NO_FEATURES + ] + ); // single namespace with two features let mut buf = str_to_cursor("1 |A a b\n"); - assert_eq!(rr.next_vowpal(&mut buf).unwrap(), [10, 1, FLOAT32_ONE, - nd(6,10) | IS_NOT_SINGLE_MASK, // |A - NO_FEATURES, // |B - NO_FEATURES, // |C - 2988156968 & MASK31, FLOAT32_ONE, // |A a - 3529656005 & MASK31, FLOAT32_ONE]); // |A b - // two namespaces + assert_eq!( + rr.next_vowpal(&mut buf).unwrap(), + [ + 10, + 1, + FLOAT32_ONE, + nd(6, 10) | IS_NOT_SINGLE_MASK, // |A + NO_FEATURES, // |B + NO_FEATURES, // |C + 2988156968 & MASK31, + FLOAT32_ONE, // |A a + 3529656005 & MASK31, + FLOAT32_ONE + ] + ); // |A b + // two namespaces let mut buf = str_to_cursor("-1 |A a |B b\n"); - assert_eq!(rr.next_vowpal(&mut buf).unwrap(), [6, 0, FLOAT32_ONE, - 2988156968 & MASK31, - 2422381320 & MASK31, - NO_FEATURES]); + assert_eq!( + rr.next_vowpal(&mut buf).unwrap(), + [ + 6, + 0, + FLOAT32_ONE, + 2988156968 & MASK31, + 2422381320 & MASK31, + NO_FEATURES + ] + ); // two namespaces, double space let mut buf = str_to_cursor("-1 |A a |B b\n"); - assert_eq!(rr.next_vowpal(&mut buf).unwrap(), [6, 0, FLOAT32_ONE, - 2988156968 & MASK31, - 2422381320 & MASK31, - NO_FEATURES]); - + assert_eq!( + rr.next_vowpal(&mut buf).unwrap(), + [ + 6, + 0, + FLOAT32_ONE, + 2988156968 & MASK31, + 2422381320 & MASK31, + NO_FEATURES + ] + ); + let mut buf = str_to_cursor("1 |UNDECLARED_NAMESPACE a\n"); let result = rr.next_vowpal(&mut buf); assert!(result.is_err()); assert_eq!(format!("{:?}", result), "Err(Custom { kind: Other, error: \"Feature name was not predeclared in vw_namespace_map.csv: UNDECLARED_NAMESPACE\" })"); - + // namespace weight test let mut buf = str_to_cursor("1 |A:1.0 a\n"); - assert_eq!(rr.next_vowpal(&mut buf).unwrap(), [6, 1, FLOAT32_ONE, - 2988156968 & MASK31, - NO_FEATURES, - NO_FEATURES]); + assert_eq!( + rr.next_vowpal(&mut buf).unwrap(), + [ + 6, + 1, + FLOAT32_ONE, + 2988156968 & MASK31, + NO_FEATURES, + NO_FEATURES + ] + ); // not a parsable number let mut buf = str_to_cursor("1 |A:not_a_parsable_number a\n"); let result = rr.next_vowpal(&mut buf); @@ -409,124 +585,201 @@ C,featureC let mut buf = str_to_cursor("1 |A:1:1 a\n"); let result = rr.next_vowpal(&mut buf); assert!(result.is_err()); - assert_eq!(format!("{:?}", result), "Err(Custom { kind: Other, error: \"Failed parsing namespace weight: 1:1\" })"); + assert_eq!( + format!("{:?}", result), + "Err(Custom { kind: Other, error: \"Failed parsing namespace weight: 1:1\" })" + ); // namespace weight test let mut buf = str_to_cursor("1 |A:2.0 a\n"); - assert_eq!(rr.next_vowpal(&mut buf).unwrap(), [8, 1, FLOAT32_ONE, - nd(6, 8) | IS_NOT_SINGLE_MASK, - NO_FEATURES, - NO_FEATURES, - 2988156968 & MASK31, 2.0f32.to_bits()]); - // feature weight + assert_eq!( + rr.next_vowpal(&mut buf).unwrap(), + [ + 8, + 1, + FLOAT32_ONE, + nd(6, 8) | IS_NOT_SINGLE_MASK, + NO_FEATURES, + NO_FEATURES, + 2988156968 & MASK31, + 2.0f32.to_bits() + ] + ); + // feature weight let mut buf = str_to_cursor("1 |A a:2.0\n"); - assert_eq!(rr.next_vowpal(&mut buf).unwrap(), [8, 1, FLOAT32_ONE, - nd(6, 8) | IS_NOT_SINGLE_MASK, - NO_FEATURES, - NO_FEATURES, - 2988156968 & MASK31, 2.0f32.to_bits()]); - - // two feature weights + assert_eq!( + rr.next_vowpal(&mut buf).unwrap(), + [ + 8, + 1, + FLOAT32_ONE, + nd(6, 8) | IS_NOT_SINGLE_MASK, + NO_FEATURES, + NO_FEATURES, + 2988156968 & MASK31, + 2.0f32.to_bits() + ] + ); + + // two feature weights let mut buf = str_to_cursor("1 |A a:2.0 b:3.0\n"); - assert_eq!(rr.next_vowpal(&mut buf).unwrap(), [10, 1, FLOAT32_ONE, - nd(6, 10) | IS_NOT_SINGLE_MASK, - NO_FEATURES, - NO_FEATURES, - 2988156968 & MASK31, 2.0f32.to_bits(), - 3529656005 & MASK31, 3.0f32.to_bits(), - ]); - - // feature weight + namespace weight - let mut buf = str_to_cursor("1 |A:3 a:2.0\n"); - assert_eq!(rr.next_vowpal(&mut buf).unwrap(), [8, 1, FLOAT32_ONE, - nd(6, 8) | IS_NOT_SINGLE_MASK, - NO_FEATURES, - NO_FEATURES, - 2988156968 & MASK31, 6.0f32.to_bits()]); + assert_eq!( + rr.next_vowpal(&mut buf).unwrap(), + [ + 10, + 1, + FLOAT32_ONE, + nd(6, 10) | IS_NOT_SINGLE_MASK, + NO_FEATURES, + NO_FEATURES, + 2988156968 & MASK31, + 2.0f32.to_bits(), + 3529656005 & MASK31, + 3.0f32.to_bits(), + ] + ); - // bad feature weight + // feature weight + namespace weight + let mut buf = str_to_cursor("1 |A:3 a:2.0\n"); + assert_eq!( + rr.next_vowpal(&mut buf).unwrap(), + [ + 8, + 1, + FLOAT32_ONE, + nd(6, 8) | IS_NOT_SINGLE_MASK, + NO_FEATURES, + NO_FEATURES, + 2988156968 & MASK31, + 6.0f32.to_bits() + ] + ); + + // bad feature weight let mut buf = str_to_cursor("1 |A a:2x0\n"); let result = rr.next_vowpal(&mut buf); assert!(result.is_err()); - assert_eq!(format!("{:?}", result), "Err(Custom { kind: Other, error: \"Failed parsing feature weight: 2x0\" })"); - + assert_eq!( + format!("{:?}", result), + "Err(Custom { kind: Other, error: \"Failed parsing feature weight: 2x0\" })" + ); - // first no weight, then two weighted features + // first no weight, then two weighted features let mut buf = str_to_cursor("1 |A a b:2.0 c:3.0\n"); - assert_eq!(rr.next_vowpal(&mut buf).unwrap(), [12, 1, FLOAT32_ONE, - nd(6, 12) | IS_NOT_SINGLE_MASK, - NO_FEATURES, - NO_FEATURES, - 2988156968 & MASK31, 1.0f32.to_bits(), - 3529656005 & MASK31, 2.0f32.to_bits(), - 906509 & MASK31, 3.0f32.to_bits(), - ]); - - - - + assert_eq!( + rr.next_vowpal(&mut buf).unwrap(), + [ + 12, + 1, + FLOAT32_ONE, + nd(6, 12) | IS_NOT_SINGLE_MASK, + NO_FEATURES, + NO_FEATURES, + 2988156968 & MASK31, + 1.0f32.to_bits(), + 3529656005 & MASK31, + 2.0f32.to_bits(), + 906509 & MASK31, + 3.0f32.to_bits(), + ] + ); + // LABEL TESTS // without label let mut buf = str_to_cursor("|A a\n"); - assert_eq!(rr.next_vowpal(&mut buf).unwrap(), [6, NO_LABEL, FLOAT32_ONE, - 2988156968 & MASK31, - NO_FEATURES, - NO_FEATURES]); - - /* Should we support this ? + assert_eq!( + rr.next_vowpal(&mut buf).unwrap(), + [ + 6, + NO_LABEL, + FLOAT32_ONE, + 2988156968 & MASK31, + NO_FEATURES, + NO_FEATURES + ] + ); + + /* Should we support this ? let mut buf = str_to_cursor(" |A a\n"); assert_eq!(rr.next_vowpal(&mut buf).unwrap(), [6, NO_LABEL, FLOAT32_ONE, - 2988156968 & MASK31, - NO_FEATURES, + 2988156968 & MASK31, + NO_FEATURES, NO_FEATURES]); */ - + //println!("{:?}", rr.output_buffer); // now we test if end-of-stream works correctly str_to_cursor(""); assert_eq!(rr.next_vowpal(&mut buf).unwrap().len(), 0); - + // flush should return [999] let mut buf = str_to_cursor("flush"); - assert_eq!(rr.next_vowpal(&mut buf).err().unwrap().is::(), true); + assert_eq!( + rr.next_vowpal(&mut buf).err().unwrap().is::(), + true + ); // Unrecognized label -> Error let mut buf = str_to_cursor("$1"); let result = rr.next_vowpal(&mut buf); assert!(result.is_err()); - assert_eq!(format!("{:?}", result), "Err(Custom { kind: Other, error: \"Cannot parse an example\" })"); + assert_eq!( + format!("{:?}", result), + "Err(Custom { kind: Other, error: \"Cannot parse an example\" })" + ); // Example importance is negative -> Error let mut buf = str_to_cursor("1 -0.1 |A a\n"); let result = rr.next_vowpal(&mut buf); assert!(result.is_err()); - assert_eq!(format!("{:?}", result), "Err(Custom { kind: Other, error: \"Example importance cannot be negative: -0.1! \" })"); + assert_eq!( + format!("{:?}", result), + "Err(Custom { kind: Other, error: \"Example importance cannot be negative: -0.1! \" })" + ); // After label, there is neither namespace definition (|) nor example importance float let mut buf = str_to_cursor("1 fdsa |A a\n"); let result = rr.next_vowpal(&mut buf); assert!(result.is_err()); - assert_eq!(format!("{:?}", result), "Err(Custom { kind: Other, error: \"Failed parsing example importance: fdsa\" })"); - + assert_eq!( + format!("{:?}", result), + "Err(Custom { kind: Other, error: \"Failed parsing example importance: fdsa\" })" + ); + // Example importance let mut buf = str_to_cursor("1 0.1 |A a\n"); - assert_eq!(rr.next_vowpal(&mut buf).unwrap(), [6, 1, 0.1f32.to_bits(), - 2988156968 & MASK31, - NO_FEATURES, - NO_FEATURES]); + assert_eq!( + rr.next_vowpal(&mut buf).unwrap(), + [ + 6, + 1, + 0.1f32.to_bits(), + 2988156968 & MASK31, + NO_FEATURES, + NO_FEATURES + ] + ); // Example importance with bunch of spaces let mut buf = str_to_cursor("1 0.1 |A a \n"); - assert_eq!(rr.next_vowpal(&mut buf).unwrap(), [6, 1, 0.1f32.to_bits(), - 2988156968 & MASK31, - NO_FEATURES, - NO_FEATURES]); - - - + assert_eq!( + rr.next_vowpal(&mut buf).unwrap(), + [ + 6, + 1, + 0.1f32.to_bits(), + 2988156968 & MASK31, + NO_FEATURES, + NO_FEATURES + ] + ); + // flush should return FlushCommand let mut buf = str_to_cursor("flush"); - assert_eq!(rr.next_vowpal(&mut buf).err().unwrap().is::(), true); + assert_eq!( + rr.next_vowpal(&mut buf).err().unwrap().is::(), + true + ); // flush should return FlushCommand let mut buf = str_to_cursor("hogwild_load /path/to/filename"); @@ -541,7 +794,6 @@ C,featureC assert_eq!(result.is::(), true); let hogwild_command = result.downcast_ref::().unwrap(); assert_eq!(hogwild_command.filename, "/path/to/filename"); - // flush should return FlushCommand let mut buf = str_to_cursor("hogwild_load /path/to/filename "); @@ -554,20 +806,24 @@ C,featureC let mut buf = str_to_cursor("hogwild_load"); let result = rr.next_vowpal(&mut buf); assert!(result.is_err()); - assert_eq!(format!("{:?}", result), "Err(Custom { kind: Other, error: \"Cannot parse an example\" })"); + assert_eq!( + format!("{:?}", result), + "Err(Custom { kind: Other, error: \"Cannot parse an example\" })" + ); let mut buf = str_to_cursor("hogwild_load "); let result = rr.next_vowpal(&mut buf); assert!(result.is_err()); - assert_eq!(format!("{:?}", result), "Err(Custom { kind: Other, error: \"Cannot parse an example\" })"); - + assert_eq!( + format!("{:?}", result), + "Err(Custom { kind: Other, error: \"Cannot parse an example\" })" + ); } - #[test] fn test_float_namespaces() { fn str_to_cursor(s: &str) -> Cursor> { - Cursor::new(s.as_bytes().to_vec()) + Cursor::new(s.as_bytes().to_vec()) } let vw_map_string = r#" @@ -579,10 +835,17 @@ C,featureC let mut rr = VowpalParser::new(&vw); // we test a single record, single namespace, with string value "3" let mut buf = str_to_cursor("-1 |B 3\n"); - assert_eq!(rr.next_vowpal(&mut buf).unwrap(), [6, 0, FLOAT32_ONE, - NO_FEATURES, - 1775699190 & MASK31, - NO_FEATURES]); + assert_eq!( + rr.next_vowpal(&mut buf).unwrap(), + [ + 6, + 0, + FLOAT32_ONE, + NO_FEATURES, + 1775699190 & MASK31, + NO_FEATURES + ] + ); let vw_map_string = r#" A,featureA @@ -594,36 +857,59 @@ C,featureC let mut rr = VowpalParser::new(&vw); // we test a single record, single namespace, with string value "3" let mut buf = str_to_cursor("-1 |B 3\n"); - assert_eq!(rr.next_vowpal(&mut buf).unwrap(), [8, 0, FLOAT32_ONE, - NO_FEATURES, - nd(6, 8) | IS_NOT_SINGLE_MASK, - NO_FEATURES, - 1775699190 & MASK31, 3.0f32.to_bits()]); + assert_eq!( + rr.next_vowpal(&mut buf).unwrap(), + [ + 8, + 0, + FLOAT32_ONE, + NO_FEATURES, + nd(6, 8) | IS_NOT_SINGLE_MASK, + NO_FEATURES, + 1775699190 & MASK31, + 3.0f32.to_bits() + ] + ); let mut buf = str_to_cursor("-1 |B 3 4\n"); - assert_eq!(rr.next_vowpal(&mut buf).unwrap(), [10, 0, FLOAT32_ONE, - NO_FEATURES, - nd(6, 10) | IS_NOT_SINGLE_MASK, - NO_FEATURES, - 1775699190 & MASK31, 3.0f32.to_bits(), - 382082293 & MASK31, 4.0f32.to_bits()]); - - + assert_eq!( + rr.next_vowpal(&mut buf).unwrap(), + [ + 10, + 0, + FLOAT32_ONE, + NO_FEATURES, + nd(6, 10) | IS_NOT_SINGLE_MASK, + NO_FEATURES, + 1775699190 & MASK31, + 3.0f32.to_bits(), + 382082293 & MASK31, + 4.0f32.to_bits() + ] + ); + let mut buf = str_to_cursor("-1 |B not_a_number\n"); let result = rr.next_vowpal(&mut buf); assert!(result.is_err()); assert_eq!(format!("{:?}", result), "Err(Custom { kind: Other, error: \"Failed parsing feature value to float (for float namespace): not_a_number\" })"); - let mut buf = str_to_cursor("-1 |B 3 4\n"); - assert_eq!(rr.next_vowpal(&mut buf).unwrap(), [10, 0, FLOAT32_ONE, - NO_FEATURES, - nd(6, 10) | IS_NOT_SINGLE_MASK, - NO_FEATURES, - 1775699190 & MASK31, 3.0f32.to_bits(), - 382082293 & MASK31, 4.0f32.to_bits()]); - - + assert_eq!( + rr.next_vowpal(&mut buf).unwrap(), + [ + 10, + 0, + FLOAT32_ONE, + NO_FEATURES, + nd(6, 10) | IS_NOT_SINGLE_MASK, + NO_FEATURES, + 1775699190 & MASK31, + 3.0f32.to_bits(), + 382082293 & MASK31, + 4.0f32.to_bits() + ] + ); + let mut buf = str_to_cursor("-1 |B 3:3\n"); let result = rr.next_vowpal(&mut buf); assert!(result.is_err()); @@ -634,10 +920,8 @@ C,featureC assert!(result.is_err()); assert_eq!(format!("{:?}", result), "Err(Custom { kind: Other, error: \"Namespaces that are f32 can not have weight attached neither to namespace nor to a single feature (basically they can\' use :weight syntax\" })"); - - let mut buf = str_to_cursor("-1 |B NONE\n"); - // Now test with skip_prefix = 1 + // Now test with skip_prefix = 1 let vw_map_string = r#" A,featureA B,featureB,f32 @@ -649,37 +933,52 @@ _namespace_skip_prefix,1 let mut rr = VowpalParser::new(&vw); // we test a single record, single namespace, with string value "3" let mut buf = str_to_cursor("-1 |B B3\n"); - assert_eq!(rr.next_vowpal(&mut buf).unwrap(), [8, 0, FLOAT32_ONE, - NO_FEATURES, - nd(6, 8) | IS_NOT_SINGLE_MASK, - NO_FEATURES, - 1416737454 & MASK31, 3.0f32.to_bits()]); + assert_eq!( + rr.next_vowpal(&mut buf).unwrap(), + [ + 8, + 0, + FLOAT32_ONE, + NO_FEATURES, + nd(6, 8) | IS_NOT_SINGLE_MASK, + NO_FEATURES, + 1416737454 & MASK31, + 3.0f32.to_bits() + ] + ); // Because we skip one char, the float value of B is the float value of "" which is NAN let mut buf = str_to_cursor("-1 |B B\n"); - assert_eq!(rr.next_vowpal(&mut buf).unwrap(), [8, 0, FLOAT32_ONE, - NO_FEATURES, - nd(6, 8) | IS_NOT_SINGLE_MASK, - NO_FEATURES, - 25602353 & MASK31, f32::NAN.to_bits()]); + assert_eq!( + rr.next_vowpal(&mut buf).unwrap(), + [ + 8, + 0, + FLOAT32_ONE, + NO_FEATURES, + nd(6, 8) | IS_NOT_SINGLE_MASK, + NO_FEATURES, + 25602353 & MASK31, + f32::NAN.to_bits() + ] + ); let mut buf = str_to_cursor("-1 |B BNONE\n"); - assert_eq!(rr.next_vowpal(&mut buf).unwrap(), [8, 0, FLOAT32_ONE, - NO_FEATURES, - nd(6, 8) | IS_NOT_SINGLE_MASK, - NO_FEATURES, - 1846432377 & MASK31, f32::NAN.to_bits()]); - - - - - - - - + assert_eq!( + rr.next_vowpal(&mut buf).unwrap(), + [ + 8, + 0, + FLOAT32_ONE, + NO_FEATURES, + nd(6, 8) | IS_NOT_SINGLE_MASK, + NO_FEATURES, + 1846432377 & MASK31, + f32::NAN.to_bits() + ] + ); + } - } - #[test] fn test_multibyte_namespaces() { // Test for perfect vowpal-compatible hashing @@ -691,29 +990,38 @@ CC,featureC let vw = vwmap::VwNamespaceMap::new(vw_map_string).unwrap(); fn str_to_cursor(s: &str) -> Cursor> { - Cursor::new(s.as_bytes().to_vec()) + Cursor::new(s.as_bytes().to_vec()) } let mut rr = VowpalParser::new(&vw); // we test a single record, single namespace let mut buf = str_to_cursor("1 |AA a\n"); - assert_eq!(rr.next_vowpal(&mut buf).unwrap(), [6, 1, FLOAT32_ONE, - 292540976 & MASK31, - NO_FEATURES, - NO_FEATURES]); - + assert_eq!( + rr.next_vowpal(&mut buf).unwrap(), + [ + 6, + 1, + FLOAT32_ONE, + 292540976 & MASK31, + NO_FEATURES, + NO_FEATURES + ] + ); + // feature weight + namespace weight let mut buf = str_to_cursor("1 |AA:3 a:2.0\n"); - assert_eq!(rr.next_vowpal(&mut buf).unwrap(), [8, 1, FLOAT32_ONE, - nd(6, 8) | IS_NOT_SINGLE_MASK, - NO_FEATURES, - NO_FEATURES, - 292540976 & MASK31, 6.0f32.to_bits()]); - - + assert_eq!( + rr.next_vowpal(&mut buf).unwrap(), + [ + 8, + 1, + FLOAT32_ONE, + nd(6, 8) | IS_NOT_SINGLE_MASK, + NO_FEATURES, + NO_FEATURES, + 292540976 & MASK31, + 6.0f32.to_bits() + ] + ); } - - - - } diff --git a/src/persistence.rs b/src/persistence.rs index 79df2f0e..ded8869f 100644 --- a/src/persistence.rs +++ b/src/persistence.rs @@ -121,8 +121,7 @@ pub fn new_regressor_from_filename( Box, > { let mut input_bufreader = io::BufReader::new(fs::File::open(filename).unwrap()); - let (mut mi, vw, mut re) = - load_regressor_without_weights(&mut input_bufreader, cmd_arguments)?; + let (mut mi, vw, mut re) = load_regressor_without_weights(&mut input_bufreader, cmd_arguments)?; if !immutable { re.allocate_and_init_weights(&mi); re.overwrite_weights_from_buf(&mut input_bufreader)?; @@ -169,12 +168,12 @@ fn verify_header(input_bufreader: &mut dyn io::Read) -> Result<(), Box, @@ -7,19 +5,17 @@ pub struct PortBuffer { pub tape_len: usize, } - impl PortBuffer { pub fn new(tape_len: usize) -> PortBuffer { PortBuffer { tape: Default::default(), observations: Default::default(), - tape_len: tape_len, + tape_len: tape_len, } } - + pub fn reset(&mut self) { self.observations.truncate(0); self.tape.resize(self.tape_len, 0.0); } - -} \ No newline at end of file +} diff --git a/src/regressor.rs b/src/regressor.rs index 056bf657..ed365a3a 100644 --- a/src/regressor.rs +++ b/src/regressor.rs @@ -1,55 +1,88 @@ +use byteorder::{LittleEndian, ReadBytesExt, WriteBytesExt}; use std::any::Any; +use std::error::Error; use std::io; use std::io::Cursor; -use std::error::Error; -use byteorder::{LittleEndian, ReadBytesExt, WriteBytesExt}; -use crate::model_instance; -use crate::feature_buffer; -use crate::port_buffer; -use crate::optimizer; use crate::block_ffm; -use crate::block_lr; +use crate::block_helpers; use crate::block_loss_functions; -use crate::block_neural; -use crate::block_relu; +use crate::block_lr; use crate::block_misc; +use crate::block_neural; +use crate::block_neural::InitType; use crate::block_normalize; +use crate::block_relu; +use crate::feature_buffer; use crate::graph; -use crate::block_neural::{InitType}; -use crate::block_helpers; +use crate::model_instance; +use crate::optimizer; +use crate::port_buffer; pub trait BlockTrait { fn as_any(&mut self) -> &mut dyn Any; // This enables downcasting - fn forward_backward(&mut self, - further_blocks: &mut [Box], - fb: &feature_buffer::FeatureBuffer, - pb: &mut port_buffer::PortBuffer, - update:bool); - - fn forward( &self, - further_blocks: &[Box], - fb: &feature_buffer::FeatureBuffer, - pb: &mut port_buffer::PortBuffer, ); + fn forward_backward( + &mut self, + further_blocks: &mut [Box], + fb: &feature_buffer::FeatureBuffer, + pb: &mut port_buffer::PortBuffer, + update: bool, + ); + + fn forward( + &self, + further_blocks: &[Box], + fb: &feature_buffer::FeatureBuffer, + pb: &mut port_buffer::PortBuffer, + ); fn allocate_and_init_weights(&mut self, mi: &model_instance::ModelInstance) {} - fn get_serialized_len(&self) -> usize {0} - fn write_weights_to_buf(&self, output_bufwriter: &mut dyn io::Write) -> Result<(), Box> {Ok(())} - fn read_weights_from_buf(&mut self, input_bufreader: &mut dyn io::Read) -> Result<(), Box> {Ok(())} + fn get_serialized_len(&self) -> usize { + 0 + } + fn write_weights_to_buf( + &self, + output_bufwriter: &mut dyn io::Write, + ) -> Result<(), Box> { + Ok(()) + } + fn read_weights_from_buf( + &mut self, + input_bufreader: &mut dyn io::Read, + ) -> Result<(), Box> { + Ok(()) + } fn get_num_output_values(&self, output: graph::OutputSlot) -> usize; fn get_num_output_slots(&self) -> usize; - fn get_input_offset(&mut self, input: graph::InputSlot) -> Result> {Err(format!("get_input_offset() is only supported by CopyBlock"))?} + fn get_input_offset(&mut self, input: graph::InputSlot) -> Result> { + Err(format!("get_input_offset() is only supported by CopyBlock"))? + } fn set_input_offset(&mut self, input: graph::InputSlot, offset: usize) {} fn set_output_offset(&mut self, output: graph::OutputSlot, offset: usize) {} - fn get_block_type(&self) -> graph::BlockType {graph::BlockType::Regular} + fn get_block_type(&self) -> graph::BlockType { + graph::BlockType::Regular + } - fn read_weights_from_buf_into_forward_only(&self, input_bufreader: &mut dyn io::Read, forward: &mut Box) -> Result<(), Box> {Ok(())} + fn read_weights_from_buf_into_forward_only( + &self, + input_bufreader: &mut dyn io::Read, + forward: &mut Box, + ) -> Result<(), Box> { + Ok(()) + } /// Sets internal state of weights based on some completely object-dependent parameters - fn testing_set_weights(&mut self, aa: i32, bb: i32, index: usize, w: &[f32]) -> Result<(), Box> {Ok(())} + fn testing_set_weights( + &mut self, + aa: i32, + bb: i32, + index: usize, + w: &[f32], + ) -> Result<(), Box> { + Ok(()) + } } - pub struct Regressor { pub regressor_name: String, pub blocks_boxes: Vec>, @@ -57,9 +90,8 @@ pub struct Regressor { pub immutable: bool, } - pub fn get_regressor_without_weights(mi: &model_instance::ModelInstance) -> Regressor { - Regressor::new_without_weights(&mi) + Regressor::new_without_weights(&mi) } pub fn get_regressor_with_weights(mi: &model_instance::ModelInstance) -> Regressor { @@ -71,7 +103,7 @@ pub fn get_regressor_with_weights(mi: &model_instance::ModelInstance) -> Regress #[derive(PartialEq)] enum NNActivation { None, - Relu + Relu, } #[derive(PartialEq)] @@ -81,12 +113,9 @@ enum NNLayerNorm { AfterRelu, } - - -impl Regressor { +impl Regressor { pub fn new_without_weights(mi: &model_instance::ModelInstance) -> Regressor { - - let mut rg = Regressor{ + let mut rg = Regressor { blocks_boxes: Vec::new(), regressor_name: format!("Regressor with optimizer \"{:?}\"", mi.optimizer), immutable: false, @@ -103,22 +132,20 @@ impl Regressor { output = block_misc::new_join_block(&mut bg, vec![output, triangle_ffm]).unwrap(); } - if mi.nn_config.layers.len() > 0 { - let mut join_block : Option = None; + let mut join_block: Option = None; if mi.nn_config.topology == "one" { let (a1, a2) = block_misc::new_copy_block_2(&mut bg, output).unwrap(); output = a1; join_block = Some(a2); } else if mi.nn_config.topology == "two" { - // do not copy out the + // do not copy out the } else if mi.nn_config.topology == "four" { let (a1, a2) = block_misc::new_copy_block_2(&mut bg, output).unwrap(); output = a1; join_block = Some(a2); output = block_normalize::new_normalize_layer_block(&mut bg, &mi, output).unwrap(); - /*let (a1, a2) = block_misc::new_copy_block_2(&mut bg, output).unwrap(); output = a1; join_block = Some(a2); @@ -130,115 +157,151 @@ impl Regressor { output = a1; join_block = Some(a2); output = block_normalize::new_stop_block(&mut bg, &mi, output).unwrap(); - } else { - Err(format!("unknown nn topology: \"{}\"", mi.nn_config.topology)).unwrap() + Err(format!( + "unknown nn topology: \"{}\"", + mi.nn_config.topology + )) + .unwrap() } - - for (layer_num, layer) in mi.nn_config.layers.iter().enumerate() { let mut layer = layer.clone(); - let activation_str: String = layer.remove("activation").unwrap_or("none".to_string()).to_string(); - let layernorm_str: String = layer.remove("layernorm").unwrap_or("none".to_string()).to_string(); - let width: usize = layer.remove("width").unwrap_or("20".to_string()).parse().unwrap(); - let maxnorm: f32 = layer.remove("maxnorm").unwrap_or("0.0".to_string()).parse().unwrap(); - let dropout: f32 = layer.remove("dropout").unwrap_or("0.0".to_string()).parse().unwrap(); + let activation_str: String = layer + .remove("activation") + .unwrap_or("none".to_string()) + .to_string(); + let layernorm_str: String = layer + .remove("layernorm") + .unwrap_or("none".to_string()) + .to_string(); + let width: usize = layer + .remove("width") + .unwrap_or("20".to_string()) + .parse() + .unwrap(); + let maxnorm: f32 = layer + .remove("maxnorm") + .unwrap_or("0.0".to_string()) + .parse() + .unwrap(); + let dropout: f32 = layer + .remove("dropout") + .unwrap_or("0.0".to_string()) + .parse() + .unwrap(); //let layernorm: bool = layer.remove("layernorm").unwrap_or("false".to_string()).parse().unwrap(); - let init_type_str: String = layer.remove("init").unwrap_or("hu".to_string()).to_string(); - + let init_type_str: String = + layer.remove("init").unwrap_or("hu".to_string()).to_string(); + if layer.len() > 0 { - panic!("Unknown --nn parameter for layer number {} : {:?}", layer_num, layer); + panic!( + "Unknown --nn parameter for layer number {} : {:?}", + layer_num, layer + ); } - + let activation = match &*activation_str { "none" => NNActivation::None, "relu" => NNActivation::Relu, - _ => Err(format!("unknown nn activation type: \"{}\"", activation_str)).unwrap() + _ => Err(format!( + "unknown nn activation type: \"{}\"", + activation_str + )) + .unwrap(), }; - + let layernorm = match &*layernorm_str { "none" => NNLayerNorm::None, "before" => NNLayerNorm::BeforeRelu, "after" => NNLayerNorm::AfterRelu, - _ => Err(format!("unknown nn layer norm: \"{}\"", layernorm_str)).unwrap() + _ => Err(format!("unknown nn layer norm: \"{}\"", layernorm_str)).unwrap(), }; - + let init_type = match &*init_type_str { "xavier" => InitType::Xavier, "hu" => InitType::Hu, "one" => InitType::One, "zero" => InitType::Zero, - _ => Err(format!("unknown nn initialization type: \"{}\"", init_type_str)).unwrap() + _ => Err(format!( + "unknown nn initialization type: \"{}\"", + init_type_str + )) + .unwrap(), }; let neuron_type = block_neural::NeuronType::WeightedSum; // println!("Neuron layer: width: {}, neuron type: {:?}, dropout: {}, maxnorm: {}, init_type: {:?}", // width, neuron_type, dropout, maxnorm, init_type); - output = block_neural::new_neuronlayer_block(&mut bg, - &mi, - output, - neuron_type, - width, - init_type, - dropout, // dropout - maxnorm, // max norm - false, - ).unwrap(); - - + output = block_neural::new_neuronlayer_block( + &mut bg, + &mi, + output, + neuron_type, + width, + init_type, + dropout, // dropout + maxnorm, // max norm + false, + ) + .unwrap(); + if layernorm == NNLayerNorm::BeforeRelu { - output = block_normalize::new_normalize_layer_block(&mut bg, &mi, output).unwrap(); + output = + block_normalize::new_normalize_layer_block(&mut bg, &mi, output).unwrap(); } if activation == NNActivation::Relu { output = block_relu::new_relu_block(&mut bg, &mi, output).unwrap(); } if layernorm == NNLayerNorm::AfterRelu { - output = block_normalize::new_normalize_layer_block(&mut bg, &mi, output).unwrap(); + output = + block_normalize::new_normalize_layer_block(&mut bg, &mi, output).unwrap(); } - - } // If we have split if join_block.is_some() { - output = block_misc::new_join_block(&mut bg, vec![output, join_block.unwrap()]).unwrap(); + output = + block_misc::new_join_block(&mut bg, vec![output, join_block.unwrap()]).unwrap(); } - output = block_neural::new_neuron_block(&mut bg, &mi, output, block_neural::NeuronType::WeightedSum, block_neural::InitType::One).unwrap(); + output = block_neural::new_neuron_block( + &mut bg, + &mi, + output, + block_neural::NeuronType::WeightedSum, + block_neural::InitType::One, + ) + .unwrap(); } - // now sigmoid has a single input let lossf = block_loss_functions::new_logloss_block(&mut bg, output, true).unwrap(); bg.finalize(); rg.tape_len = bg.get_tape_size(); - + rg.blocks_boxes = bg.take_blocks(); /*for (i, block) in bg.blocks.into_iter().enumerate() { rg.blocks_boxes.push(block); }*/ - + rg } - + pub fn allocate_and_init_weights_(&mut self, mi: &model_instance::ModelInstance) { for rr in &mut self.blocks_boxes { rr.allocate_and_init_weights(mi); } } - pub fn new(mi: &model_instance::ModelInstance) -> Regressor - { + pub fn new(mi: &model_instance::ModelInstance) -> Regressor { let mut rg = Regressor::new_without_weights(mi); rg.allocate_and_init_weights(mi); rg } pub fn get_name(&self) -> String { - self.regressor_name.to_owned() + self.regressor_name.to_owned() } - - pub fn new_portbuffer(&self) -> port_buffer::PortBuffer - { + pub fn new_portbuffer(&self) -> port_buffer::PortBuffer { port_buffer::PortBuffer::new(self.tape_len) } @@ -246,13 +309,19 @@ impl Regressor { self.allocate_and_init_weights_(mi); } - pub fn learn(&mut self, fb: &feature_buffer::FeatureBuffer, pb: &mut port_buffer::PortBuffer, update: bool) -> f32 { + pub fn learn( + &mut self, + fb: &feature_buffer::FeatureBuffer, + pb: &mut port_buffer::PortBuffer, + update: bool, + ) -> f32 { if update && self.immutable { // Important to know: learn() functions in blocks aren't guaranteed to be thread-safe panic!("This regressor is immutable, you cannot call learn() with update = true"); } - let update:bool = update && (fb.example_importance != 0.0); - if !update { // Fast-path for no-update case + let update: bool = update && (fb.example_importance != 0.0); + if !update { + // Fast-path for no-update case return self.predict(fb, pb); } @@ -262,11 +331,15 @@ impl Regressor { assert_eq!(pb.observations.len(), 1); let prediction_probability = pb.observations.pop().unwrap(); - - return prediction_probability + + return prediction_probability; } - - pub fn predict(&self, fb: &feature_buffer::FeatureBuffer, pb: &mut port_buffer::PortBuffer) -> f32 { + + pub fn predict( + &self, + fb: &feature_buffer::FeatureBuffer, + pb: &mut port_buffer::PortBuffer, + ) -> f32 { // TODO: we should find a way of not using unsafe pb.reset(); // empty the tape @@ -276,12 +349,19 @@ impl Regressor { assert_eq!(pb.observations.len(), 1); let prediction_probability = pb.observations.pop().unwrap(); - return prediction_probability + return prediction_probability; } - + // Yeah, this is weird. I just didn't want to break the format compatibility at this point - pub fn write_weights_to_buf(&self, output_bufwriter: &mut dyn io::Write) -> Result<(), Box> { - let length = self.blocks_boxes.iter().map(|block| block.get_serialized_len()).sum::() as u64; + pub fn write_weights_to_buf( + &self, + output_bufwriter: &mut dyn io::Write, + ) -> Result<(), Box> { + let length = self + .blocks_boxes + .iter() + .map(|block| block.get_serialized_len()) + .sum::() as u64; output_bufwriter.write_u64::(length as u64)?; for v in &self.blocks_boxes { @@ -289,16 +369,25 @@ impl Regressor { } Ok(()) } - - pub fn overwrite_weights_from_buf(&mut self, input_bufreader: &mut dyn io::Read) -> Result<(), Box> { + pub fn overwrite_weights_from_buf( + &mut self, + input_bufreader: &mut dyn io::Read, + ) -> Result<(), Box> { // This is a bit weird format // You would expect each block to have its own sig // We'll break compatibility in next release or something similar let len = input_bufreader.read_u64::()?; - let expected_length = self.blocks_boxes.iter().map(|block| block.get_serialized_len()).sum::() as u64; + let expected_length = self + .blocks_boxes + .iter() + .map(|block| block.get_serialized_len()) + .sum::() as u64; if len != expected_length { - return Err(format!("Lenghts of weights array in regressor file differ: got {}, expected {}", len, expected_length))?; + return Err(format!( + "Lenghts of weights array in regressor file differ: got {}, expected {}", + len, expected_length + ))?; } for v in &mut self.blocks_boxes { v.read_weights_from_buf(input_bufreader)?; @@ -307,24 +396,36 @@ impl Regressor { Ok(()) } - - pub fn immutable_regressor_without_weights(&mut self, mi: &model_instance::ModelInstance) -> Result> { + pub fn immutable_regressor_without_weights( + &mut self, + mi: &model_instance::ModelInstance, + ) -> Result> { // make sure we are creating immutable regressor from SGD mi - assert!(mi.optimizer == model_instance::Optimizer::SGD); + assert!(mi.optimizer == model_instance::Optimizer::SGD); let mut rg = Regressor::new_without_weights(&mi); rg.immutable = true; - Ok(rg) + Ok(rg) } - - pub fn into_immutable_regressor_from_buf(&mut self, rg: &mut Regressor, input_bufreader: &mut dyn io::Read) -> Result<(), Box> { + pub fn into_immutable_regressor_from_buf( + &mut self, + rg: &mut Regressor, + input_bufreader: &mut dyn io::Read, + ) -> Result<(), Box> { // TODO Ideally we would make a copy, not based on model_instance. but this is easier at the moment - + let len = input_bufreader.read_u64::()?; - let expected_length = self.blocks_boxes.iter().map(|bb| bb.get_serialized_len()).sum::() as u64; + let expected_length = self + .blocks_boxes + .iter() + .map(|bb| bb.get_serialized_len()) + .sum::() as u64; if len != expected_length { - return Err(format!("Lenghts of weights array in regressor file differ: got {}, expected {}", len, expected_length))?; + return Err(format!( + "Lenghts of weights array in regressor file differ: got {}, expected {}", + len, expected_length + ))?; } for (i, v) in &mut self.blocks_boxes.iter().enumerate() { v.read_weights_from_buf_into_forward_only(input_bufreader, &mut rg.blocks_boxes[i])?; @@ -334,10 +435,13 @@ impl Regressor { } // Create immutable regressor from current regressor - pub fn immutable_regressor(&mut self, mi: &model_instance::ModelInstance) -> Result> { - // Only to be used by unit tests + pub fn immutable_regressor( + &mut self, + mi: &model_instance::ModelInstance, + ) -> Result> { + // Only to be used by unit tests // make sure we are creating immutable regressor from SGD mi - assert!(mi.optimizer == model_instance::Optimizer::SGD); + assert!(mi.optimizer == model_instance::Optimizer::SGD); let mut rg = self.immutable_regressor_without_weights(&mi)?; rg.allocate_and_init_weights(&mi); @@ -350,51 +454,80 @@ impl Regressor { } Ok(rg) } - - } - mod tests { // Note this useful idiom: importing names from outer (for mod tests) scope. use super::*; use crate::feature_buffer::HashAndValue; /* LR TESTS */ - fn lr_vec(v:Vec) -> feature_buffer::FeatureBuffer { + fn lr_vec(v: Vec) -> feature_buffer::FeatureBuffer { feature_buffer::FeatureBuffer { - label: 0.0, - example_importance: 1.0, - example_number: 0, - lr_buffer: v, - ffm_buffer: Vec::new(), - ffm_fields_count: 0, + label: 0.0, + example_importance: 1.0, + example_number: 0, + lr_buffer: v, + ffm_buffer: Vec::new(), + ffm_fields_count: 0, } } - #[test] fn test_learning_turned_off() { - let mut mi = model_instance::ModelInstance::new_empty().unwrap(); + let mut mi = model_instance::ModelInstance::new_empty().unwrap(); mi.optimizer = model_instance::Optimizer::AdagradLUT; let mut re = Regressor::new(&mi); let mut pb = re.new_portbuffer(); // Empty model: no matter how many features, prediction is 0.5 assert_eq!(re.learn(&lr_vec(vec![]), &mut pb, false), 0.5); - assert_eq!(re.learn(&lr_vec(vec![HashAndValue{hash: 1, value: 1.0, combo_index: 0,}]), &mut pb, false), 0.5); - assert_eq!(re.learn(&lr_vec(vec![HashAndValue{hash: 1, value: 1.0, combo_index: 0,}, HashAndValue{hash:2, value: 1.0, combo_index: 0,}]), &mut pb, false), 0.5); + assert_eq!( + re.learn( + &lr_vec(vec![HashAndValue { + hash: 1, + value: 1.0, + combo_index: 0, + }]), + &mut pb, + false + ), + 0.5 + ); + assert_eq!( + re.learn( + &lr_vec(vec![ + HashAndValue { + hash: 1, + value: 1.0, + combo_index: 0, + }, + HashAndValue { + hash: 2, + value: 1.0, + combo_index: 0, + } + ]), + &mut pb, + false + ), + 0.5 + ); } #[test] fn test_power_t_zero() { // When power_t is zero, then all optimizers behave exactly like SGD - // So we want to test all three + // So we want to test all three let mut mi = model_instance::ModelInstance::new_empty().unwrap(); mi.learning_rate = 0.1; mi.power_t = 0.0; - - let vec_in = &lr_vec(vec![HashAndValue{hash: 1, value: 1.0, combo_index: 0,}]); - + + let vec_in = &lr_vec(vec![HashAndValue { + hash: 1, + value: 1.0, + combo_index: 0, + }]); + // Here learning rate mechanism does not affect the observations, so let's verify three different ones mi.optimizer = model_instance::Optimizer::AdagradFlex; @@ -402,10 +535,10 @@ mod tests { //Box::new(Regressor::::new(&mi)), Box::new(Regressor::new(&mi)), //Box::new(Regressor::::new(&mi)) - ]; - + ]; + let mut pb = regressors[0].new_portbuffer(); - + for re in &mut regressors { assert_eq!(re.learn(vec_in, &mut pb, true), 0.5); assert_eq!(re.learn(vec_in, &mut pb, true), 0.48750263); @@ -422,108 +555,252 @@ mod tests { mi.learning_rate = 0.1; mi.power_t = 0.0; mi.optimizer = model_instance::Optimizer::AdagradLUT; - + let mut re = Regressor::new(&mi); let mut pb = re.new_portbuffer(); - let vec_in = &lr_vec(vec![HashAndValue{hash: 1, value: 1.0, combo_index: 0}, HashAndValue{hash: 1, value: 2.0, combo_index: 0,}]); + let vec_in = &lr_vec(vec![ + HashAndValue { + hash: 1, + value: 1.0, + combo_index: 0, + }, + HashAndValue { + hash: 1, + value: 2.0, + combo_index: 0, + }, + ]); assert_eq!(re.learn(vec_in, &mut pb, true), 0.5); assert_eq!(re.learn(vec_in, &mut pb, true), 0.38936076); assert_eq!(re.learn(vec_in, &mut pb, true), 0.30993468); } - #[test] fn test_power_t_half__() { - let mut mi = model_instance::ModelInstance::new_empty().unwrap(); + let mut mi = model_instance::ModelInstance::new_empty().unwrap(); mi.learning_rate = 0.1; mi.power_t = 0.5; mi.init_acc_gradient = 0.0; mi.optimizer = model_instance::Optimizer::AdagradFlex; let mut re = Regressor::new(&mi); let mut pb = re.new_portbuffer(); - - assert_eq!(re.learn(&lr_vec(vec![HashAndValue{hash:1, value: 1.0, combo_index: 0}]), &mut pb, true), 0.5); - assert_eq!(re.learn(&lr_vec(vec![HashAndValue{hash:1, value: 1.0, combo_index: 0}]), &mut pb, true), 0.4750208); - assert_eq!(re.learn(&lr_vec(vec![HashAndValue{hash:1, value: 1.0, combo_index: 0}]), &mut pb, true), 0.45788094); + + assert_eq!( + re.learn( + &lr_vec(vec![HashAndValue { + hash: 1, + value: 1.0, + combo_index: 0 + }]), + &mut pb, + true + ), + 0.5 + ); + assert_eq!( + re.learn( + &lr_vec(vec![HashAndValue { + hash: 1, + value: 1.0, + combo_index: 0 + }]), + &mut pb, + true + ), + 0.4750208 + ); + assert_eq!( + re.learn( + &lr_vec(vec![HashAndValue { + hash: 1, + value: 1.0, + combo_index: 0 + }]), + &mut pb, + true + ), + 0.45788094 + ); } #[test] fn test_power_t_half_fastmath() { - let mut mi = model_instance::ModelInstance::new_empty().unwrap(); + let mut mi = model_instance::ModelInstance::new_empty().unwrap(); mi.learning_rate = 0.1; mi.power_t = 0.5; mi.fastmath = true; mi.optimizer = model_instance::Optimizer::AdagradLUT; mi.init_acc_gradient = 0.0; - + let mut re = get_regressor_with_weights(&mi); let mut pb = re.new_portbuffer(); let mut p: f32; - - p = re.learn(&lr_vec(vec![HashAndValue{hash:1, value: 1.0, combo_index: 0}]), &mut pb, true); + + p = re.learn( + &lr_vec(vec![HashAndValue { + hash: 1, + value: 1.0, + combo_index: 0, + }]), + &mut pb, + true, + ); assert_eq!(p, 0.5); - p = re.learn(&lr_vec(vec![HashAndValue{hash:1, value: 1.0, combo_index: 0}]), &mut pb, true); - if optimizer::FASTMATH_LR_LUT_BITS == 12 { + p = re.learn( + &lr_vec(vec![HashAndValue { + hash: 1, + value: 1.0, + combo_index: 0, + }]), + &mut pb, + true, + ); + if optimizer::FASTMATH_LR_LUT_BITS == 12 { assert_eq!(p, 0.47539312); - } else if optimizer::FASTMATH_LR_LUT_BITS == 11 { + } else if optimizer::FASTMATH_LR_LUT_BITS == 11 { assert_eq!(p, 0.475734); } else { - assert!(false, "Exact value for the test is missing, please edit the test"); + assert!( + false, + "Exact value for the test is missing, please edit the test" + ); } } #[test] fn test_power_t_half_two_features() { - let mut mi = model_instance::ModelInstance::new_empty().unwrap(); + let mut mi = model_instance::ModelInstance::new_empty().unwrap(); mi.learning_rate = 0.1; mi.power_t = 0.5; mi.bit_precision = 18; mi.init_acc_gradient = 0.0; mi.optimizer = model_instance::Optimizer::AdagradFlex; - + let mut re = Regressor::new(&mi); let mut pb = re.new_portbuffer(); // Here we take twice two features and then once just one - assert_eq!(re.learn(&lr_vec(vec![HashAndValue{hash: 1, value: 1.0, combo_index: 0}, HashAndValue{hash:2, value: 1.0, combo_index: 0}]), &mut pb, true), 0.5); - assert_eq!(re.learn(&lr_vec(vec![HashAndValue{hash: 1, value: 1.0, combo_index: 0}, HashAndValue{hash:2, value: 1.0, combo_index: 0}]), &mut pb, true), 0.45016602); - assert_eq!(re.learn(&lr_vec(vec![HashAndValue{hash: 1, value: 1.0, combo_index: 0}]), &mut pb, true), 0.45836908); + assert_eq!( + re.learn( + &lr_vec(vec![ + HashAndValue { + hash: 1, + value: 1.0, + combo_index: 0 + }, + HashAndValue { + hash: 2, + value: 1.0, + combo_index: 0 + } + ]), + &mut pb, + true + ), + 0.5 + ); + assert_eq!( + re.learn( + &lr_vec(vec![ + HashAndValue { + hash: 1, + value: 1.0, + combo_index: 0 + }, + HashAndValue { + hash: 2, + value: 1.0, + combo_index: 0 + } + ]), + &mut pb, + true + ), + 0.45016602 + ); + assert_eq!( + re.learn( + &lr_vec(vec![HashAndValue { + hash: 1, + value: 1.0, + combo_index: 0 + }]), + &mut pb, + true + ), + 0.45836908 + ); } #[test] fn test_non_one_weight() { - let mut mi = model_instance::ModelInstance::new_empty().unwrap(); + let mut mi = model_instance::ModelInstance::new_empty().unwrap(); mi.learning_rate = 0.1; mi.power_t = 0.0; mi.bit_precision = 18; mi.optimizer = model_instance::Optimizer::AdagradLUT; - + let mut re = Regressor::new(&mi); let mut pb = re.new_portbuffer(); - - assert_eq!(re.learn(&lr_vec(vec![HashAndValue{hash:1, value: 2.0, combo_index: 0}]), &mut pb, true), 0.5); - assert_eq!(re.learn(&lr_vec(vec![HashAndValue{hash:1, value: 2.0, combo_index: 0}]), &mut pb, true), 0.45016602); - assert_eq!(re.learn(&lr_vec(vec![HashAndValue{hash:1, value: 2.0, combo_index: 0}]), &mut pb, true), 0.40611085); + + assert_eq!( + re.learn( + &lr_vec(vec![HashAndValue { + hash: 1, + value: 2.0, + combo_index: 0 + }]), + &mut pb, + true + ), + 0.5 + ); + assert_eq!( + re.learn( + &lr_vec(vec![HashAndValue { + hash: 1, + value: 2.0, + combo_index: 0 + }]), + &mut pb, + true + ), + 0.45016602 + ); + assert_eq!( + re.learn( + &lr_vec(vec![HashAndValue { + hash: 1, + value: 2.0, + combo_index: 0 + }]), + &mut pb, + true + ), + 0.40611085 + ); } #[test] fn test_example_importance() { - let mut mi = model_instance::ModelInstance::new_empty().unwrap(); + let mut mi = model_instance::ModelInstance::new_empty().unwrap(); mi.learning_rate = 0.1; mi.power_t = 0.0; mi.bit_precision = 18; mi.optimizer = model_instance::Optimizer::AdagradLUT; mi.fastmath = true; - + let mut re = Regressor::new(&mi); let mut pb = re.new_portbuffer(); - - let mut fb_instance = lr_vec(vec![HashAndValue{hash: 1, value: 1.0, combo_index: 0}]); + + let mut fb_instance = lr_vec(vec![HashAndValue { + hash: 1, + value: 1.0, + combo_index: 0, + }]); fb_instance.example_importance = 0.5; assert_eq!(re.learn(&fb_instance, &mut pb, true), 0.5); assert_eq!(re.learn(&fb_instance, &mut pb, true), 0.49375027); assert_eq!(re.learn(&fb_instance, &mut pb, true), 0.4875807); } - } - diff --git a/src/serving.rs b/src/serving.rs index 05e1622d..aa859c1e 100644 --- a/src/serving.rs +++ b/src/serving.rs @@ -1,24 +1,22 @@ +use daemonize::Daemonize; use std::error::Error; -use std::net; -use std::io::{BufReader, BufWriter}; use std::io; -use std::thread; +use std::io::{BufReader, BufWriter}; +use std::net; +use std::ops::DerefMut; use std::sync::mpsc; use std::sync::Arc; use std::sync::Mutex; -use std::ops::DerefMut; -use daemonize::Daemonize; +use std::thread; -use crate::parser; -use crate::vwmap; -use crate::regressor; use crate::feature_buffer; use crate::model_instance; +use crate::multithread_helpers::BoxedRegressorTrait; +use crate::parser; use crate::persistence; -use crate::multithread_helpers::{BoxedRegressorTrait}; use crate::port_buffer; - - +use crate::regressor; +use crate::vwmap; pub struct Serving { listening_interface: String, @@ -45,7 +43,7 @@ impl IsEmpty for io::BufReader<&net::TcpStream> { } // These are used only for unit-tests -#[derive (Debug, PartialEq)] +#[derive(Debug, PartialEq)] pub enum ConnectionEnd { EndOfStream, StreamWriteError, @@ -55,9 +53,9 @@ pub enum ConnectionEnd { impl WorkerThread { pub fn new( - id: u32, - re_fixed: BoxedRegressorTrait, - fbt: feature_buffer::FeatureBufferTranslator, + id: u32, + re_fixed: BoxedRegressorTrait, + fbt: feature_buffer::FeatureBufferTranslator, pa: parser::VowpalParser, pb: port_buffer::PortBuffer, receiver: Arc>>, @@ -76,12 +74,12 @@ impl WorkerThread { Ok(thread) } - pub fn handle_connection(&mut self, - reader: &mut (impl io::BufRead + IsEmpty), - writer: &mut impl io::Write, - ) -> ConnectionEnd - { - let mut i = 0u64; // This is per-thread example number + pub fn handle_connection( + &mut self, + reader: &mut (impl io::BufRead + IsEmpty), + writer: &mut impl io::Write, + ) -> ConnectionEnd { + let mut i = 0u64; // This is per-thread example number loop { let reading_result = self.pa.next_vowpal(reader); @@ -89,68 +87,93 @@ impl WorkerThread { Ok([]) => return ConnectionEnd::EndOfStream, // EOF Ok(buffer2) => { self.fbt.translate(buffer2, i); - let p = self.re_fixed.predict(&(self.fbt.feature_buffer), &mut self.pb); + let p = self + .re_fixed + .predict(&(self.fbt.feature_buffer), &mut self.pb); let p_res = format!("{:.6}\n", p); match writer.write_all(p_res.as_bytes()) { - Ok(_) => {}, - Err(_e) => { /*println!("Write to socket failed, dropping it"); */ return ConnectionEnd::StreamWriteError; } + Ok(_) => {} + Err(_e) => { + /*println!("Write to socket failed, dropping it"); */ + return ConnectionEnd::StreamWriteError; + } }; - }, - Err(e) => - { - if e.is::() { - // FlushCommand just causes us to flush, not to break - match writer.flush() { - Ok(_) => {}, - Err(_e) => { /*println!("Flushing the socket failed, dropping it");*/ return ConnectionEnd::StreamFlushError; } + } + Err(e) => { + if e.is::() { + // FlushCommand just causes us to flush, not to break + match writer.flush() { + Ok(_) => {} + Err(_e) => { + /*println!("Flushing the socket failed, dropping it");*/ + return ConnectionEnd::StreamFlushError; + } + } + } else if e.is::() { + // FlushCommand just causes us to flush, not to break + let hogwild_command = + e.downcast_ref::().unwrap(); + match persistence::hogwild_load( + self.re_fixed.deref_mut(), + &hogwild_command.filename, + ) { + Ok(_) => { + let p_res = format!("hogwild_load success\n"); + match writer.write_all(p_res.as_bytes()) { + Ok(_) => {} + Err(_e) => { + /*println!("Write to socket failed, dropping it"); */ + return ConnectionEnd::StreamWriteError; + } + }; } - } else if e.is::() { - // FlushCommand just causes us to flush, not to break - let hogwild_command = e.downcast_ref::().unwrap(); - match persistence::hogwild_load(self.re_fixed.deref_mut(), &hogwild_command.filename) { - Ok(_) => { - let p_res = format!("hogwild_load success\n"); - match writer.write_all(p_res.as_bytes()) { - Ok(_) => {}, - Err(_e) => { /*println!("Write to socket failed, dropping it"); */ return ConnectionEnd::StreamWriteError; } - }; - }, + Err(_e) => { + // TODO This kind of error should fold the whole daemon... + let p_res = format!("ERR: hogwild_load fail\n"); + match writer.write_all(p_res.as_bytes()) { + Ok(_) => {} + Err(_e) => { + /*println!("Write to socket failed, dropping it"); */ + return ConnectionEnd::StreamWriteError; + } + }; + return ConnectionEnd::StreamWriteError; + } + } + } else { + let p_res = format!("ERR: {}\n", e.to_string()); + match writer.write_all(p_res.as_bytes()) { + Ok(_) => match writer.flush() { + Ok(_) => {} Err(_e) => { - // TODO This kind of error should fold the whole daemon... - let p_res = format!("ERR: hogwild_load fail\n"); - match writer.write_all(p_res.as_bytes()) { - Ok(_) => {}, - Err(_e) => { /*println!("Write to socket failed, dropping it"); */ return ConnectionEnd::StreamWriteError; } - }; - return ConnectionEnd::StreamWriteError; + /*println!("Flushing the socket failed, dropping it");*/ + return ConnectionEnd::StreamFlushError; } - } - } else - { - let p_res = format!("ERR: {}\n", e.to_string()); - match writer.write_all(p_res.as_bytes()) { - Ok(_) => match writer.flush() { - Ok(_) => {}, - Err(_e) => { /*println!("Flushing the socket failed, dropping it");*/ return ConnectionEnd::StreamFlushError; } - }, - Err(_e) => { /*println!("Write to socket failed, dropping it"); */return ConnectionEnd::StreamWriteError; } - }; - return ConnectionEnd::ParseError; - } - }, + }, + Err(_e) => { + /*println!("Write to socket failed, dropping it"); */ + return ConnectionEnd::StreamWriteError; + } + }; + return ConnectionEnd::ParseError; + } + } }; // lazy flushing if reader.is_empty() { match writer.flush() { - Ok(_) => {}, - Err(_e) => { /*println!("Flushing the socket failed, dropping it");*/ return ConnectionEnd::StreamFlushError; } + Ok(_) => {} + Err(_e) => { + /*println!("Flushing the socket failed, dropping it");*/ + return ConnectionEnd::StreamFlushError; + } }; } i += 1; } } - + pub fn start(&mut self, receiver: Arc>>) -> () { // Simple endless serving loop: receive new connection and serve it // when handle_connection exits, the connection is dropped @@ -161,25 +184,22 @@ impl WorkerThread { self.handle_connection(&mut reader, &mut writer); } } - } - impl Serving { - pub fn new<'a>(cl: &clap::ArgMatches<'a>, - vw: &vwmap::VwNamespaceMap, - re_fixed: Box, - mi: &model_instance::ModelInstance, + pub fn new<'a>( + cl: &clap::ArgMatches<'a>, + vw: &vwmap::VwNamespaceMap, + re_fixed: Box, + mi: &model_instance::ModelInstance, ) -> Result> { let port = match cl.value_of("port") { Some(port) => port.parse().expect("Port should be integer"), - None => 26542 + None => 26542, }; let (sender, receiver) = mpsc::channel(); let receiver = Arc::new(Mutex::new(receiver)); - - let listening_interface = format!("127.0.0.1:{}", port); println!("Starting to listen on {}", listening_interface); let mut s = Serving { @@ -190,8 +210,10 @@ impl Serving { }; let num_children = match cl.value_of("num_children") { - Some(num_children) => num_children.parse().expect("num_children should be integer"), - None => 10 + Some(num_children) => num_children + .parse() + .expect("num_children should be integer"), + None => 10, }; println!("Number of threads {}", num_children); @@ -212,12 +234,13 @@ impl Serving { let fbt = feature_buffer::FeatureBufferTranslator::new(mi); let pa = parser::VowpalParser::new(&vw); for i in 0..num_children { - let newt = WorkerThread::new(i, - re_fixed2.clone(), - fbt.clone(), - pa.clone(), - pb.clone(), - Arc::clone(&receiver), + let newt = WorkerThread::new( + i, + re_fixed2.clone(), + fbt.clone(), + pa.clone(), + pb.clone(), + Arc::clone(&receiver), )?; s.worker_threads.push(newt); } @@ -225,7 +248,8 @@ impl Serving { } pub fn serve(&mut self) -> Result<(), Box> { - let listener = net::TcpListener::bind(&self.listening_interface).expect("Cannot bind to the interface"); + let listener = net::TcpListener::bind(&self.listening_interface) + .expect("Cannot bind to the interface"); println!("Bind done, deamonizing and calling accept"); for stream in listener.incoming() { self.sender.send(stream?)?; @@ -234,31 +258,28 @@ impl Serving { } } - #[cfg(test)] mod tests { // Note this useful idiom: importing names from outer (for mod tests) scope. use super::*; + use crate::feature_buffer; use crate::regressor; + use mockstream::{FailingMockStream, SharedMockStream}; use std::io::ErrorKind; - use mockstream::{SharedMockStream, FailingMockStream}; - use crate::feature_buffer; - use tempfile::{tempdir}; use std::str; - + use tempfile::tempdir; impl IsEmpty for std::io::BufReader { fn is_empty(&mut self) -> bool { - return true + return true; } } -impl IsEmpty for std::io::BufReader { + impl IsEmpty for std::io::BufReader { fn is_empty(&mut self) -> bool { - return true + return true; } } - #[test] fn test_handle_connection() { let vw_map_string = r#" @@ -267,7 +288,7 @@ B,featureB C,featureC "#; let vw = vwmap::VwNamespaceMap::new(vw_map_string).unwrap(); - let mut mi = model_instance::ModelInstance::new_empty().unwrap(); + let mut mi = model_instance::ModelInstance::new_empty().unwrap(); mi.optimizer = model_instance::Optimizer::AdagradLUT; let mut re = regressor::Regressor::new(&mi); mi.optimizer = model_instance::Optimizer::SGD; @@ -276,41 +297,50 @@ C,featureC let pa = parser::VowpalParser::new(&vw); let pb = re_fixed.new_portbuffer(); - let mut newt = WorkerThread {id: 1, - fbt: fbt, - pa: pa, - re_fixed: re_fixed, - pb - }; + let mut newt = WorkerThread { + id: 1, + fbt: fbt, + pa: pa, + re_fixed: re_fixed, + pb, + }; - { // WORKING STREAM TEST + { + // WORKING STREAM TEST let mut mocked_stream = SharedMockStream::new(); let mut reader = BufReader::new(mocked_stream.clone()); let mut writer = BufWriter::new(mocked_stream.clone()); // Just passes through, as the stream is empty newt.handle_connection(&mut reader, &mut writer); - // now let's start playing mocked_stream.push_bytes_to_read(b"|A 0 |A 0"); - assert_eq!(ConnectionEnd::EndOfStream, newt.handle_connection(&mut reader, &mut writer)); + assert_eq!( + ConnectionEnd::EndOfStream, + newt.handle_connection(&mut reader, &mut writer) + ); let x = mocked_stream.pop_bytes_written(); assert_eq!(x, b"0.500000\n"); mocked_stream.push_bytes_to_read(b"1 |A 0 |A 0"); - assert_eq!(ConnectionEnd::EndOfStream, newt.handle_connection(&mut reader, &mut writer)); + assert_eq!( + ConnectionEnd::EndOfStream, + newt.handle_connection(&mut reader, &mut writer) + ); let x = mocked_stream.pop_bytes_written(); assert_eq!(x, b"0.500000\n"); - mocked_stream.push_bytes_to_read(b"! exclamation mark is not a valid label"); - assert_eq!(ConnectionEnd::ParseError, newt.handle_connection(&mut reader, &mut writer)); + assert_eq!( + ConnectionEnd::ParseError, + newt.handle_connection(&mut reader, &mut writer) + ); let x = mocked_stream.pop_bytes_written(); assert_eq!(&x[..] == &b"ERR: Cannot parse an example\n"[..], true); - } - + } + // Non Working stream test - + { let mut mocked_stream_ok = SharedMockStream::new(); let mocked_stream_error = FailingMockStream::new(ErrorKind::Other, "Failing", 3); @@ -318,33 +348,36 @@ C,featureC let mut writer = BufWriter::new(mocked_stream_error.clone()); mocked_stream_ok.push_bytes_to_read(b"|A 0 |A 0"); // Now there will be an error - assert_eq!(ConnectionEnd::StreamFlushError, newt.handle_connection(&mut reader, &mut writer)); + assert_eq!( + ConnectionEnd::StreamFlushError, + newt.handle_connection(&mut reader, &mut writer) + ); let mut reader = BufReader::new(mocked_stream_error.clone()); let mut writer = BufWriter::new(mocked_stream_error.clone()); // Now there will be an error - assert_eq!(ConnectionEnd::StreamFlushError, newt.handle_connection(&mut reader, &mut writer)); - - + assert_eq!( + ConnectionEnd::StreamFlushError, + newt.handle_connection(&mut reader, &mut writer) + ); } - // println!("Return value {:?}", std::str::from_utf8(&x).unwrap()); - - - - + // println!("Return value {:?}", std::str::from_utf8(&x).unwrap()); } - fn lr_and_ffm_vec(v1:Vec, v2:Vec, ffm_fields_count:u32) -> feature_buffer::FeatureBuffer { + fn lr_and_ffm_vec( + v1: Vec, + v2: Vec, + ffm_fields_count: u32, + ) -> feature_buffer::FeatureBuffer { feature_buffer::FeatureBuffer { - label: 0.0, - example_importance: 1.0, - example_number: 0, - lr_buffer: v1, - ffm_buffer: v2, - ffm_fields_count: ffm_fields_count, + label: 0.0, + example_importance: 1.0, + example_number: 0, + lr_buffer: v1, + ffm_buffer: v2, + ffm_fields_count: ffm_fields_count, } } - #[test] fn test_hogwild() { let vw_map_string = r#" @@ -353,7 +386,7 @@ B,featureB C,featureC "#; let vw = vwmap::VwNamespaceMap::new(vw_map_string).unwrap(); - let mut mi = model_instance::ModelInstance::new_empty().unwrap(); + let mut mi = model_instance::ModelInstance::new_empty().unwrap(); mi.learning_rate = 0.1; mi.power_t = 0.0; mi.bit_precision = 18; @@ -361,7 +394,7 @@ C,featureC mi.ffm_bit_precision = 18; mi.ffm_power_t = 0.0; mi.ffm_learning_rate = 0.1; - mi.ffm_fields = vec![vec![],vec![]]; + mi.ffm_fields = vec![vec![], vec![]]; mi.optimizer = model_instance::Optimizer::AdagradLUT; let mut re_1 = regressor::Regressor::new(&mi); mi.optimizer = model_instance::Optimizer::SGD; @@ -369,10 +402,20 @@ C,featureC let mut p: f32; let dir = tempdir().unwrap(); - let regressor_filepath_1 = dir.path().join("test_regressor1.fw").to_str().unwrap().to_owned(); + let regressor_filepath_1 = dir + .path() + .join("test_regressor1.fw") + .to_str() + .unwrap() + .to_owned(); persistence::save_regressor_to_filename(®ressor_filepath_1, &mi, &vw, re_1).unwrap(); - let regressor_filepath_2 = dir.path().join("test_regressor2.fw").to_str().unwrap().to_owned(); + let regressor_filepath_2 = dir + .path() + .join("test_regressor2.fw") + .to_str() + .unwrap() + .to_owned(); persistence::save_regressor_to_filename(®ressor_filepath_2, &mi, &vw, re_2).unwrap(); // OK NOW EVERYTHING IS READY... Let's start @@ -384,54 +427,61 @@ C,featureC let pa = parser::VowpalParser::new(&vw); let pb = re_fixed.new_portbuffer(); - let mut newt = WorkerThread {id: 1, - fbt: fbt, - pa: pa, - re_fixed: re_fixed, - pb, - }; + let mut newt = WorkerThread { + id: 1, + fbt: fbt, + pa: pa, + re_fixed: re_fixed, + pb, + }; - { // WORKING STREAM TEST + { + // WORKING STREAM TEST let mut mocked_stream = SharedMockStream::new(); let mut reader = BufReader::new(mocked_stream.clone()); let mut writer = BufWriter::new(mocked_stream.clone()); // Just passes through, as the stream is empty newt.handle_connection(&mut reader, &mut writer); - // now let's start playing - mocked_stream.push_bytes_to_read(&format!("hogwild_load {}", ®ressor_filepath_1).as_bytes()); - assert_eq!(ConnectionEnd::EndOfStream, newt.handle_connection(&mut reader, &mut writer)); + mocked_stream + .push_bytes_to_read(&format!("hogwild_load {}", ®ressor_filepath_1).as_bytes()); + assert_eq!( + ConnectionEnd::EndOfStream, + newt.handle_connection(&mut reader, &mut writer) + ); let x = mocked_stream.pop_bytes_written(); - assert_eq!(str::from_utf8(&x), str::from_utf8(b"hogwild_load success\n")); + assert_eq!( + str::from_utf8(&x), + str::from_utf8(b"hogwild_load success\n") + ); // now incompatible regressor - should return error - mocked_stream.push_bytes_to_read(&format!("hogwild_load {}", ®ressor_filepath_2).as_bytes()); - assert_eq!(ConnectionEnd::EndOfStream, newt.handle_connection(&mut reader, &mut writer)); + mocked_stream + .push_bytes_to_read(&format!("hogwild_load {}", ®ressor_filepath_2).as_bytes()); + assert_eq!( + ConnectionEnd::EndOfStream, + newt.handle_connection(&mut reader, &mut writer) + ); let x = mocked_stream.pop_bytes_written(); - assert_eq!(str::from_utf8(&x), str::from_utf8(b"hogwild_load success\n")); -/* - - assert_eq!(ConnectionEnd::StreamWriteError, newt.handle_connection(&mut reader, &mut writer)); - let x = mocked_stream.pop_bytes_written(); - assert_eq!(str::from_utf8(&x), str::from_utf8(b"")); -*/ + assert_eq!( + str::from_utf8(&x), + str::from_utf8(b"hogwild_load success\n") + ); + /* + + assert_eq!(ConnectionEnd::StreamWriteError, newt.handle_connection(&mut reader, &mut writer)); + let x = mocked_stream.pop_bytes_written(); + assert_eq!(str::from_utf8(&x), str::from_utf8(b"")); + */ // file does not exist mocked_stream.push_bytes_to_read("hogwild_load /fba/baba/ba".as_bytes()); - assert_eq!(ConnectionEnd::StreamWriteError, newt.handle_connection(&mut reader, &mut writer)); + assert_eq!( + ConnectionEnd::StreamWriteError, + newt.handle_connection(&mut reader, &mut writer) + ); let x = mocked_stream.pop_bytes_written(); assert_eq!(str::from_utf8(&x), str::from_utf8(b"")); - - } - - - + } } - - - - - - } - diff --git a/src/vwmap.rs b/src/vwmap.rs index 32d718c9..93d37ffc 100644 --- a/src/vwmap.rs +++ b/src/vwmap.rs @@ -1,11 +1,11 @@ +use serde::{Deserialize, Serialize}; use std::collections::HashMap; use std::error::Error; -use std::path::PathBuf; -use std::io::prelude::*; use std::fs; -use serde::{Serialize,Deserialize}; -use std::io::ErrorKind; +use std::io::prelude::*; use std::io::Error as IOError; +use std::io::ErrorKind; +use std::path::PathBuf; #[derive(Clone, Copy, Debug, PartialEq, Serialize, Deserialize, Eq)] pub enum NamespaceType { @@ -16,10 +16,9 @@ pub enum NamespaceType { #[derive(Clone, Copy, Debug, PartialEq, Serialize, Deserialize, Eq)] pub enum NamespaceFormat { Categorical = 0, // categorical (binary) features encoding (we have the hash and weight of each feature, value of the feature is assumed to be 1.0 (binary)) - F32 = 1, // f32 features encoding (we have the hash and value of each feature, weight is assumed to be 1.0) + F32 = 1, // f32 features encoding (we have the hash and value of each feature, weight is assumed to be 1.0) } - #[derive(Clone, Debug, Serialize, Deserialize, PartialEq, Copy)] pub struct NamespaceDescriptor { pub namespace_index: u16, @@ -27,14 +26,13 @@ pub struct NamespaceDescriptor { pub namespace_format: NamespaceFormat, } - #[derive(Clone, Debug)] pub struct VwNamespaceMap { pub num_namespaces: usize, - pub map_verbose_to_namespace_descriptor: HashMap , - pub map_vwname_to_namespace_descriptor: HashMap , NamespaceDescriptor>, - pub map_vwname_to_name: HashMap , std::string::String>, - pub vw_source: VwNamespaceMapSource, // this is the source from which VwNamespaceMap can be constructed - for persistence + pub map_verbose_to_namespace_descriptor: HashMap, + pub map_vwname_to_namespace_descriptor: HashMap, NamespaceDescriptor>, + pub map_vwname_to_name: HashMap, std::string::String>, + pub vw_source: VwNamespaceMapSource, // this is the source from which VwNamespaceMap can be constructed - for persistence } // this is serializible source from which VwNamespaceMap can be constructed @@ -43,7 +41,7 @@ pub struct VwNamespaceMapEntry { pub namespace_vwname: std::string::String, namespace_verbose: std::string::String, namespace_index: u16, - namespace_format: NamespaceFormat, + namespace_format: NamespaceFormat, } #[derive(Serialize, Deserialize, Debug, Eq, PartialEq, Clone)] @@ -53,40 +51,51 @@ pub struct VwNamespaceMapSource { } impl VwNamespaceMap { - pub fn new_from_source(vw_source: VwNamespaceMapSource) -> Result> { + pub fn new_from_source( + vw_source: VwNamespaceMapSource, + ) -> Result> { let mut vw = VwNamespaceMap { - num_namespaces:0, - map_verbose_to_namespace_descriptor:HashMap::new(), - map_vwname_to_namespace_descriptor: HashMap::new(), - map_vwname_to_name: HashMap::new(), - vw_source: vw_source, - }; + num_namespaces: 0, + map_verbose_to_namespace_descriptor: HashMap::new(), + map_vwname_to_namespace_descriptor: HashMap::new(), + map_vwname_to_name: HashMap::new(), + vw_source: vw_source, + }; for vw_entry in &vw.vw_source.entries { //let record = result?; let name_str = &vw_entry.namespace_verbose; let vwname_str = &vw_entry.namespace_vwname; - - let namespace_descriptor = NamespaceDescriptor{ - namespace_index: vw_entry.namespace_index, - namespace_type: NamespaceType::Primitive, - namespace_format: vw_entry.namespace_format, - }; - - vw.map_vwname_to_name.insert(vwname_str.as_bytes().to_vec(), String::from(name_str)); - vw.map_vwname_to_namespace_descriptor.insert(vwname_str.as_bytes().to_vec(), namespace_descriptor.clone()); - vw.map_verbose_to_namespace_descriptor.insert(String::from(name_str), namespace_descriptor.clone()); + + let namespace_descriptor = NamespaceDescriptor { + namespace_index: vw_entry.namespace_index, + namespace_type: NamespaceType::Primitive, + namespace_format: vw_entry.namespace_format, + }; + + vw.map_vwname_to_name + .insert(vwname_str.as_bytes().to_vec(), String::from(name_str)); + vw.map_vwname_to_namespace_descriptor + .insert(vwname_str.as_bytes().to_vec(), namespace_descriptor.clone()); + vw.map_verbose_to_namespace_descriptor + .insert(String::from(name_str), namespace_descriptor.clone()); if vw_entry.namespace_index as usize > vw.num_namespaces { vw.num_namespaces = vw_entry.namespace_index as usize; - } + } } vw.num_namespaces += 1; Ok(vw) } pub fn new_from_csv_filepath(path: PathBuf) -> Result> { - let mut input_bufreader = fs::File::open(&path).expect(&format!("Could not find vw_namespace_map.csv in input dataset directory of {:?}", path).to_string()); + let mut input_bufreader = fs::File::open(&path).expect( + &format!( + "Could not find vw_namespace_map.csv in input dataset directory of {:?}", + path + ) + .to_string(), + ); let mut s = String::new(); input_bufreader.read_to_string(&mut s)?; VwNamespaceMap::new(&s) @@ -96,23 +105,30 @@ impl VwNamespaceMap { let mut rdr = csv::ReaderBuilder::new() .has_headers(false) .flexible(true) - .from_reader(data.as_bytes() - ); - let mut vw_source = VwNamespaceMapSource { entries: vec![], namespace_skip_prefix: 0}; + .from_reader(data.as_bytes()); + let mut vw_source = VwNamespaceMapSource { + entries: vec![], + namespace_skip_prefix: 0, + }; for (i, record_w) in rdr.records().enumerate() { let record = record_w?; let vwname_str = &record[0]; - if vwname_str.as_bytes().len() != 1 && i ==0 { + if vwname_str.as_bytes().len() != 1 && i == 0 { println!("Warning: multi-byte namespace names are not compatible with old style namespace arguments"); } - + if vwname_str == "_namespace_skip_prefix" { - let namespace_skip_prefix = record[1].parse().expect("Couldn't parse _namespace_skip_prefix in vw_namespaces_map.csv"); - println!("_namespace_skip_prefix set in vw_namespace_map.csv is {}", namespace_skip_prefix); + let namespace_skip_prefix = record[1] + .parse() + .expect("Couldn't parse _namespace_skip_prefix in vw_namespaces_map.csv"); + println!( + "_namespace_skip_prefix set in vw_namespace_map.csv is {}", + namespace_skip_prefix + ); vw_source.namespace_skip_prefix = namespace_skip_prefix; - continue + continue; } - + let name_str = &record[1]; let namespace_format = match &record.get(2) { Some("f32") => NamespaceFormat::F32, @@ -120,7 +136,7 @@ impl VwNamespaceMap { None => NamespaceFormat::Categorical, Some(unknown_type) => return Err(Box::new(IOError::new(ErrorKind::Other, format!("Unknown type used for the feature in vw_namespace_map.csv: \"{}\". Only \"f32\" is possible.", unknown_type)))) }; - + vw_source.entries.push(VwNamespaceMapEntry { namespace_vwname: vwname_str.to_string(), namespace_verbose: name_str.to_string(), @@ -131,10 +147,8 @@ impl VwNamespaceMap { VwNamespaceMap::new_from_source(vw_source) } - } - #[cfg(test)] mod tests { // Note this useful idiom: importing names from outer (for mod tests) scope. @@ -150,40 +164,51 @@ C,featureC let vw = VwNamespaceMap::new(vw_map_string).unwrap(); assert_eq!(vw.vw_source.entries.len(), 3); assert_eq!(vw.vw_source.namespace_skip_prefix, 0); - assert_eq!(vw.vw_source.entries[0], + assert_eq!( + vw.vw_source.entries[0], VwNamespaceMapEntry { namespace_vwname: "A".to_string(), namespace_verbose: "featureA".to_string(), namespace_index: 0, - namespace_format: NamespaceFormat::Categorical}); + namespace_format: NamespaceFormat::Categorical + } + ); - assert_eq!(vw.vw_source.entries[1], + assert_eq!( + vw.vw_source.entries[1], VwNamespaceMapEntry { namespace_vwname: "B".to_string(), namespace_verbose: "featureB".to_string(), namespace_index: 1, - namespace_format: NamespaceFormat::Categorical}); + namespace_format: NamespaceFormat::Categorical + } + ); - assert_eq!(vw.vw_source.entries[2], + assert_eq!( + vw.vw_source.entries[2], VwNamespaceMapEntry { namespace_vwname: "C".to_string(), namespace_verbose: "featureC".to_string(), namespace_index: 2, - namespace_format: NamespaceFormat::Categorical}); + namespace_format: NamespaceFormat::Categorical + } + ); } - #[test] fn test_f32() { { let vw_map_string = "A,featureA,f32\n_namespace_skip_prefix,2"; let vw = VwNamespaceMap::new(vw_map_string).unwrap(); - assert_eq!(vw.vw_source.entries[0], + assert_eq!( + vw.vw_source.entries[0], VwNamespaceMapEntry { namespace_vwname: "A".to_string(), namespace_verbose: "featureA".to_string(), namespace_index: 0, - namespace_format: NamespaceFormat::F32}); + namespace_format: NamespaceFormat::F32 + } + ); assert_eq!(vw.vw_source.namespace_skip_prefix, 2); } { @@ -193,19 +218,4 @@ C,featureC assert_eq!(format!("{:?}", result), "Err(Custom { kind: Other, error: \"Unknown type used for the feature in vw_namespace_map.csv: \\\"blah\\\". Only \\\"f32\\\" is possible.\" })"); } } - - - - } - - - - - - - - - - - From 138b04035935b821a128127c4143710c36e04cc4 Mon Sep 17 00:00:00 2001 From: bskrlj Date: Fri, 6 Jan 2023 11:40:23 +0100 Subject: [PATCH 2/2] Some terminal whitespaces --- src/block_ffm.rs | 2 +- src/cmdline.rs | 560 +++++++++++++++++++++--------------------- src/feature_buffer.rs | 18 +- 3 files changed, 290 insertions(+), 290 deletions(-) diff --git a/src/block_ffm.rs b/src/block_ffm.rs index 509cc9b6..5f03ce0e 100644 --- a/src/block_ffm.rs +++ b/src/block_ffm.rs @@ -37,7 +37,7 @@ pub struct BlockFFM { } macro_rules! specialize_1f32 { - ( $input_expr:expr, + ( $input_expr:expr, $output_const:ident, $code_block:block ) => { if $input_expr == 1.0 { diff --git a/src/cmdline.rs b/src/cmdline.rs index 3b3873cb..6add2f90 100644 --- a/src/cmdline.rs +++ b/src/cmdline.rs @@ -8,295 +8,295 @@ pub fn parse<'a>() -> clap::ArgMatches<'a> { pub fn create_expected_args<'a>() -> App<'a, 'a> { App::new("fwumious wabbit") - .version(version::LATEST) - .author("Andraz Tori ") - .about("Superfast Logistic Regression & Field Aware Factorization Machines") - .setting(AppSettings::DeriveDisplayOrder) - .arg(Arg::with_name("data") - .long("data") - .short("d") - .value_name("filename") - .help("File with input examples") - .takes_value(true)) - .arg(Arg::with_name("quiet") - .long("quiet") - .help("Quiet mode, does nothing currently (as we don't output diagnostic data anyway)") - .takes_value(false)) - .arg(Arg::with_name("predictions") - .short("p") - .value_name("output predictions file") - .help("Output predictions file") - .takes_value(true)) - .arg(Arg::with_name("cache") - .short("c") - .long("cache") - .help("Use cache file") - .takes_value(false)) - .arg(Arg::with_name("save_resume") - .long("save_resume") - .help("save extra state so learning can be resumed later with new data") - .takes_value(false)) - .arg(Arg::with_name("interactions") - .long("interactions") - .value_name("namespace_char,namespace_char[:value]") - .help("Adds interactions") - .multiple(true) - .takes_value(true)) - .arg(Arg::with_name("linear") - .long("linear") - .value_name("verbose_namespace,verbose_namespace[:value]") - .help("Adds linear feature term with optional value") - .multiple(true) - .takes_value(true)) - .arg(Arg::with_name("keep") - .long("keep") - .value_name("namespace") - .help("Adds single features") - .multiple(true) - .takes_value(true)) - .arg(Arg::with_name("build_cache_only") - .long("build_cache_without_training") - .value_name("arg") - .help("Build cache file without training the first model instance") - .takes_value(false)) + .version(version::LATEST) + .author("Andraz Tori ") + .about("Superfast Logistic Regression & Field Aware Factorization Machines") + .setting(AppSettings::DeriveDisplayOrder) + .arg(Arg::with_name("data") + .long("data") + .short("d") + .value_name("filename") + .help("File with input examples") + .takes_value(true)) + .arg(Arg::with_name("quiet") + .long("quiet") + .help("Quiet mode, does nothing currently (as we don't output diagnostic data anyway)") + .takes_value(false)) + .arg(Arg::with_name("predictions") + .short("p") + .value_name("output predictions file") + .help("Output predictions file") + .takes_value(true)) + .arg(Arg::with_name("cache") + .short("c") + .long("cache") + .help("Use cache file") + .takes_value(false)) + .arg(Arg::with_name("save_resume") + .long("save_resume") + .help("save extra state so learning can be resumed later with new data") + .takes_value(false)) + .arg(Arg::with_name("interactions") + .long("interactions") + .value_name("namespace_char,namespace_char[:value]") + .help("Adds interactions") + .multiple(true) + .takes_value(true)) + .arg(Arg::with_name("linear") + .long("linear") + .value_name("verbose_namespace,verbose_namespace[:value]") + .help("Adds linear feature term with optional value") + .multiple(true) + .takes_value(true)) + .arg(Arg::with_name("keep") + .long("keep") + .value_name("namespace") + .help("Adds single features") + .multiple(true) + .takes_value(true)) + .arg(Arg::with_name("build_cache_only") + .long("build_cache_without_training") + .value_name("arg") + .help("Build cache file without training the first model instance") + .takes_value(false)) - .arg(Arg::with_name("learning_rate") - .short("l") - .long("learning_rate") - .value_name("0.5") - .help("Learning rate") - .takes_value(true)) - .arg(Arg::with_name("ffm_learning_rate") - .long("ffm_learning_rate") - .value_name("0.5") - .help("Learning rate") - .takes_value(true)) - .arg(Arg::with_name("nn_learning_rate") - .long("nn_learning_rate") - .value_name("0.5") - .help("Learning rate") - .takes_value(true)) + .arg(Arg::with_name("learning_rate") + .short("l") + .long("learning_rate") + .value_name("0.5") + .help("Learning rate") + .takes_value(true)) + .arg(Arg::with_name("ffm_learning_rate") + .long("ffm_learning_rate") + .value_name("0.5") + .help("Learning rate") + .takes_value(true)) + .arg(Arg::with_name("nn_learning_rate") + .long("nn_learning_rate") + .value_name("0.5") + .help("Learning rate") + .takes_value(true)) - .arg(Arg::with_name("minimum_learning_rate") - .long("minimum_learning_rate") - .value_name("0.0") - .help("Minimum learning rate (in adaptive algos)") - .takes_value(true)) - .arg(Arg::with_name("power_t") - .long("power_t") - .value_name("0.5") - .help("How to apply Adagrad (0.5 = sqrt)") - .takes_value(true)) - .arg(Arg::with_name("ffm_power_t") - .long("ffm_power_t") - .value_name("0.5") - .help("How to apply Adagrad (0.5 = sqrt)") - .takes_value(true)) - .arg(Arg::with_name("nn_power_t") - .long("nn_power_t") - .value_name("0.5") - .help("How to apply Adagrad (0.5 = sqrt)") - .takes_value(true)) - .arg(Arg::with_name("l2") - .long("l2") - .value_name("0.0") - .help("Regularization is not supported (only 0.0 will work)") - .takes_value(true)) + .arg(Arg::with_name("minimum_learning_rate") + .long("minimum_learning_rate") + .value_name("0.0") + .help("Minimum learning rate (in adaptive algos)") + .takes_value(true)) + .arg(Arg::with_name("power_t") + .long("power_t") + .value_name("0.5") + .help("How to apply Adagrad (0.5 = sqrt)") + .takes_value(true)) + .arg(Arg::with_name("ffm_power_t") + .long("ffm_power_t") + .value_name("0.5") + .help("How to apply Adagrad (0.5 = sqrt)") + .takes_value(true)) + .arg(Arg::with_name("nn_power_t") + .long("nn_power_t") + .value_name("0.5") + .help("How to apply Adagrad (0.5 = sqrt)") + .takes_value(true)) + .arg(Arg::with_name("l2") + .long("l2") + .value_name("0.0") + .help("Regularization is not supported (only 0.0 will work)") + .takes_value(true)) - .arg(Arg::with_name("sgd") - .long("sgd") - .value_name("") - .help("Disable the Adagrad, normalization and invariant updates") - .takes_value(false)) - .arg(Arg::with_name("adaptive") - .long("adaptive") - .value_name("") - .help("Use Adagrad") - .takes_value(false)) - .arg(Arg::with_name("noconstant") - .long("noconstant") - .value_name("") - .help("No intercept") - .takes_value(false)) - .arg(Arg::with_name("link") - .long("link") - .value_name("logistic") - .help("What link function to use") - .takes_value(true)) - .arg(Arg::with_name("loss_function") - .long("loss_function") - .value_name("logistic") - .help("What loss function to use") - .takes_value(true)) - .arg(Arg::with_name("bit_precision") - .short("b") - .long("bit_precision") - .value_name("18") - .help("Size of the hash space for feature weights") - .takes_value(true)) - .arg(Arg::with_name("hash") - .long("hash") - .value_name("all") - .help("We do not support treating strings as already hashed numbers, so you have to use --hash all") - .takes_value(true)) - - // Regressor - .arg(Arg::with_name("final_regressor") - .short("f") - .long("final_regressor") - .value_name("arg") - .help("Final regressor to save (arg is filename)") - .takes_value(true)) - .arg(Arg::with_name("initial_regressor") - .short("i") - .long("initial_regressor") - .value_name("arg") - .help("Initial regressor(s) to load into memory (arg is filename)") - .takes_value(true)) - .arg(Arg::with_name("testonly") - .short("t") - .long("testonly") - .help("Ignore label information and just test") - .takes_value(false)) - .arg(Arg::with_name("vwcompat") - .long("vwcompat") - .help("vowpal compatibility mode. Uses slow adagrad, emits warnings for non-compatible features") - .multiple(false) - .takes_value(false)) - .arg(Arg::with_name("convert_inference_regressor") - .long("convert_inference_regressor") - .value_name("arg") - .conflicts_with("adaptive") - .help("Inference regressor to save (arg is filename)") - .takes_value(true)) + .arg(Arg::with_name("sgd") + .long("sgd") + .value_name("") + .help("Disable the Adagrad, normalization and invariant updates") + .takes_value(false)) + .arg(Arg::with_name("adaptive") + .long("adaptive") + .value_name("") + .help("Use Adagrad") + .takes_value(false)) + .arg(Arg::with_name("noconstant") + .long("noconstant") + .value_name("") + .help("No intercept") + .takes_value(false)) + .arg(Arg::with_name("link") + .long("link") + .value_name("logistic") + .help("What link function to use") + .takes_value(true)) + .arg(Arg::with_name("loss_function") + .long("loss_function") + .value_name("logistic") + .help("What loss function to use") + .takes_value(true)) + .arg(Arg::with_name("bit_precision") + .short("b") + .long("bit_precision") + .value_name("18") + .help("Size of the hash space for feature weights") + .takes_value(true)) + .arg(Arg::with_name("hash") + .long("hash") + .value_name("all") + .help("We do not support treating strings as already hashed numbers, so you have to use --hash all") + .takes_value(true)) - .arg(Arg::with_name("transform") - .long("transform") - .value_name("target_namespace=func(source_namespaces)(parameters)") - .help("Create new namespace by transforming one or more other namespaces") - .multiple(true) - .takes_value(true)) + // Regressor + .arg(Arg::with_name("final_regressor") + .short("f") + .long("final_regressor") + .value_name("arg") + .help("Final regressor to save (arg is filename)") + .takes_value(true)) + .arg(Arg::with_name("initial_regressor") + .short("i") + .long("initial_regressor") + .value_name("arg") + .help("Initial regressor(s) to load into memory (arg is filename)") + .takes_value(true)) + .arg(Arg::with_name("testonly") + .short("t") + .long("testonly") + .help("Ignore label information and just test") + .takes_value(false)) + .arg(Arg::with_name("vwcompat") + .long("vwcompat") + .help("vowpal compatibility mode. Uses slow adagrad, emits warnings for non-compatible features") + .multiple(false) + .takes_value(false)) + .arg(Arg::with_name("convert_inference_regressor") + .long("convert_inference_regressor") + .value_name("arg") + .conflicts_with("adaptive") + .help("Inference regressor to save (arg is filename)") + .takes_value(true)) - .arg(Arg::with_name("ffm_field") - .long("ffm_field") - .value_name("namespace,namespace,...") - .help("Define a FFM field by listing namespace letters") - .multiple(true) - .takes_value(true)) - .arg(Arg::with_name("ffm_field_verbose") - .long("ffm_field_verbose") - .value_name("namespace_verbose,namespace_verbose,...") - .help("Define a FFM field by listing verbose namespace names") - .multiple(true) - .takes_value(true)) - .arg(Arg::with_name("ffm_k") - .long("ffm_k") - .value_name("k") - .help("Lenght of a vector to use for FFM") - .takes_value(true)) - .arg(Arg::with_name("ffm_bit_precision") - .long("ffm_bit_precision") - .value_name("N") - .help("Bits to use for ffm hash space") - .takes_value(true)) - .arg(Arg::with_name("ffm_k_threshold") - .long("ffm_k_threshold") - .help("A minum gradient on left and right side to increase k") - .multiple(false) - .takes_value(true)) - .arg(Arg::with_name("ffm_init_center") - .long("ffm_init_center") - .help("Center of the initial weights distribution") - .multiple(false) - .takes_value(true)) - .arg(Arg::with_name("ffm_init_width") - .long("ffm_init_width") - .help("Total width of the initial weights distribution") - .multiple(false) - .takes_value(true)) - .arg(Arg::with_name("ffm_init_zero_band") - .long("ffm_init_zero_band") - .help("Percentage of ffm_init_width where init is zero") - .multiple(false) - .takes_value(true)) + .arg(Arg::with_name("transform") + .long("transform") + .value_name("target_namespace=func(source_namespaces)(parameters)") + .help("Create new namespace by transforming one or more other namespaces") + .multiple(true) + .takes_value(true)) - .arg(Arg::with_name("nn_init_acc_gradient") - .long("nn_init_acc_gradient") - .help("Adagrad initial accumulated gradient for nn") - .multiple(false) - .takes_value(true)) - .arg(Arg::with_name("ffm_init_acc_gradient") - .long("ffm_init_acc_gradient") - .help("Adagrad initial accumulated gradient for ffm") - .multiple(false) - .takes_value(true)) - .arg(Arg::with_name("init_acc_gradient") - .long("init_acc_gradient") - .help("Adagrad initial accumulated gradient for ") - .multiple(false) - .takes_value(true)) + .arg(Arg::with_name("ffm_field") + .long("ffm_field") + .value_name("namespace,namespace,...") + .help("Define a FFM field by listing namespace letters") + .multiple(true) + .takes_value(true)) + .arg(Arg::with_name("ffm_field_verbose") + .long("ffm_field_verbose") + .value_name("namespace_verbose,namespace_verbose,...") + .help("Define a FFM field by listing verbose namespace names") + .multiple(true) + .takes_value(true)) + .arg(Arg::with_name("ffm_k") + .long("ffm_k") + .value_name("k") + .help("Lenght of a vector to use for FFM") + .takes_value(true)) + .arg(Arg::with_name("ffm_bit_precision") + .long("ffm_bit_precision") + .value_name("N") + .help("Bits to use for ffm hash space") + .takes_value(true)) + .arg(Arg::with_name("ffm_k_threshold") + .long("ffm_k_threshold") + .help("A minum gradient on left and right side to increase k") + .multiple(false) + .takes_value(true)) + .arg(Arg::with_name("ffm_init_center") + .long("ffm_init_center") + .help("Center of the initial weights distribution") + .multiple(false) + .takes_value(true)) + .arg(Arg::with_name("ffm_init_width") + .long("ffm_init_width") + .help("Total width of the initial weights distribution") + .multiple(false) + .takes_value(true)) + .arg(Arg::with_name("ffm_init_zero_band") + .long("ffm_init_zero_band") + .help("Percentage of ffm_init_width where init is zero") + .multiple(false) + .takes_value(true)) + .arg(Arg::with_name("nn_init_acc_gradient") + .long("nn_init_acc_gradient") + .help("Adagrad initial accumulated gradient for nn") + .multiple(false) + .takes_value(true)) + .arg(Arg::with_name("ffm_init_acc_gradient") + .long("ffm_init_acc_gradient") + .help("Adagrad initial accumulated gradient for ffm") + .multiple(false) + .takes_value(true)) + .arg(Arg::with_name("init_acc_gradient") + .long("init_acc_gradient") + .help("Adagrad initial accumulated gradient for ") + .multiple(false) + .takes_value(true)) - .arg(Arg::with_name("nn_layers") - .long("nn_layers") - .help("Enable deep neural network on top of LR+FFM") - .multiple(false) - .takes_value(true)) - - .arg(Arg::with_name("nn") - .long("nn") - .help("Parameters of layers, for example 1:activation:relu or 2:width:20") - .multiple(true) - .takes_value(true)) - - .arg(Arg::with_name("nn_topology") - .long("nn_topology") - .help("How should connections be organized - possiblities 'one' and 'two'") - .multiple(false) - .takes_value(true)) + .arg(Arg::with_name("nn_layers") + .long("nn_layers") + .help("Enable deep neural network on top of LR+FFM") + .multiple(false) + .takes_value(true)) - // Daemon parameterts - .arg(Arg::with_name("daemon") - .long("daemon") - .help("read data from port 26542") - .takes_value(false)) - .arg(Arg::with_name("ffm_initialization_type") - .long("ffm_initialization_type") - .help("Which weight initialization to consider") - .multiple(false) - .takes_value(true)) - .arg(Arg::with_name("port") - .long("port") - .value_name("arg") - .help("port to listen on") - .takes_value(true)) - .arg(Arg::with_name("num_children") - .long("num_children") - .value_name("arg (=10") - .help("number of children for persistent daemon mode") - .takes_value(true)) - .arg(Arg::with_name("foreground") - .long("foreground") - .help("in daemon mode, do not fork and run and run fw process in the foreground") - .takes_value(false)) - .arg(Arg::with_name("prediction_model_delay") - .conflicts_with("test_only") - .long("prediction_model_delay") - .value_name("examples (0)") - .help("Output predictions with a model that is delayed by a number of examples") - .takes_value(true)) - .arg(Arg::with_name("predictions_after") - .long("predictions_after") - .value_name("examples (=0)") - .help("After how many examples start printing predictions") - .takes_value(true)) - .arg(Arg::with_name("holdout_after") - .conflicts_with("testonly") - .required(false) - .long("holdout_after") - .value_name("examples") - .help("After how many examples stop updating weights") - .takes_value(true)) + .arg(Arg::with_name("nn") + .long("nn") + .help("Parameters of layers, for example 1:activation:relu or 2:width:20") + .multiple(true) + .takes_value(true)) + + .arg(Arg::with_name("nn_topology") + .long("nn_topology") + .help("How should connections be organized - possiblities 'one' and 'two'") + .multiple(false) + .takes_value(true)) + + + // Daemon parameterts + .arg(Arg::with_name("daemon") + .long("daemon") + .help("read data from port 26542") + .takes_value(false)) + .arg(Arg::with_name("ffm_initialization_type") + .long("ffm_initialization_type") + .help("Which weight initialization to consider") + .multiple(false) + .takes_value(true)) + .arg(Arg::with_name("port") + .long("port") + .value_name("arg") + .help("port to listen on") + .takes_value(true)) + .arg(Arg::with_name("num_children") + .long("num_children") + .value_name("arg (=10") + .help("number of children for persistent daemon mode") + .takes_value(true)) + .arg(Arg::with_name("foreground") + .long("foreground") + .help("in daemon mode, do not fork and run and run fw process in the foreground") + .takes_value(false)) + .arg(Arg::with_name("prediction_model_delay") + .conflicts_with("test_only") + .long("prediction_model_delay") + .value_name("examples (0)") + .help("Output predictions with a model that is delayed by a number of examples") + .takes_value(true)) + .arg(Arg::with_name("predictions_after") + .long("predictions_after") + .value_name("examples (=0)") + .help("After how many examples start printing predictions") + .takes_value(true)) + .arg(Arg::with_name("holdout_after") + .conflicts_with("testonly") + .required(false) + .long("holdout_after") + .value_name("examples") + .help("After how many examples stop updating weights") + .takes_value(true)) } diff --git a/src/feature_buffer.rs b/src/feature_buffer.rs index 10b947d6..0f0d353a 100644 --- a/src/feature_buffer.rs +++ b/src/feature_buffer.rs @@ -47,11 +47,11 @@ pub struct FeatureBufferTranslator { // this simplifies a lot of the code, as it is used often #[macro_export] macro_rules! feature_reader { - ( $record_buffer:ident, + ( $record_buffer:ident, $transform_executors:expr, - $namespace_descriptor:expr, - $hash_index:ident, - $hash_value:ident, + $namespace_descriptor:expr, + $hash_index:ident, + $hash_value:ident, $bl:block ) => { if $namespace_descriptor.namespace_type == NamespaceType::Transformed { // This is super-unoptimized @@ -110,11 +110,11 @@ macro_rules! feature_reader { #[macro_export] macro_rules! feature_reader_float_namespace { - ( $record_buffer:ident, - $namespace_descriptor:expr, - $hash_index:ident, - $hash_value:ident, - $float_value:ident, + ( $record_buffer:ident, + $namespace_descriptor:expr, + $hash_index:ident, + $hash_value:ident, + $float_value:ident, $bl:block ) => { let namespace_index = $namespace_descriptor.namespace_index as usize; let first_token =