Can't launch reachy stack

[cdussieux]

Observed this issue 3 times on 2 robots, full physical reboot always solved it so far
Maybe buffer is not clean on device side on boot ? Sometimes ?

log is in reverse order

unch]: Default logging verbosity is set to INFO
unch]: All log files can be found below /home/reachy/.ros/log/2023-04-11-13-59-53-788448-reachy-2023-0005-1931

os2_control_node-1]: process has died [pid 1925, exit code -6, cmd '/opt/ros/humble/lib/controller_manager/ros2_control_node --ros-args >
rol_node-1] Aborted (Signal sent by tkill() 1925 1000)
rol_node-1]       Source "./nptl/pthread_kill.c", line 44, in __pthread_kill [0x7faabc496a7c]
rol_node-1]     | Source "./nptl/pthread_kill.c", line 78, in __pthread_kill_implementation
rol_node-1] #0  | Source "./nptl/pthread_kill.c", line 89, in __pthread_kill_internal
rol_node-1] #1    Source "../sysdeps/posix/raise.c", line 26, in raise [0x7faabc442475]
rol_node-1] #2    Source "./stdlib/abort.c", line 79, in abort [0x7faabc4287f2]
rol_node-1] #3    Source "library/std/src/sys/unix/mod.rs", line 350, in abort_internal [0x7faab6392a16]
rol_node-1] #4    Source "library/std/src/panicking.rs", line 742, in rust_panic [0x7faab6387951]
rol_node-1] #5    Source "library/std/src/panicking.rs", line 710, in rust_panic_with_hook [0x7faab6387746]
rol_node-1] #6    Source "library/std/src/panicking.rs", line 579, in {closure#0} [0x7faab6387488]
rol_node-1] #7    Source "library/std/src/sys_common/backtrace.rs", line 137, in __rust_end_short_backtrace<std::panicking::begin_panic_>
rol_node-1] #8    Source "library/std/src/panicking.rs", line 575, in begin_panic_handler [0x7faab6387191]
rol_node-1] #9    Source "library/core/src/panicking.rs", line 64, in panic_fmt [0x7faab61fd5b2]
rol_node-1] #10   Source "library/core/src/result.rs", line 1790, in unwrap_failed [0x7faab61fda62]
rol_node-1] #11   Source "/rustc/9eb3afe9ebe9c7d2b84b71002d44f4a0edac95e0/library/core/src/result.rs", line 1112, in unwrap<core::time::>
rol_node-1]         145:     }
rol_node-1]         144:         }
rol_node-1]         143:             None => false,
rol_node-1]       > 142:             Some(expiry) => t.elapsed().unwrap() > expiry,
rol_node-1]         141:         match expiry_duration {
rol_node-1]         140:     fn has_expired(expiry_duration: Option<Duration>, t: &SystemTime) -> bool {
rol_node-1] #12   Source "/home/reachy/.cargo/registry/src/github.com-1ecc6299db9ec823/cache_cache-0.1.0/src/lib.rs", line 142, in has_e>
rol_node-1]         166:             },
rol_node-1]         165:                 false => Some(v),
rol_node-1]         164:                 true => None,
rol_node-1]       > 163:             Some((v, t)) => match Self::has_expired(self.expiry_duration, t) {
rol_node-1]         162:         match self.hash_map.get(k) {
rol_node-1]         161:     {
rol_node-1]         160:         Q: Hash + Eq,
rol_node-1] #13   Source "/home/reachy/.cargo/registry/src/github.com-1ecc6299db9ec823/cache_cache-0.1.0/src/lib.rs", line 163, in get<u>
rol_node-1]         105:                 k: key,
rol_node-1]         104:             None => Entry::Vacant(VacantEntry {
rol_node-1]         103:             Some(&v) => Entry::Occupied(OccupiedEntry { k: key, v }),
rol_node-1]       > 102:         match self.get(&key) {
rol_node-1]         101:     pub fn entry(&mut self, key: K) -> Entry<'_, K, V> {
rol_node-1]         100:     /// assert_eq!(motors_temperature.get(&11), Some(&42.0));
rol_node-1]          99:     /// let temp = motors_temperature.entry(11).or_insert_with(get_motor_temperature);
rol_node-1] #14   Source "/home/reachy/.cargo/registry/src/github.com-1ecc6299db9ec823/cache_cache-0.1.0/src/lib.rs", line 102, in entry>
rol_node-1] #15   Source "src/lib.rs", line 376, in read_force_sensor [0x7faab62543f5]
rol_node-1] #16   Source "src/lib.rs", line 324, in arm_hwi_read_force_sensor [0x7faab6219b0b]
rol_node-1] #17   Object "/home/reachy/reachy_ws/build/arm_system_hwi/libarm_system_hwi.so", at 0x7faab6203e25, in arm_system_hwi::ArmSy>
rol_node-1] #18   Object "/opt/ros/humble/lib/libhardware_interface.so", at 0x7faabc7e77c4, in hardware_interface::System::read(rclcpp::>
rol_node-1] #19   Object "/opt/ros/humble/lib/libhardware_interface.so", at 0x7faabc7c0401, in hardware_interface::ResourceManager::read>
rol_node-1] #20   Object "/opt/ros/humble/lib/controller_manager/ros2_control_node", at 0x565027464810, in
rol_node-1] #21   Object "/usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.30", at 0x7faabc8dc2b2, in
rol_node-1] #22   Source "./nptl/pthread_create.c", line 442, in start_thread [0x7faabc494b42]
rol_node-1] #23   Source "../sysdeps/unix/sysv/linux/x86_64/clone3.S", line 81, in __clone3 [0x7faabc5269ff]
rol_node-1] #24   Object "", at 0xffffffffffffffff, in
rol_node-1] Stack trace (most recent call last) in thread 1959:
rol_node-1] fatal runtime error: failed to initiate panic, error 5
rol_node-1] note: run with `RUST_BACKTRACE=1` environment variable to display a backtrace
rol_node-1] thread '<unnamed>' panicked at 'called `Result::unwrap()` on an `Err` value: SystemTimeError(384.821411ms)', /home/reachy/.c>
te_publisher-2] [WARN] [1681213958.100396476] [robot_state_publisher]: Moved backwards in time, re-publishing joint transforms!
cus-6] [INFO] [1681213958.143409368] [camera_focus]: Autofocus node for left_eye ready!