diff --git a/esp-hal/src/dma/gdma.rs b/esp-hal/src/dma/gdma.rs index 21459966429..8c2ca9a2468 100644 --- a/esp-hal/src/dma/gdma.rs +++ b/esp-hal/src/dma/gdma.rs @@ -121,6 +121,14 @@ use embassy_sync::waitqueue::AtomicWaker; static TX_WAKERS: [AtomicWaker; CHANNEL_COUNT] = [const { AtomicWaker::new() }; CHANNEL_COUNT]; static RX_WAKERS: [AtomicWaker; CHANNEL_COUNT] = [const { AtomicWaker::new() }; CHANNEL_COUNT]; +cfg_if::cfg_if! { + if #[cfg(any(esp32c2, esp32c3))] { + use portable_atomic::AtomicBool; + static TX_IS_ASYNC: [AtomicBool; CHANNEL_COUNT] = [const { AtomicBool::new(false) }; CHANNEL_COUNT]; + static RX_IS_ASYNC: [AtomicBool; CHANNEL_COUNT] = [const { AtomicBool::new(false) }; CHANNEL_COUNT]; + } +} + impl crate::private::Sealed for ChannelTxImpl {} impl ChannelTxImpl { @@ -314,6 +322,24 @@ impl InterruptAccess for ChannelTxImpl { fn waker(&self) -> &'static AtomicWaker { &TX_WAKERS[self.0.number() as usize] } + + fn is_async(&self) -> bool { + cfg_if::cfg_if! { + if #[cfg(any(esp32c2, esp32c3))] { + TX_IS_ASYNC[self.0.number() as usize].load(portable_atomic::Ordering::Acquire) + } else { + true + } + } + } + + fn set_async(&self, _is_async: bool) { + cfg_if::cfg_if! { + if #[cfg(any(esp32c2, esp32c3))] { + TX_IS_ASYNC[self.0.number() as usize].store(_is_async, portable_atomic::Ordering::Release); + } + } + } } #[non_exhaustive] @@ -513,6 +539,24 @@ impl InterruptAccess for ChannelRxImpl { fn waker(&self) -> &'static AtomicWaker { &RX_WAKERS[self.0.number() as usize] } + + fn is_async(&self) -> bool { + cfg_if::cfg_if! { + if #[cfg(any(esp32c2, esp32c3))] { + RX_IS_ASYNC[self.0.number() as usize].load(portable_atomic::Ordering::Acquire) + } else { + true + } + } + } + + fn set_async(&self, _is_async: bool) { + cfg_if::cfg_if! { + if #[cfg(any(esp32c2, esp32c3))] { + RX_IS_ASYNC[self.0.number() as usize].store(_is_async, portable_atomic::Ordering::Release); + } + } + } } /// A Channel can be created from this @@ -611,16 +655,16 @@ cfg_if::cfg_if! { impl_channel!(2, DMA_CH2, asynch_handler::interrupt_handler_ch2); } else if #[cfg(any(esp32c6, esp32h2))] { const CHANNEL_COUNT: usize = 3; - impl_channel!(0, DMA_IN_CH0, asynch_handler::interrupt_handler_ch0, DMA_OUT_CH0, asynch_handler::interrupt_handler_ch0); - impl_channel!(1, DMA_IN_CH1, asynch_handler::interrupt_handler_ch1, DMA_OUT_CH1, asynch_handler::interrupt_handler_ch1); - impl_channel!(2, DMA_IN_CH2, asynch_handler::interrupt_handler_ch2, DMA_OUT_CH2, asynch_handler::interrupt_handler_ch2); + impl_channel!(0, DMA_IN_CH0, asynch_handler::interrupt_handler_in_ch0, DMA_OUT_CH0, asynch_handler::interrupt_handler_out_ch0); + impl_channel!(1, DMA_IN_CH1, asynch_handler::interrupt_handler_in_ch1, DMA_OUT_CH1, asynch_handler::interrupt_handler_out_ch1); + impl_channel!(2, DMA_IN_CH2, asynch_handler::interrupt_handler_in_ch2, DMA_OUT_CH2, asynch_handler::interrupt_handler_out_ch2); } else if #[cfg(esp32s3)] { const CHANNEL_COUNT: usize = 5; - impl_channel!(0, DMA_IN_CH0, asynch_handler::interrupt_handler_ch0, DMA_OUT_CH0, asynch_handler::interrupt_handler_ch0); - impl_channel!(1, DMA_IN_CH1, asynch_handler::interrupt_handler_ch1, DMA_OUT_CH1, asynch_handler::interrupt_handler_ch1); - impl_channel!(2, DMA_IN_CH2, asynch_handler::interrupt_handler_ch2, DMA_OUT_CH2, asynch_handler::interrupt_handler_ch2); - impl_channel!(3, DMA_IN_CH3, asynch_handler::interrupt_handler_ch3, DMA_OUT_CH3, asynch_handler::interrupt_handler_ch3); - impl_channel!(4, DMA_IN_CH4, asynch_handler::interrupt_handler_ch4, DMA_OUT_CH4, asynch_handler::interrupt_handler_ch4); + impl_channel!(0, DMA_IN_CH0, asynch_handler::interrupt_handler_in_ch0, DMA_OUT_CH0, asynch_handler::interrupt_handler_out_ch0); + impl_channel!(1, DMA_IN_CH1, asynch_handler::interrupt_handler_in_ch1, DMA_OUT_CH1, asynch_handler::interrupt_handler_out_ch1); + impl_channel!(2, DMA_IN_CH2, asynch_handler::interrupt_handler_in_ch2, DMA_OUT_CH2, asynch_handler::interrupt_handler_out_ch2); + impl_channel!(3, DMA_IN_CH3, asynch_handler::interrupt_handler_in_ch3, DMA_OUT_CH3, asynch_handler::interrupt_handler_out_ch3); + impl_channel!(4, DMA_IN_CH4, asynch_handler::interrupt_handler_in_ch4, DMA_OUT_CH4, asynch_handler::interrupt_handler_out_ch4); } } diff --git a/esp-hal/src/dma/mod.rs b/esp-hal/src/dma/mod.rs index e5bee324491..d52a121fa51 100644 --- a/esp-hal/src/dma/mod.rs +++ b/esp-hal/src/dma/mod.rs @@ -1679,6 +1679,13 @@ where // channel was previously used for a mem2mem transfer. rx_impl.set_mem2mem_mode(false); + if let Some(interrupt) = rx_impl.peripheral_interrupt() { + for cpu in Cpu::all() { + crate::interrupt::disable(cpu, interrupt); + } + } + rx_impl.set_async(false); + Self { burst_mode: false, rx_impl, @@ -1691,6 +1698,7 @@ where if let Some(handler) = self.rx_impl.async_handler() { self.set_interrupt_handler(handler); } + self.rx_impl.set_async(true); ChannelRx { burst_mode: self.burst_mode, rx_impl: self.rx_impl, @@ -1724,6 +1732,7 @@ where if let Some(interrupt) = self.rx_impl.peripheral_interrupt() { crate::interrupt::disable(Cpu::current(), interrupt); } + self.rx_impl.set_async(false); ChannelRx { burst_mode: self.burst_mode, rx_impl: self.rx_impl, @@ -1960,6 +1969,13 @@ where CH: DmaChannel, { fn new(tx_impl: CH::Tx) -> Self { + if let Some(interrupt) = tx_impl.peripheral_interrupt() { + for cpu in Cpu::all() { + crate::interrupt::disable(cpu, interrupt); + } + } + tx_impl.set_async(false); + Self { burst_mode: false, tx_impl, @@ -1972,6 +1988,7 @@ where if let Some(handler) = self.tx_impl.async_handler() { self.set_interrupt_handler(handler); } + self.tx_impl.set_async(true); ChannelTx { burst_mode: self.burst_mode, tx_impl: self.tx_impl, @@ -2005,6 +2022,7 @@ where if let Some(interrupt) = self.tx_impl.peripheral_interrupt() { crate::interrupt::disable(Cpu::current(), interrupt); } + self.tx_impl.set_async(false); ChannelTx { burst_mode: self.burst_mode, tx_impl: self.tx_impl, @@ -2259,6 +2277,9 @@ pub trait InterruptAccess: crate::private::Sealed { fn clear(&self, interrupts: impl Into>); fn pending_interrupts(&self) -> EnumSet; fn waker(&self) -> &'static embassy_sync::waitqueue::AtomicWaker; + + fn is_async(&self) -> bool; + fn set_async(&self, is_async: bool); } /// DMA Channel @@ -2982,9 +3003,12 @@ pub(crate) mod asynch { } } - fn handle_interrupt() { + fn handle_in_interrupt() { let rx = CH::rx_interrupts(); - let tx = CH::tx_interrupts(); + + if !rx.is_async() { + return; + } if rx.pending_interrupts().is_disjoint( DmaRxInterrupt::DescriptorError @@ -3001,16 +3025,6 @@ pub(crate) mod asynch { rx.waker().wake() } - if tx - .pending_interrupts() - .contains(DmaTxInterrupt::DescriptorError) - { - tx.unlisten( - DmaTxInterrupt::DescriptorError | DmaTxInterrupt::TotalEof | DmaTxInterrupt::Done, - ); - tx.waker().wake() - } - if rx .pending_interrupts() .contains(DmaRxInterrupt::SuccessfulEof) @@ -3023,6 +3037,24 @@ pub(crate) mod asynch { rx.unlisten(DmaRxInterrupt::Done); rx.waker().wake() } + } + + fn handle_out_interrupt() { + let tx = CH::tx_interrupts(); + + if !tx.is_async() { + return; + } + + if tx + .pending_interrupts() + .contains(DmaTxInterrupt::DescriptorError) + { + tx.unlisten( + DmaTxInterrupt::DescriptorError | DmaTxInterrupt::TotalEof | DmaTxInterrupt::Done, + ); + tx.waker().wake() + } if tx.pending_interrupts().contains(DmaTxInterrupt::TotalEof) && tx.is_listening().contains(DmaTxInterrupt::TotalEof) @@ -3037,69 +3069,78 @@ pub(crate) mod asynch { } } - #[cfg(not(any(esp32, esp32s2)))] + #[cfg(gdma)] pub(crate) mod interrupt { - use procmacros::handler; - use super::*; - - #[handler(priority = crate::interrupt::Priority::max())] - pub(crate) fn interrupt_handler_ch0() { - handle_interrupt::(); + use crate::{interrupt::Priority, macros::handler}; + + // Single interrupt handler for IN and OUT + #[cfg(any(esp32c2, esp32c3))] + macro_rules! interrupt_handler { + ($ch:literal) => { + paste::paste! { + #[handler(priority = Priority::max())] + pub(crate) fn []() { + handle_in_interrupt::<[< DmaChannel $ch >]>(); + handle_out_interrupt::<[< DmaChannel $ch >]>(); + } + } + }; } - #[cfg(not(esp32c2))] - #[handler(priority = crate::interrupt::Priority::max())] - pub(crate) fn interrupt_handler_ch1() { - handle_interrupt::(); - } + #[cfg(not(any(esp32c2, esp32c3)))] + macro_rules! interrupt_handler { + ($ch:literal) => { + paste::paste! { + #[handler(priority = Priority::max())] + pub(crate) fn []() { + handle_in_interrupt::<[< DmaChannel $ch >]>(); + } - #[cfg(not(esp32c2))] - #[handler(priority = crate::interrupt::Priority::max())] - pub(crate) fn interrupt_handler_ch2() { - handle_interrupt::(); + #[handler(priority = Priority::max())] + pub(crate) fn []() { + handle_out_interrupt::<[< DmaChannel $ch >]>(); + } + } + }; } + interrupt_handler!(0); + #[cfg(not(esp32c2))] + interrupt_handler!(1); + #[cfg(not(esp32c2))] + interrupt_handler!(2); #[cfg(esp32s3)] - #[handler(priority = crate::interrupt::Priority::max())] - pub(crate) fn interrupt_handler_ch3() { - handle_interrupt::(); - } - + interrupt_handler!(3); #[cfg(esp32s3)] - #[handler(priority = crate::interrupt::Priority::max())] - pub(crate) fn interrupt_handler_ch4() { - handle_interrupt::(); - } + interrupt_handler!(4); } - #[cfg(any(esp32, esp32s2))] + #[cfg(pdma)] pub(crate) mod interrupt { - use procmacros::handler; - use super::*; - - #[handler(priority = crate::interrupt::Priority::max())] - pub(crate) fn interrupt_handler_spi2_dma() { - handle_interrupt::(); + use crate::{interrupt::Priority, macros::handler}; + + // Single interrupt handler for IN and OUT + macro_rules! interrupt_handler { + ($ch:ident) => { + paste::paste! { + #[handler(priority = Priority::max())] + pub(crate) fn []() { + handle_in_interrupt::<[< $ch DmaChannel >]>(); + handle_out_interrupt::<[< $ch DmaChannel >]>(); + } + } + }; } + interrupt_handler!(Spi2); #[cfg(spi3)] - #[handler(priority = crate::interrupt::Priority::max())] - pub(crate) fn interrupt_handler_spi3_dma() { - handle_interrupt::(); - } + interrupt_handler!(Spi3); #[cfg(i2s0)] - #[handler(priority = crate::interrupt::Priority::max())] - pub(crate) fn interrupt_handler_i2s0_dma() { - handle_interrupt::(); - } - + interrupt_handler!(I2s0); #[cfg(i2s1)] - #[handler(priority = crate::interrupt::Priority::max())] - pub(crate) fn interrupt_handler_i2s1_dma() { - handle_interrupt::(); - } + interrupt_handler!(I2s1); } } diff --git a/esp-hal/src/dma/pdma.rs b/esp-hal/src/dma/pdma.rs index 776ea57fd07..609c89f24e4 100644 --- a/esp-hal/src/dma/pdma.rs +++ b/esp-hal/src/dma/pdma.rs @@ -12,6 +12,7 @@ //! [I2S]: ../i2s/index.html use embassy_sync::waitqueue::AtomicWaker; +use portable_atomic::{AtomicBool, Ordering}; use crate::{ dma::*, @@ -35,6 +36,8 @@ pub trait PdmaChannel: crate::private::Sealed { fn peripheral_interrupt(&self) -> Interrupt; fn async_handler(&self) -> InterruptHandler; + fn rx_async_flag(&self) -> &'static AtomicBool; + fn tx_async_flag(&self) -> &'static AtomicBool; } #[doc(hidden)] @@ -198,6 +201,14 @@ impl> InterruptAccess &'static AtomicWaker { self.0.tx_waker() } + + fn is_async(&self) -> bool { + self.0.tx_async_flag().load(Ordering::Acquire) + } + + fn set_async(&self, is_async: bool) { + self.0.tx_async_flag().store(is_async, Ordering::Release); + } } impl> RegisterAccess for SpiDmaRxChannelImpl { @@ -348,6 +359,14 @@ impl> InterruptAccess &'static AtomicWaker { self.0.rx_waker() } + + fn is_async(&self) -> bool { + self.0.rx_async_flag().load(Ordering::Relaxed) + } + + fn set_async(&self, _is_async: bool) { + self.0.rx_async_flag().store(_is_async, Ordering::Relaxed); + } } #[doc(hidden)] @@ -382,12 +401,12 @@ macro_rules! ImplSpiChannel { fn register_block(&self) -> &SpiRegisterBlock { unsafe { &*crate::peripherals::[]::PTR } } - fn tx_waker(&self) -> &'static embassy_sync::waitqueue::AtomicWaker { - static WAKER: embassy_sync::waitqueue::AtomicWaker = embassy_sync::waitqueue::AtomicWaker::new(); + fn tx_waker(&self) -> &'static AtomicWaker { + static WAKER: AtomicWaker = AtomicWaker::new(); &WAKER } - fn rx_waker(&self) -> &'static embassy_sync::waitqueue::AtomicWaker { - static WAKER: embassy_sync::waitqueue::AtomicWaker = embassy_sync::waitqueue::AtomicWaker::new(); + fn rx_waker(&self) -> &'static AtomicWaker { + static WAKER: AtomicWaker = AtomicWaker::new(); &WAKER } @@ -402,6 +421,14 @@ macro_rules! ImplSpiChannel { fn async_handler(&self) -> InterruptHandler { super::asynch::interrupt::[< interrupt_handler_spi $num _dma >] } + fn rx_async_flag(&self) -> &'static AtomicBool { + static FLAG: AtomicBool = AtomicBool::new(false); + &FLAG + } + fn tx_async_flag(&self) -> &'static AtomicBool { + static FLAG: AtomicBool = AtomicBool::new(false); + &FLAG + } } impl DmaChannelConvert for [] { @@ -614,6 +641,14 @@ impl> InterruptAccess &'static AtomicWaker { self.0.tx_waker() } + + fn is_async(&self) -> bool { + self.0.tx_async_flag().load(Ordering::Relaxed) + } + + fn set_async(&self, _is_async: bool) { + self.0.tx_async_flag().store(_is_async, Ordering::Relaxed); + } } impl> RegisterAccess for I2sDmaRxChannelImpl { @@ -770,6 +805,14 @@ impl> InterruptAccess &'static AtomicWaker { self.0.rx_waker() } + + fn is_async(&self) -> bool { + self.0.rx_async_flag().load(Ordering::Relaxed) + } + + fn set_async(&self, _is_async: bool) { + self.0.rx_async_flag().store(_is_async, Ordering::Relaxed); + } } macro_rules! ImplI2sChannel { @@ -800,12 +843,12 @@ macro_rules! ImplI2sChannel { fn register_block(&self) -> &I2sRegisterBlock { unsafe { &*crate::peripherals::[< I2S $num >]::PTR } } - fn tx_waker(&self) -> &'static embassy_sync::waitqueue::AtomicWaker { - static WAKER: embassy_sync::waitqueue::AtomicWaker = embassy_sync::waitqueue::AtomicWaker::new(); + fn tx_waker(&self) -> &'static AtomicWaker { + static WAKER: AtomicWaker = AtomicWaker::new(); &WAKER } - fn rx_waker(&self) -> &'static embassy_sync::waitqueue::AtomicWaker { - static WAKER: embassy_sync::waitqueue::AtomicWaker = embassy_sync::waitqueue::AtomicWaker::new(); + fn rx_waker(&self) -> &'static AtomicWaker { + static WAKER: AtomicWaker = AtomicWaker::new(); &WAKER } fn is_compatible_with(&self, peripheral: DmaPeripheral) -> bool { @@ -819,6 +862,14 @@ macro_rules! ImplI2sChannel { fn async_handler(&self) -> InterruptHandler { super::asynch::interrupt::[< interrupt_handler_i2s $num _dma >] } + fn rx_async_flag(&self) -> &'static AtomicBool { + static FLAG: AtomicBool = AtomicBool::new(false); + &FLAG + } + fn tx_async_flag(&self) -> &'static AtomicBool { + static FLAG: AtomicBool = AtomicBool::new(false); + &FLAG + } } impl DmaChannelConvert for [] { @@ -968,6 +1019,8 @@ impl PdmaChannel for AnySpiDmaChannelInner { fn is_compatible_with(&self, peripheral: DmaPeripheral) -> bool; fn peripheral_interrupt(&self) -> Interrupt; fn async_handler(&self) -> InterruptHandler; + fn rx_async_flag(&self) -> &'static AtomicBool; + fn tx_async_flag(&self) -> &'static AtomicBool; } } } @@ -1008,6 +1061,8 @@ impl PdmaChannel for AnyI2sDmaChannelInner { fn is_compatible_with(&self, peripheral: DmaPeripheral) -> bool; fn peripheral_interrupt(&self) -> Interrupt; fn async_handler(&self) -> InterruptHandler; + fn rx_async_flag(&self) -> &'static AtomicBool; + fn tx_async_flag(&self) -> &'static AtomicBool; } } } diff --git a/esp-hal/src/lib.rs b/esp-hal/src/lib.rs index 8e17996ebc8..755f02b1e7b 100644 --- a/esp-hal/src/lib.rs +++ b/esp-hal/src/lib.rs @@ -386,7 +386,7 @@ impl Cpu { /// Returns an iterator over the "other" cores. #[inline(always)] - pub fn other() -> impl Iterator { + pub(crate) fn other() -> impl Iterator { cfg_if::cfg_if! { if #[cfg(multi_core)] { match Self::current() { @@ -398,6 +398,18 @@ impl Cpu { } } } + + /// Returns an iterator over all cores. + #[inline(always)] + pub(crate) fn all() -> impl Iterator { + cfg_if::cfg_if! { + if #[cfg(multi_core)] { + [Cpu::ProCpu, Cpu::AppCpu].into_iter() + } else { + [Cpu::ProCpu].into_iter() + } + } + } } /// Returns the raw value of the mhartid register.