From a723ac1fd67d12b03d39a7fb634e5719af65a83c Mon Sep 17 00:00:00 2001 From: Will Tatam Date: Sun, 29 Oct 2023 14:27:35 +0000 Subject: [PATCH 01/13] WiP on adding ESP-NOW audio sync --- usermods/audioreactive/audio_reactive.h | 59 ++++++++++++++++++++----- 1 file changed, 48 insertions(+), 11 deletions(-) diff --git a/usermods/audioreactive/audio_reactive.h b/usermods/audioreactive/audio_reactive.h index 4726ee26f3..e332e488cf 100644 --- a/usermods/audioreactive/audio_reactive.h +++ b/usermods/audioreactive/audio_reactive.h @@ -92,6 +92,8 @@ static bool udpSyncConnected = false; // UDP connection status -> true i #define NUM_GEQ_CHANNELS 16 // number of frequency channels. Don't change !! +uint8_t broadcastAddress[] = {0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF}; + // audioreactive variables #ifdef ARDUINO_ARCH_ESP32 static float micDataReal = 0.0f; // MicIn data with full 24bit resolution - lowest 8bit after decimal point @@ -1478,13 +1480,27 @@ class AudioReactive : public Usermod { transmitData.FFT_Magnitude = my_magnitude; transmitData.FFT_MajorPeak = FFT_MajorPeak; - if (fftUdp.beginMulticastPacket() != 0) { // beginMulticastPacket returns 0 in case of error - fftUdp.write(reinterpret_cast(&transmitData), sizeof(transmitData)); - fftUdp.endPacket(); + if(audioSyncEnabled == 3) { // Send using ESP-NOW + // Send message via ESP-NOW + Serial.printf("esp_now_send(%u)\n", transmitData.frameCounter); + esp_err_t result = esp_now_send(broadcastAddress, (uint8_t *) &transmitData, sizeof(transmitData)); + if (result == ESP_OK) { + Serial.println("Sent with success"); + } + else { + Serial.println("Error sending the data"); + } + } + else { + if (fftUdp.beginMulticastPacket() != 0) { // beginMulticastPacket returns 0 in case of error + fftUdp.write(reinterpret_cast(&transmitData), sizeof(transmitData)); + fftUdp.endPacket(); + } } frameCounter++; } // transmitAudioData() + #endif static bool isValidUdpSyncVersion(const char *header) { return strncmp_P(header, UDP_SYNC_HEADER, 6) == 0; @@ -1834,7 +1850,11 @@ class AudioReactive : public Usermod { DEBUGSR_PRINTLN(F("AR connected(): old UDP connection closed.")); } - if (audioSyncPort > 0 && (audioSyncEnabled & 0x03)) { + if(audioSyncEnabled == 3) { + DEBUGSR_PRINTLN(F("AR connected(): audioSyncEnabled == 3")); + udpSyncConnected = true; // TODO: better name this flag + } + else if (audioSyncPort > 0 && (audioSyncEnabled & 0x03)) { #ifdef ARDUINO_ARCH_ESP32 udpSyncConnected = fftUdp.beginMulticast(IPAddress(239, 0, 0, 1), audioSyncPort); #else @@ -2235,6 +2255,17 @@ class AudioReactive : public Usermod { infoArr.add(F("sound sync Off")); } #else // ESP32 only + } else if (audioSyncEnabled & 0x04) { + infoArr = user.createNestedArray(F("Audio Source")); + infoArr.add(F("ESP-NOW sound sync")); + // if (udpSyncConnected) { + // if (millis() - last_UDPTime < 2500) + // infoArr.add(F(" - receiving")); + // else + // infoArr.add(F(" - idle")); + // } else { + // infoArr.add(F(" - no connection")); + // } } else { // Analog or I2S digital input if (audioSource && (audioSource->isInitialized())) { @@ -2284,13 +2315,17 @@ class AudioReactive : public Usermod { } #endif // UDP Sound Sync status - infoArr = user.createNestedArray(F("UDP Sound Sync")); + infoArr = user.createNestedArray(F("Sound Sync")); if (audioSyncEnabled) { - if (audioSyncEnabled & 0x01) { - infoArr.add(F("send mode")); + if (audioSyncEnabled == 1) { + infoArr.add(F("UDP send mode")); if ((udpSyncConnected) && (millis() - lastTime < 2500)) infoArr.add(F(" v2")); - } else if (audioSyncEnabled & 0x02) { - infoArr.add(F("receive mode")); + } else if (audioSyncEnabled == 2) { + infoArr.add(F("UDP receive mode")); + } else if (audioSyncEnabled == 3) { + infoArr.add(F("ESP-NOW send mode")); + } else if (audioSyncEnabled == 4) { + infoArr.add(F("ESP-NOW receive mode")); } } else infoArr.add("off"); @@ -2689,9 +2724,11 @@ class AudioReactive : public Usermod { oappend(SET_F("dd=addDropdown('AudioReactive','sync:mode');")); oappend(SET_F("addOption(dd,'Off',0);")); #ifdef ARDUINO_ARCH_ESP32 - oappend(SET_F("addOption(dd,'Send',1);")); + oappend(SET_F("addOption(dd,'Send (UDP)',1);")); + oappend(SET_F("addOption(dd,'Send (ESP-NOW)',3);")); #endif - oappend(SET_F("addOption(dd,'Receive',2);")); + oappend(SET_F("addOption(dd,'Receive (UDP)',2);")); + oappend(SET_F("addOption(dd,'Receive (ESP-NOW)',4);")); oappend(SET_F("addInfo('AudioReactive:sync:mode',1,'
Sync audio data with other WLEDs');")); oappend(SET_F("addInfo('AudioReactive:digitalmic:type',1,'requires reboot!');")); // 0 is field type, 1 is actual field From d9813eaf6b0389c3928a15c953567697cae6dae5 Mon Sep 17 00:00:00 2001 From: Will Tatam Date: Sun, 29 Oct 2023 15:30:37 +0000 Subject: [PATCH 02/13] WiP on adding ESP-NOW audio sync --- usermods/audioreactive/audio_reactive.h | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/usermods/audioreactive/audio_reactive.h b/usermods/audioreactive/audio_reactive.h index e332e488cf..cf82b3e3de 100644 --- a/usermods/audioreactive/audio_reactive.h +++ b/usermods/audioreactive/audio_reactive.h @@ -1482,16 +1482,17 @@ class AudioReactive : public Usermod { if(audioSyncEnabled == 3) { // Send using ESP-NOW // Send message via ESP-NOW - Serial.printf("esp_now_send(%u)\n", transmitData.frameCounter); + Serial.printf("esp_now_send(%u) - ", transmitData.frameCounter); esp_err_t result = esp_now_send(broadcastAddress, (uint8_t *) &transmitData, sizeof(transmitData)); if (result == ESP_OK) { Serial.println("Sent with success"); } else { - Serial.println("Error sending the data"); + Serial.printf("Error sending the data error(%d) = %s\n", result, esp_err_to_name(result)); } } else { + Serial.printf("beginMulticastPacket(%u)\n", transmitData.frameCounter); if (fftUdp.beginMulticastPacket() != 0) { // beginMulticastPacket returns 0 in case of error fftUdp.write(reinterpret_cast(&transmitData), sizeof(transmitData)); fftUdp.endPacket(); @@ -1851,8 +1852,13 @@ class AudioReactive : public Usermod { } if(audioSyncEnabled == 3) { - DEBUGSR_PRINTLN(F("AR connected(): audioSyncEnabled == 3")); - udpSyncConnected = true; // TODO: better name this flag + if (esp_now_init() != ESP_OK) { + DEBUGSR_PRINTLN("Error initializing ESP-NOW"); + udpSyncConnected = false; + } + else { + udpSyncConnected = true; // TODO: better name this flag + } } else if (audioSyncPort > 0 && (audioSyncEnabled & 0x03)) { #ifdef ARDUINO_ARCH_ESP32 From 1221ee14e2011f6767daeba61684a9e5377445e6 Mon Sep 17 00:00:00 2001 From: Will Tatam Date: Sun, 29 Oct 2023 15:41:12 +0000 Subject: [PATCH 03/13] WiP on adding ESP-NOW audio sync --- usermods/audioreactive/audio_reactive.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/usermods/audioreactive/audio_reactive.h b/usermods/audioreactive/audio_reactive.h index cf82b3e3de..e04a5411da 100644 --- a/usermods/audioreactive/audio_reactive.h +++ b/usermods/audioreactive/audio_reactive.h @@ -2002,7 +2002,7 @@ class AudioReactive : public Usermod { connectUDPSoundSync(); // ensure we have a connection - if needed // UDP Microphone Sync - receive mode - if ((audioSyncEnabled & 0x02) && udpSyncConnected) { + if ((audioSyncEnabled == 2) && udpSyncConnected) { // Only run the audio listener code if we're in Receive mode static float syncVolumeSmth = 0; bool have_new_sample = false; @@ -2067,8 +2067,8 @@ class AudioReactive : public Usermod { #endif #ifdef ARDUINO_ARCH_ESP32 - //UDP Microphone Sync - transmit mode - if ((audioSyncEnabled & 0x01) && (millis() - lastTime > 20)) { + // Audio Sync - transmit mode + if (((audioSyncEnabled == 1) || (audioSyncEnabled == 3)) && (millis() - lastTime > 20)) { // Only run the transmit code IF we're in Transmit mode transmitAudioData(); lastTime = millis(); From d179fed4811dff4607a4f2e927a5cede8029ed9d Mon Sep 17 00:00:00 2001 From: Will Tatam Date: Sun, 29 Oct 2023 16:17:41 +0000 Subject: [PATCH 04/13] WiP on adding ESP-NOW audio sync --- usermods/audioreactive/audio_reactive.h | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/usermods/audioreactive/audio_reactive.h b/usermods/audioreactive/audio_reactive.h index e04a5411da..9402c370d3 100644 --- a/usermods/audioreactive/audio_reactive.h +++ b/usermods/audioreactive/audio_reactive.h @@ -2022,7 +2022,7 @@ class AudioReactive : public Usermod { receivedFormat = 0; } - if ( (audioSyncEnabled & 0x02) // receive mode + if ( (audioSyncEnabled == 2) // receive mode && udpSyncConnected // connected && (receivedFormat > 0) // we actually received something in the past && ((millis() - last_UDPTime) > 25000)) { // close connection after 25sec idle @@ -2245,7 +2245,7 @@ class AudioReactive : public Usermod { // The following can be used for troubleshooting user errors and is so not enclosed in #ifdef WLED_DEBUG // current Audio input infoArr = user.createNestedArray(F("Audio Source")); - if (audioSyncEnabled & 0x02) { + if (audioSyncEnabled == 2) { // UDP sound sync - receive mode infoArr.add(F("UDP sound sync")); if (udpSyncConnected) { @@ -2261,17 +2261,17 @@ class AudioReactive : public Usermod { infoArr.add(F("sound sync Off")); } #else // ESP32 only - } else if (audioSyncEnabled & 0x04) { + } else if (audioSyncEnabled == 4) { infoArr = user.createNestedArray(F("Audio Source")); infoArr.add(F("ESP-NOW sound sync")); - // if (udpSyncConnected) { - // if (millis() - last_UDPTime < 2500) - // infoArr.add(F(" - receiving")); - // else - // infoArr.add(F(" - idle")); - // } else { - // infoArr.add(F(" - no connection")); - // } + if (udpSyncConnected) { + if (millis() - last_UDPTime < 2500) + infoArr.add(F(" - receiving")); + else + infoArr.add(F(" - idle")); + } else { + infoArr.add(F(" - no connection")); + } } else { // Analog or I2S digital input if (audioSource && (audioSource->isInitialized())) { From cdeeae612372be7d6736c555b118e6ed99995272 Mon Sep 17 00:00:00 2001 From: Will Tatam Date: Sun, 29 Oct 2023 16:22:37 +0000 Subject: [PATCH 05/13] handle WLED_DISABLE_ESPNOW --- usermods/audioreactive/audio_reactive.h | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/usermods/audioreactive/audio_reactive.h b/usermods/audioreactive/audio_reactive.h index 9402c370d3..a09d5d52d1 100644 --- a/usermods/audioreactive/audio_reactive.h +++ b/usermods/audioreactive/audio_reactive.h @@ -1481,6 +1481,7 @@ class AudioReactive : public Usermod { transmitData.FFT_MajorPeak = FFT_MajorPeak; if(audioSyncEnabled == 3) { // Send using ESP-NOW + #ifndef WLED_DISABLE_ESPNOW // Send message via ESP-NOW Serial.printf("esp_now_send(%u) - ", transmitData.frameCounter); esp_err_t result = esp_now_send(broadcastAddress, (uint8_t *) &transmitData, sizeof(transmitData)); @@ -1488,8 +1489,9 @@ class AudioReactive : public Usermod { Serial.println("Sent with success"); } else { - Serial.printf("Error sending the data error(%d) = %s\n", result, esp_err_to_name(result)); + Serial.printf("Error sending the data error(%x) = %s\n", result, esp_err_to_name(result)); } + #endif } else { Serial.printf("beginMulticastPacket(%u)\n", transmitData.frameCounter); @@ -1852,6 +1854,7 @@ class AudioReactive : public Usermod { } if(audioSyncEnabled == 3) { + #ifndef WLED_DISABLE_ESPNOW if (esp_now_init() != ESP_OK) { DEBUGSR_PRINTLN("Error initializing ESP-NOW"); udpSyncConnected = false; @@ -1859,6 +1862,7 @@ class AudioReactive : public Usermod { else { udpSyncConnected = true; // TODO: better name this flag } + #endif } else if (audioSyncPort > 0 && (audioSyncEnabled & 0x03)) { #ifdef ARDUINO_ARCH_ESP32 From ca2aea9806c4bfe82ebf7cdb2df58fbdaec1254b Mon Sep 17 00:00:00 2001 From: Will Tatam Date: Sun, 29 Oct 2023 16:23:45 +0000 Subject: [PATCH 06/13] handle WLED_DISABLE_ESPNOW --- usermods/audioreactive/audio_reactive.h | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/usermods/audioreactive/audio_reactive.h b/usermods/audioreactive/audio_reactive.h index a09d5d52d1..4a7005242f 100644 --- a/usermods/audioreactive/audio_reactive.h +++ b/usermods/audioreactive/audio_reactive.h @@ -2735,10 +2735,14 @@ class AudioReactive : public Usermod { oappend(SET_F("addOption(dd,'Off',0);")); #ifdef ARDUINO_ARCH_ESP32 oappend(SET_F("addOption(dd,'Send (UDP)',1);")); + #ifndef WLED_DISABLE_ESPNOW oappend(SET_F("addOption(dd,'Send (ESP-NOW)',3);")); + #endif #endif oappend(SET_F("addOption(dd,'Receive (UDP)',2);")); + #ifndef WLED_DISABLE_ESPNOW oappend(SET_F("addOption(dd,'Receive (ESP-NOW)',4);")); + #endif oappend(SET_F("addInfo('AudioReactive:sync:mode',1,'
Sync audio data with other WLEDs');")); oappend(SET_F("addInfo('AudioReactive:digitalmic:type',1,'requires reboot!');")); // 0 is field type, 1 is actual field From 3a9a95c0eee472c7428a452af2058c4ae7af55b4 Mon Sep 17 00:00:00 2001 From: Will Tatam Date: Sun, 29 Oct 2023 16:52:13 +0000 Subject: [PATCH 07/13] esp-now send and recieve --- usermods/audioreactive/audio_reactive.h | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/usermods/audioreactive/audio_reactive.h b/usermods/audioreactive/audio_reactive.h index 4a7005242f..a14c315cf8 100644 --- a/usermods/audioreactive/audio_reactive.h +++ b/usermods/audioreactive/audio_reactive.h @@ -910,6 +910,8 @@ static void autoResetPeak(void) { } } +void OnDataRecvAudio(const uint8_t * mac, const uint8_t *incomingData, int len); + //////////////////// // usermod class // //////////////////// @@ -1483,13 +1485,19 @@ class AudioReactive : public Usermod { if(audioSyncEnabled == 3) { // Send using ESP-NOW #ifndef WLED_DISABLE_ESPNOW // Send message via ESP-NOW - Serial.printf("esp_now_send(%u) - ", transmitData.frameCounter); + esp_now_peer_info_t peerInfo = {}; + memcpy(&peerInfo.peer_addr, broadcastAddress, 6); + if (!esp_now_is_peer_exist(broadcastAddress)) { + esp_now_add_peer(&peerInfo); + } esp_err_t result = esp_now_send(broadcastAddress, (uint8_t *) &transmitData, sizeof(transmitData)); if (result == ESP_OK) { - Serial.println("Sent with success"); + // DEBUGSR_PRINTF("esp_now_send(%u) - ", transmitData.frameCounter); + // DEBUGSR_PRINTF("Sent with success"); } else { - Serial.printf("Error sending the data error(%x) = %s\n", result, esp_err_to_name(result)); + DEBUGSR_PRINTF("esp_now_send(%u) - ", transmitData.frameCounter); + DEBUGSR_PRINTF("Error sending the data error(%x) = %s\n", result, esp_err_to_name(result)); } #endif } @@ -1853,7 +1861,7 @@ class AudioReactive : public Usermod { DEBUGSR_PRINTLN(F("AR connected(): old UDP connection closed.")); } - if(audioSyncEnabled == 3) { + if(audioSyncEnabled == 3 || audioSyncEnabled == 4) { #ifndef WLED_DISABLE_ESPNOW if (esp_now_init() != ESP_OK) { DEBUGSR_PRINTLN("Error initializing ESP-NOW"); @@ -1862,6 +1870,7 @@ class AudioReactive : public Usermod { else { udpSyncConnected = true; // TODO: better name this flag } + esp_now_register_recv_cb(OnDataRecvAudio); #endif } else if (audioSyncPort > 0 && (audioSyncEnabled & 0x03)) { @@ -2810,7 +2819,9 @@ class AudioReactive : public Usermod { return USERMOD_ID_AUDIOREACTIVE; } }; - +void OnDataRecvAudio(const uint8_t * mac, const uint8_t *incomingData, int len) { + Serial.println("OnDataRecv"); +} // strings to reduce flash memory usage (used more than twice) const char AudioReactive::_name[] PROGMEM = "AudioReactive"; const char AudioReactive::_enabled[] PROGMEM = "enabled"; From e6e27aa40497b365562b6f4d428a528a55c72aa8 Mon Sep 17 00:00:00 2001 From: Will Tatam Date: Sun, 29 Oct 2023 17:14:08 +0000 Subject: [PATCH 08/13] Audio decode --- usermods/audioreactive/audio_reactive.h | 80 +++++++++++++------------ 1 file changed, 41 insertions(+), 39 deletions(-) diff --git a/usermods/audioreactive/audio_reactive.h b/usermods/audioreactive/audio_reactive.h index a14c315cf8..e4ac43ea0e 100644 --- a/usermods/audioreactive/audio_reactive.h +++ b/usermods/audioreactive/audio_reactive.h @@ -1520,44 +1520,6 @@ class AudioReactive : public Usermod { return strncmp_P(header, UDP_SYNC_HEADER_v1, 6) == 0; } - void decodeAudioData(int packetSize, uint8_t *fftBuff) { - audioSyncPacket *receivedPacket = reinterpret_cast(fftBuff); - - static uint8_t lastFrameCounter = 0; - if(receivedPacket->frameCounter <= lastFrameCounter && receivedPacket->frameCounter != 0) { // TODO: might need extra checks here - DEBUGSR_PRINTF("Skipping audio frame out of order or duplicated - %u vs %u\n", lastFrameCounter, receivedPacket->frameCounter); - return; - } - else { - lastFrameCounter = receivedPacket->frameCounter; - } - - // update samples for effects - volumeSmth = fmaxf(receivedPacket->sampleSmth, 0.0f); - volumeRaw = fmaxf(receivedPacket->sampleRaw, 0.0f); -#ifdef ARDUINO_ARCH_ESP32 - // update internal samples - sampleRaw = volumeRaw; - sampleAvg = volumeSmth; - rawSampleAgc = volumeRaw; - sampleAgc = volumeSmth; - multAgc = 1.0f; -#endif - // Only change samplePeak IF it's currently false. - // If it's true already, then the animation still needs to respond. - autoResetPeak(); - if (!samplePeak) { - samplePeak = receivedPacket->samplePeak >0 ? true:false; - if (samplePeak) timeOfPeak = millis(); - //userVar1 = samplePeak; - } - //These values are only computed by ESP32 - for (int i = 0; i < NUM_GEQ_CHANNELS; i++) fftResult[i] = receivedPacket->fftResult[i]; - my_magnitude = fmaxf(receivedPacket->FFT_Magnitude, 0.0f); - FFT_Magnitude = my_magnitude; - FFT_MajorPeak = constrain(receivedPacket->FFT_MajorPeak, 1.0f, 11025.0f); // restrict value to range expected by effects - } - void decodeAudioData_v1(int packetSize, uint8_t *fftBuff) { audioSyncPacket_v1 *receivedPacket = reinterpret_cast(fftBuff); // update samples for effects @@ -2818,9 +2780,49 @@ class AudioReactive : public Usermod { { return USERMOD_ID_AUDIOREACTIVE; } + + static void decodeAudioData(int packetSize, uint8_t *fftBuff) { + audioSyncPacket *receivedPacket = reinterpret_cast(fftBuff); + + static uint8_t lastFrameCounter = 0; + if(receivedPacket->frameCounter <= lastFrameCounter && receivedPacket->frameCounter != 0) { // TODO: might need extra checks here + DEBUGSR_PRINTF("Skipping audio frame out of order or duplicated - %u vs %u\n", lastFrameCounter, receivedPacket->frameCounter); + return; + } + else { + lastFrameCounter = receivedPacket->frameCounter; + DEBUGSR_PRINTF("frameCounter: %u\n", lastFrameCounter); + } + + // update samples for effects + volumeSmth = fmaxf(receivedPacket->sampleSmth, 0.0f); + // volumeRaw = fmaxf(receivedPacket->sampleRaw, 0.0f); +#ifdef ARDUINO_ARCH_ESP32 + // update internal samples + // sampleRaw = volumeRaw; + sampleAvg = volumeSmth; + // rawSampleAgc = volumeRaw; + sampleAgc = volumeSmth; + multAgc = 1.0f; +#endif + // Only change samplePeak IF it's currently false. + // If it's true already, then the animation still needs to respond. + autoResetPeak(); + if (!samplePeak) { + samplePeak = receivedPacket->samplePeak >0 ? true:false; + if (samplePeak) timeOfPeak = millis(); + //userVar1 = samplePeak; + } + //These values are only computed by ESP32 + for (int i = 0; i < NUM_GEQ_CHANNELS; i++) fftResult[i] = receivedPacket->fftResult[i]; + float my_magnitude = fmaxf(receivedPacket->FFT_Magnitude, 0.0f); + // FFT_Magnitude = my_magnitude; + FFT_MajorPeak = constrain(receivedPacket->FFT_MajorPeak, 1.0f, 11025.0f); // restrict value to range expected by effects + } }; void OnDataRecvAudio(const uint8_t * mac, const uint8_t *incomingData, int len) { - Serial.println("OnDataRecv"); + uint8_t * bptr = (uint8_t*)&incomingData; + AudioReactive::decodeAudioData(len, bptr); } // strings to reduce flash memory usage (used more than twice) const char AudioReactive::_name[] PROGMEM = "AudioReactive"; From 88eac1cc2c74a60501feee55b3f93eb047e1382e Mon Sep 17 00:00:00 2001 From: Will Tatam Date: Sun, 29 Oct 2023 17:43:20 +0000 Subject: [PATCH 09/13] Audio decode --- usermods/audioreactive/audio_reactive.h | 34 +++++++++++++++---------- 1 file changed, 21 insertions(+), 13 deletions(-) diff --git a/usermods/audioreactive/audio_reactive.h b/usermods/audioreactive/audio_reactive.h index e4ac43ea0e..c3eb1ab75a 100644 --- a/usermods/audioreactive/audio_reactive.h +++ b/usermods/audioreactive/audio_reactive.h @@ -964,17 +964,6 @@ class AudioReactive : public Usermod { int8_t mclkPin = MCLK_PIN; #endif #endif - // new "V2" audiosync struct - 40 Bytes - struct audioSyncPacket { - char header[6]; // 06 Bytes - float sampleRaw; // 04 Bytes - either "sampleRaw" or "rawSampleAgc" depending on soundAgc setting - float sampleSmth; // 04 Bytes - either "sampleAvg" or "sampleAgc" depending on soundAgc setting - uint8_t samplePeak; // 01 Bytes - 0 no peak; >=1 peak detected. In future, this will also provide peak Magnitude - uint8_t frameCounter; // 01 Bytes - track duplicate/out of order packets - uint8_t fftResult[16]; // 16 Bytes - float FFT_Magnitude; // 04 Bytes - float FFT_MajorPeak; // 04 Bytes - }; // old "V1" audiosync struct - 83 Bytes - for backwards compatibility struct audioSyncPacket_v1 { @@ -2781,9 +2770,24 @@ class AudioReactive : public Usermod { return USERMOD_ID_AUDIOREACTIVE; } + // new "V2" audiosync struct - 40 Bytes + struct audioSyncPacket { + char header[6]; // 06 Bytes + float sampleRaw; // 04 Bytes - either "sampleRaw" or "rawSampleAgc" depending on soundAgc setting + float sampleSmth; // 04 Bytes - either "sampleAvg" or "sampleAgc" depending on soundAgc setting + uint8_t samplePeak; // 01 Bytes - 0 no peak; >=1 peak detected. In future, this will also provide peak Magnitude + uint8_t frameCounter; // 01 Bytes - track duplicate/out of order packets + uint8_t fftResult[16]; // 16 Bytes + float FFT_Magnitude; // 04 Bytes + float FFT_MajorPeak; // 04 Bytes + }; + static void decodeAudioData(int packetSize, uint8_t *fftBuff) { audioSyncPacket *receivedPacket = reinterpret_cast(fftBuff); + handleAudioSyncPacket(receivedPacket); + } + static void handleAudioSyncPacket(AudioReactive::audioSyncPacket *receivedPacket) { static uint8_t lastFrameCounter = 0; if(receivedPacket->frameCounter <= lastFrameCounter && receivedPacket->frameCounter != 0) { // TODO: might need extra checks here DEBUGSR_PRINTF("Skipping audio frame out of order or duplicated - %u vs %u\n", lastFrameCounter, receivedPacket->frameCounter); @@ -2821,8 +2825,12 @@ class AudioReactive : public Usermod { } }; void OnDataRecvAudio(const uint8_t * mac, const uint8_t *incomingData, int len) { - uint8_t * bptr = (uint8_t*)&incomingData; - AudioReactive::decodeAudioData(len, bptr); + AudioReactive::audioSyncPacket* receivedPacket; + memcpy(&receivedPacket, incomingData, sizeof(receivedPacket)); + // Serial.printf("OnDataRecvAudio len=%u\n", len); + if(len == 44) { + AudioReactive::handleAudioSyncPacket(receivedPacket); + } } // strings to reduce flash memory usage (used more than twice) const char AudioReactive::_name[] PROGMEM = "AudioReactive"; From fa42fe6358a4eb1757307461a074659b61a7a8bf Mon Sep 17 00:00:00 2001 From: Will Tatam Date: Mon, 30 Oct 2023 20:40:30 +0000 Subject: [PATCH 10/13] Working ESP-NOW recieve --- usermods/audioreactive/audio_reactive.h | 26 ++++++++++++++++--------- 1 file changed, 17 insertions(+), 9 deletions(-) diff --git a/usermods/audioreactive/audio_reactive.h b/usermods/audioreactive/audio_reactive.h index c3eb1ab75a..7611207dd0 100644 --- a/usermods/audioreactive/audio_reactive.h +++ b/usermods/audioreactive/audio_reactive.h @@ -1475,6 +1475,8 @@ class AudioReactive : public Usermod { #ifndef WLED_DISABLE_ESPNOW // Send message via ESP-NOW esp_now_peer_info_t peerInfo = {}; + peerInfo.channel = 0; + peerInfo.encrypt = false; memcpy(&peerInfo.peer_addr, broadcastAddress, 6); if (!esp_now_is_peer_exist(broadcastAddress)) { esp_now_add_peer(&peerInfo); @@ -1502,9 +1504,6 @@ class AudioReactive : public Usermod { } // transmitAudioData() #endif - static bool isValidUdpSyncVersion(const char *header) { - return strncmp_P(header, UDP_SYNC_HEADER, 6) == 0; - } static bool isValidUdpSyncVersion_v1(const char *header) { return strncmp_P(header, UDP_SYNC_HEADER_v1, 6) == 0; } @@ -2795,7 +2794,7 @@ class AudioReactive : public Usermod { } else { lastFrameCounter = receivedPacket->frameCounter; - DEBUGSR_PRINTF("frameCounter: %u\n", lastFrameCounter); + // DEBUGSR_PRINTF("header: %s frameCounter: %u\n", String(receivedPacket->header), lastFrameCounter); } // update samples for effects @@ -2823,13 +2822,22 @@ class AudioReactive : public Usermod { // FFT_Magnitude = my_magnitude; FFT_MajorPeak = constrain(receivedPacket->FFT_MajorPeak, 1.0f, 11025.0f); // restrict value to range expected by effects } + + static bool isValidUdpSyncVersion(const char *header) { + return strncmp_P(header, UDP_SYNC_HEADER, 6) == 0; + } }; void OnDataRecvAudio(const uint8_t * mac, const uint8_t *incomingData, int len) { - AudioReactive::audioSyncPacket* receivedPacket; - memcpy(&receivedPacket, incomingData, sizeof(receivedPacket)); - // Serial.printf("OnDataRecvAudio len=%u\n", len); - if(len == 44) { - AudioReactive::handleAudioSyncPacket(receivedPacket); + bool validData = AudioReactive::isValidUdpSyncVersion((const char *)incomingData); + if(validData) { + AudioReactive::audioSyncPacket receivedPacket; + if(len >= sizeof(receivedPacket)) { // TODO: should be == ? + memcpy(&receivedPacket, incomingData, sizeof(receivedPacket)); + AudioReactive::handleAudioSyncPacket(&receivedPacket); + } + } + else { + DEBUGSR_PRINT("E"); } } // strings to reduce flash memory usage (used more than twice) From 6f473cbebe37b50e39482223d8f76b7f82872324 Mon Sep 17 00:00:00 2001 From: Will Tatam Date: Tue, 31 Oct 2023 14:11:39 +0000 Subject: [PATCH 11/13] Track time interval between sync packets --- platformio.ini | 1 + usermods/audioreactive/audio_reactive.h | 13 +++++++++++-- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/platformio.ini b/platformio.ini index 1e511cf7a0..0e6112b388 100644 --- a/platformio.ini +++ b/platformio.ini @@ -928,6 +928,7 @@ build_flags_S = lib_deps_S = ;; https://github.com/kosme/arduinoFFT#develop @ 1.9.2+sha.419d7b0 ;; used for USERMOD_AUDIOREACTIVE - using "known working" hash https://github.com/softhack007/arduinoFFT.git#develop @ 1.9.2 ;; used for USERMOD_AUDIOREACTIVE - optimized version, 10% faster on -S2/-C3 + https://github.com/RobTillaart/RunningAverage.git@0.4.3 animartrix_build_flags = -D USERMOD_ANIMARTRIX ;; WLEDMM usermod: CC BY-NC 3.0 licensed effects by Stefan Petrick diff --git a/usermods/audioreactive/audio_reactive.h b/usermods/audioreactive/audio_reactive.h index 7611207dd0..17b4b8c310 100644 --- a/usermods/audioreactive/audio_reactive.h +++ b/usermods/audioreactive/audio_reactive.h @@ -1,6 +1,8 @@ #pragma once #include "wled.h" +#include "RunningAverage.h" +RunningAverage syncAverage(1000); #ifdef ARDUINO_ARCH_ESP32 @@ -911,6 +913,7 @@ static void autoResetPeak(void) { } void OnDataRecvAudio(const uint8_t * mac, const uint8_t *incomingData, int len); +unsigned long last_UDPTime = 0; // time of last valid UDP sound sync datapacket //////////////////// // usermod class // @@ -1017,7 +1020,6 @@ class AudioReactive : public Usermod { float soundPressure = 0; // Sound Pressure estimation, based on microphone raw readings. 0 ->5db, 255 ->105db // used to feed "Info" Page - unsigned long last_UDPTime = 0; // time of last valid UDP sound sync datapacket int receivedFormat = 0; // last received UDP sound sync format - 0=none, 1=v1 (0.13.x), 2=v2 (0.14.x) float maxSample5sec = 0.0f; // max sample (after AGC) in last 5 seconds unsigned long sampleMaxTimer = 0; // last time maxSample5sec was reset @@ -1971,7 +1973,10 @@ class AudioReactive : public Usermod { bool have_new_sample = false; if (millis() - lastTime > delayMs) { have_new_sample = receiveAudioData(); - if (have_new_sample) last_UDPTime = millis(); + if (have_new_sample) { + syncAverage.addValue(millis() - last_UDPTime); + last_UDPTime = millis(); + } lastTime = millis(); } else { #ifdef ARDUINO_ARCH_ESP32 @@ -2219,6 +2224,7 @@ class AudioReactive : public Usermod { } else { infoArr.add(F(" - no connection")); } + Serial.printf("avg=%.3f, min=%.3f, max=%.3f\n", syncAverage.getAverage(), syncAverage.getMinInBuffer(), syncAverage.getMaxInBuffer()); #ifndef ARDUINO_ARCH_ESP32 // substitute for 8266 } else { infoArr.add(F("sound sync Off")); @@ -2235,6 +2241,7 @@ class AudioReactive : public Usermod { } else { infoArr.add(F(" - no connection")); } + Serial.printf("avg=%.3f, min=%.3f, max=%.3f\n", syncAverage.getAverage(), syncAverage.getMinInBuffer(), syncAverage.getMaxInBuffer()); } else { // Analog or I2S digital input if (audioSource && (audioSource->isInitialized())) { @@ -2834,6 +2841,8 @@ void OnDataRecvAudio(const uint8_t * mac, const uint8_t *incomingData, int len) if(len >= sizeof(receivedPacket)) { // TODO: should be == ? memcpy(&receivedPacket, incomingData, sizeof(receivedPacket)); AudioReactive::handleAudioSyncPacket(&receivedPacket); + syncAverage.addValue(millis() - last_UDPTime); + last_UDPTime = millis(); } } else { From 81c126aef2a443c3e4f9d376fcf9fbadcdf2acd1 Mon Sep 17 00:00:00 2001 From: Will Tatam Date: Tue, 31 Oct 2023 20:59:50 +0000 Subject: [PATCH 12/13] Working ESP-NOW audio sync --- usermods/audioreactive/audio_reactive.h | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/usermods/audioreactive/audio_reactive.h b/usermods/audioreactive/audio_reactive.h index 17b4b8c310..cad8616ce6 100644 --- a/usermods/audioreactive/audio_reactive.h +++ b/usermods/audioreactive/audio_reactive.h @@ -446,7 +446,7 @@ void FFTcode(void * parameter) // taskYIELD(), yield(), vTaskDelay() and esp_task_wdt_feed() didn't seem to work. // Don't run FFT computing code if we're in Receive mode or in realtime mode - if (disableSoundProcessing || (audioSyncEnabled & 0x02)) { + if (disableSoundProcessing || (audioSyncEnabled == 2) || (audioSyncEnabled == 4)) { isFirstRun = false; vTaskDelayUntil( &xLastWakeTime, xFrequency); // release CPU, and let I2S fill its buffers continue; @@ -1451,7 +1451,10 @@ class AudioReactive : public Usermod { #ifdef ARDUINO_ARCH_ESP32 void transmitAudioData() { - if (!udpSyncConnected) return; + if (!udpSyncConnected) { + // DEBUGSR_PRINTLN("transmitAudioData not connected"); + return; + } static uint8_t frameCounter = 0; //DEBUGSR_PRINTLN("Transmitting UDP Mic Packet"); @@ -1822,7 +1825,7 @@ class AudioReactive : public Usermod { else { udpSyncConnected = true; // TODO: better name this flag } - esp_now_register_recv_cb(OnDataRecvAudio); + if(audioSyncEnabled == 4) esp_now_register_recv_cb(OnDataRecvAudio); #endif } else if (audioSyncPort > 0 && (audioSyncEnabled & 0x03)) { @@ -1896,6 +1899,8 @@ class AudioReactive : public Usermod { if (audioSyncEnabled & 0x02) disableSoundProcessing = true; // make sure everything is disabled IF in audio Receive mode if (audioSyncEnabled & 0x01) disableSoundProcessing = false; // keep running audio IF we're in audio Transmit mode + if (audioSyncEnabled == 3) disableSoundProcessing = false; // keep running audio IF we're in audio Transmit mode + if (audioSyncEnabled == 4) disableSoundProcessing = true; // make sure everything is disabled IF in audio Receive mode #ifdef ARDUINO_ARCH_ESP32 if (!audioSource->isInitialized()) disableSoundProcessing = true; // no audio source @@ -1910,7 +1915,7 @@ class AudioReactive : public Usermod { #endif // Only run the sampling code IF we're not in Receive mode or realtime mode - if (!(audioSyncEnabled & 0x02) && !disableSoundProcessing) { + if (!disableSoundProcessing) { if (soundAgc > AGC_NUM_PRESETS) soundAgc = 0; // make sure that AGC preset is valid (to avoid array bounds violation) unsigned long t_now = millis(); // remember current time @@ -2215,7 +2220,7 @@ class AudioReactive : public Usermod { infoArr = user.createNestedArray(F("Audio Source")); if (audioSyncEnabled == 2) { // UDP sound sync - receive mode - infoArr.add(F("UDP sound sync")); + infoArr.add(F("UDP sync")); if (udpSyncConnected) { if (millis() - last_UDPTime < 2500) infoArr.add(F(" - receiving")); @@ -2232,7 +2237,7 @@ class AudioReactive : public Usermod { #else // ESP32 only } else if (audioSyncEnabled == 4) { infoArr = user.createNestedArray(F("Audio Source")); - infoArr.add(F("ESP-NOW sound sync")); + infoArr.add(F("ESP-NOW sync")); if (udpSyncConnected) { if (millis() - last_UDPTime < 2500) infoArr.add(F(" - receiving")); @@ -2835,6 +2840,7 @@ class AudioReactive : public Usermod { } }; void OnDataRecvAudio(const uint8_t * mac, const uint8_t *incomingData, int len) { + if(audioSyncEnabled != 4) return; bool validData = AudioReactive::isValidUdpSyncVersion((const char *)incomingData); if(validData) { AudioReactive::audioSyncPacket receivedPacket; From f3365e61ddaaf41b9d42af744fe142e0686d26d1 Mon Sep 17 00:00:00 2001 From: Will Tatam Date: Wed, 1 Nov 2023 21:37:05 +0000 Subject: [PATCH 13/13] Display avg packet times --- usermods/audioreactive/audio_reactive.h | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/usermods/audioreactive/audio_reactive.h b/usermods/audioreactive/audio_reactive.h index cad8616ce6..c7557fa2b4 100644 --- a/usermods/audioreactive/audio_reactive.h +++ b/usermods/audioreactive/audio_reactive.h @@ -2172,7 +2172,7 @@ class AudioReactive : public Usermod { void addToJsonInfo(JsonObject& root) { #ifdef ARDUINO_ARCH_ESP32 - char myStringBuffer[16]; // buffer for snprintf() - not used yet on 8266 + char myStringBuffer[21]; // buffer for snprintf() - not used yet on 8266 #endif JsonObject user = root["u"]; if (user.isNull()) user = root.createNestedObject("u"); @@ -2229,7 +2229,10 @@ class AudioReactive : public Usermod { } else { infoArr.add(F(" - no connection")); } - Serial.printf("avg=%.3f, min=%.3f, max=%.3f\n", syncAverage.getAverage(), syncAverage.getMinInBuffer(), syncAverage.getMaxInBuffer()); + DEBUG_PRINTF("std=%.3f avg=%.3f, min=%.3f, max=%.3f\n", syncAverage.getStandardDeviation(), syncAverage.getAverage(), syncAverage.getMinInBuffer(), syncAverage.getMaxInBuffer()); + snprintf_P(myStringBuffer, 20, PSTR("s:%.2f a:%.2f"), syncAverage.getStandardDeviation(), syncAverage.getAverage()); + infoArr = user.createNestedArray(F("Audio Sync")); + infoArr.add(myStringBuffer); #ifndef ARDUINO_ARCH_ESP32 // substitute for 8266 } else { infoArr.add(F("sound sync Off")); @@ -2246,7 +2249,10 @@ class AudioReactive : public Usermod { } else { infoArr.add(F(" - no connection")); } - Serial.printf("avg=%.3f, min=%.3f, max=%.3f\n", syncAverage.getAverage(), syncAverage.getMinInBuffer(), syncAverage.getMaxInBuffer()); + DEBUG_PRINTF("std=%.3f avg=%.3f, min=%.3f, max=%.3f\n", syncAverage.getStandardDeviation(), syncAverage.getAverage(), syncAverage.getMinInBuffer(), syncAverage.getMaxInBuffer()); + snprintf_P(myStringBuffer, 20, PSTR("s:%.2f a:%.2f"), syncAverage.getStandardDeviation(), syncAverage.getAverage()); + infoArr = user.createNestedArray(F("Audio Sync")); + infoArr.add(myStringBuffer); } else { // Analog or I2S digital input if (audioSource && (audioSource->isInitialized())) {