From 7d320cc2a341321f977d057a8ef62d285d518e2a Mon Sep 17 00:00:00 2001 From: Ajay Raj Date: Mon, 8 Jul 2024 18:36:34 -0700 Subject: [PATCH 1/9] adds twilio dtmf --- .coverage | Bin 0 -> 77824 bytes vocode/streaming/action/dtmf.py | 22 +++++++--- .../output_device/twilio_output_device.py | 16 ++++++- vocode/streaming/utils/dtmf.py | 40 ++++++++++++++++++ 4 files changed, 71 insertions(+), 7 deletions(-) create mode 100644 .coverage create mode 100644 vocode/streaming/utils/dtmf.py diff --git a/.coverage b/.coverage new file mode 100644 index 0000000000000000000000000000000000000000..cd4158d3ecec6dac5523e64651fb6a13e315cdd2 GIT binary patch literal 77824 zcmeHw349aRwfBrP+BGxAfH$x$7GqY6#bGme0Sp05%)S{s9!X{o3z)O-h!wq>rS*P1~0yAt7BJX+wadS)8OPkho#D_0HTGNh8Uo1daM$ z`bIzSnf3nf`QLNTz30xoGiz4WxPqLe(dYNDLCXv*1H*Bw(qh3d%n1L7!vDxm2OC6W z2P7`p9&FQym9*bQ((5s^_Ujm3Pu9{?%ny^}OusdI40oCK>sW&wPAETx0fhmD0fmA8 z-3)|QnRVH@x%j@_LDuHxf_~P{`Gvpa!qwGPYpX45s}|H$TZChl@kR^$rcAL^S^T~Z zODpHMG`ifJ#pQLl>}=5Gby|W=T;%Gu0O#NvIyeSR9e-g7x}V=-Tn@M^$T=a2R=>-` z`a3NxT<2tTfru95wg-g+kO=2;dig6X^Vbm46Ksd9@yrF{uaZw1$}6rfx>5| z!5{N&L-e}r!b1FxZOBk~dKjB47>I5d;*2JGco5U!&kt@ld@YGL622C>-4kGibA`3kNfus0lFBqMOE0u2z>lVg$a=fXqlNEm7#= zOej*41Gv%1DG$d6+WZkmaxEtrxg+K!BXvYP1L{oCX|tvf3j72gaCse9ts(l2QRPWV(ktV5{oe4$8E>}cYWk`r2^MQo4@x;&hP zzXW9l8CC}yu-N?^oP@S?Lb3WNo6P`N>9sI6G`djsRx}WG) zXy4a1Yc6WOr70y|Cz{otsc%;osk&7Q@DsQnr?E#Os42?UVgd_s)^!0`2x4WM*-k&( zYz_GB)^?xW=isLBs~G%V)*abrTV0V&SV75&+txr3R&aP=J*73sxw+ORpSRO$ce^-m z5U%jTx-Ul9gXJUaI6zz?BVt?7*&*0Kr`OK-Iarn$09F8q z8w)_SG6Kce3v!-TH!Oqh}dmFyrLK7R|Mu*CxD*U6YZtQKo+3o;I_-DT&jY@5U7W9&_BUS65@LIB&A zKCnn(#pMBT9R+~9LPlHfg&f$O}XFDZY zxnK~CjkULMUI(btIt+m1G)2J}MCWH?1MqeSWFEp1K(r49L^&G`x2v6NaRvFc@v&0M zl)02d{-7!J2JC(pyf=Y2xgmfer^$&OmLRn546M}!pTxt|X8=q(Tpr&vLHZ3eq+cy# z4HeXom~%aq9t}{b@EVXZYakZ4hYJMwR}%n&IXwu_v*KTb00^835acu_A!(~17*r+z z<1hk@oc>FMK~-JY29#z4plp)Sq(l@xet?L9b-S4;)>b`WRmhL!2znYpl^Pu+pPzE_ z{L*wtZKW1cD@r*vN!?nlffOg)c#1&?SpwEZwuR&00AO|TtpZCxnkhtLLYmPYfoOve z0S2I7EYRAZ28a}#&L9xb2sHAiK#5uvAdE?s7-@pI90@WImk+6dKN-a;_^YBP@`3tL z`Vo2}^$__t@&{(h+(s@jpU~f?KW}o?nCLN;n7h0wfX?LxO@qgY9BGe&g$w&(R@5KMPP9n4_UC@$#P)_QB#6VTQf(YfOp1c_U>sAq=qSRB9)Gh5AA~Yd| z6oo)KKK|FF7>NxMob!hx@jr3>6-o$ZX)h6CQqVIg9+h@s99$#Y<_Bs4Z=)354+Dosem(gYPC_0C_N}Z(slR8ShKpm!@qW+C~l==qc zr)sHc%0dmNjO07y8{|>)1@am4AbB5o2iZX~WDPl*97gKQ=gcR}zcfE(e$;%A*==5K zE;pB&$CADA9AebaP@X@_ZzsnnEZJa2r*c-;7F<9CcU<7(pq<6Pq?qtU1` z^cvnV{K4>w;ZDO&gUirhm}?kgFzMgWckA!d->nbmoAlYbur8o$)UDO2wV!F<1ErLo z!hphn!hpiS|1t*5%>-`9Ykv8O-X|{X+HH36d#;HGj`i;PKutHno~5bzQfJSr7vA07 zf2r1~!407T=ksT~9+~~!V}HJ4!!gVP|M{Oj1G7|qi0$m!gJG4la7_3c%8>4%?QZy6 ziTx_05fDZHj=%y`8OIo!u=gFZtQGB^NmU#8=-u4_cYMVc;z)uY(G z4|sFCiT##Wnf(drvFV=|CLEwRz6z34128|FE;;!rqGBdB=|IuBf#kPHd-cuQ=vB-S~>y0C}kJ%Eq3zuJ%54&-VkprY+y_rM#+ ze*YTtm2#fd#g!O=??%dD*y}p9F%=_*Z-t8|Z$0$D$$}>$#gl{Wn!W`NH*Ps~32X;LkVYe-$rEB!968)XP)1Lu08F-eeXz-t?Pe2+&2rKdhT-HCu&%m`J|3tpoxJoHmDaJ-+mv_MSQga zqaVe)%6DU3*xS6y2>NxkMYooRT07TcTEi#2#(1vcu@^KMt9jhjhj)JBKJnSJ-|PSM zIdT;o8M&&YET}m%~Qa;BL`;fy?Qo$CEOPdhtI%f8pdW=$qkU%gu-G zg;H+qDxZ%VmcW5wOPZhhB|da9Y>r>N{=V1yi^BaUo(cC~?kOy&hJ)qRhaoA;+u?h1 z_iX%n7i@1Y?br4lD8(vAE`sxO7acl>{h1EW`)g13@4`pkf1r2bk>YUZh&q2EoSnDu z%5q~KV$V+H=dyAg(yplKd&(992Jo89428(*T_d?Dp zZ`(OU4_O3{z<-uJV~AQi57MUQX?0~hSlL7Bg1N9!KKD>)@1Bdtjvje$=R4{rU%Bw& z)=T|An{?kxg*x*bIG;D?Gix?nxw?4;T?&_2N{>%J@a`_v%odJ$=s; zr+?Aadth(HrGl(!aHM3~^D~_PVLtYG&!w|RE|lZXRnHqb6^@UedVKa3=WEW-Cv}aj zc%auj+zJOLTMx44!t;eEX4M^XUzt`{p*5Aj;hd7qm(Fasve!HXcJij&e_-ziM~~v; zC&S+A$p-@lCcLb^>+Hl=vaxgfo>60;6!!J+r!HPH;MiPnU#DjbnFLpko%E*WJDZ4$ zTXz5M!bf=4L^xD3@f11j!qsq2s zc(~Afvv*~(8zHam0cN`Vf}AD`PWzf_`-7S4kA-f1& zE*#0oo$k`+zy_6r>$3R+*`Jkbv-ks9xPCau%;66m!*#>>Q^W2br5(zj8ag*W6Xd+i z_YMC-)Zk zXk7-MR))VnhlY*mbpJ%W0xQ7EYksA=_drF#C*fWlNx_+s)XNL+^qkA>&Nq^G# zY{m>-mKnB7%m=yG9)5Vvi>FIX1*adLF{4wZHNkPpl$mJ+InH?53@ z3~;Q#Ff-#5b*r*<5 zKhbL72&I`$6Cf?b%sYGgW`5N7n+szMYB(@VJss=&npp*#d8#w!w?D-i8Z|g!Fi<%E z{J)tTiNU-7{zm_m{(wGB|B?PJ{WASa`Z*X0I7t6L`ce8p`d<2O`Zjtuj0yPYCYq%; z(`)D&x|%MhXVFvX3G`@s1U-xfRTA5-sAZ^3xMYt)NWH}&t-)6`?s|D_(G z?x*gd?xaH04k|#oDUPbAHc%_6rPKndj4Gw3P~)gVDu)_E87Vb+mHeDML-vqwlYb(A zN4`S-iu^fwh-HODPElAYWA+dQg5}P(5v2i038#W-ZemxTF)*-QWEfQ)kEEXgR3y~-&Kq5aMiIF3b7%>8gygVdwbCJl&K_WXF ziL5Lnh7U(#*f1o94n-m}6Nw>1kjOwGK2FnUkD`zuNhHi>BupkGj7B631|;-)By>6? zv|1!I8YBn;3AGvtm5P7?0QCGHLxa}WSt*pu6b2Lq6b2Lq6b2Lq6b2Lq6b2Lq6b2Lq z6b2LqzH|)m&;OP9|4UbFN`@5%6b2Lq6b2Lq6b2Lq6b2Lq6b2Lq6b2Lqz7PZa`hO+< z|3Z{07b^@X3@8jJ3@8jJ3@8jJ3@8jJ3@8jJ3@8kI=@{Uj{}*AGG5SfGp>gU*)N=Ae zvXeBL|Brc`>9A>o@vQN7V~*i&!w~%s^<}!(bvEsJ?cLh(n&X-~HTlFLVx{^6^&RSw zsvoLu!e7E|*kwG3ZRTm%LWIgv{8nN!5!#SK5{aLH6ut?Iz5ody@s#_U(BkJo(dT&^ zIX`1(-ER28;>Ilq^6Gs1p= z@pIeS;Cr2%9~7J>_gA8$pq#xe$T11vfH*Zp^BYu6uY-ZldAD=^02_4qyZ{BSyh`i$ zV_igOT^b53`IDvP|Cmqvrm0Q|CK3BiaheQ(wFSUS*rlI(E&HGTgRuuiYQDgJIM(Cx zIujjQDK<}uMJoaGwQ^q874_&#Q|$Cz14;P!Jg+8{ zr+gB2yF)61mp+Hp&2^y^H0??7MvdwI#pKZ1V{C2AlY)nC39YUzcKHKVw zY(lswG(Z6>1KuB+00#6o>P^@ftV6A@F&fnyRA*J+P!;RS$rkM!BxQaW>%f1FbJ`9q zrTLL&Iq?_bc4CzJS=|rlhv~7@GuT_^arz%owWcLTx8ZK%8-{H14~DC%i`Ov-;vhoH zQ{W{jv~s)JxfWN@3RDGxEcC!YwV@#$v2Tc+C~-};^|~9=8ioi}$(aw2&%wE&{KeL0 zWQsK}iOop*$anHonq?2`P=6qybzbCXAVLe}BueNBIDI~ct&{IWR;1YeCov@e*qhiO z<7^EwkrG*GCqhf*S0t{FF<CNALG!TGIr)(?T4i)G^d)8-+RJ%dpaz|^|$Nh%9fM$*-s zmtqu|gdpM18b4UH-baL3iE3Z8n|w_u)Cx9h^1lR^q~u6>>MOLIsP0Mcbvt5P$8I zv$Q0(ZWS1dflAH+fw^)wh?3-`QEN_$!;?w%JKi8gXVTQ>Z2_e$9YkoGoO~vLMVSFeBd-?7T_mvKTbn(^88wh z2`t3j_-IE`F$W`2hgIm3!q^D^oQ4thfDd*YATE&+QFzc5fJZfME{LA>I*R~ejqDIC zR26&N6LW4X0M*I}6k{*Qd0O4@EP;;@cZ~s<-D!al3n|(FMl%pAF#BjgUoIoMP+)d= zz6OsXven20_oSun0}lqiS&&`ch+7AVB69MHkdne6aelD8F9rR zM$$w;QQ7hVWvToqQm0uz5-{X62T>E$pz}J6NRJND>H`?9d4Q2t*DChvqOP?w7Z5vS zEE!3`|DA40V-7&CN)xms!r`(3Cp{ilI*~|zI`Yl%GqcP^!UM%DRU_$4B80ev)m9sk<;YF4oi@4{$jIO>N5Z)9WIZb z;{oodfrj*}Wvros8j`wMPo+l#R4Tj%q|6$K#T^^|Z7`Um}d~cmzGP0%e*FfU-$OlM+$*rxp+~!0cXeo+@B%g)g{TdOsY;)EMdQJ7W}|6raaAk7pav81Uwl8GQ9tqp2` zNWtk00wH-^U#kLyF{u(GO%RtOK}pf^@qZEa9gO}8T}=IqS^=~F?;s1!Pnye3FPT;v z-#6|tW*UBISg3znAJXUQ4(e*OA8Pk#^EE%!+(LXt+(nF4A5pJYeX6=$H3C0~kAkFR z`GvGK1h#L;jZ>Et`^U#b*ZuH5WLypz$-VK6Bw9&)KKQfDb!Vn!85DmCfns7_gds$f zhuDxFYC6GBG%W?Nb@C}L0ZVxB-+VJ5$?2-2(VEcxi|&F}_9cKNXWuWHxHUF6*0~r! z8MLUp`GQxiwp);WNahOVk@m! z{3H;`g2m>W04IgUIH}+z9q+`6CNcVZ^aSe1 zR1JBO^pk4yedaNypP6npzGn;=!iGByI{mlw1-c*U=4)TjuGO5^d`&Znc$QeAKA~=f zo0Xr!z&`^6Y%^3t89R$YE##Y=XoTo;0YnNeGO_)VKqPh;o0|Yi#*Ul>DyjYQI|0ij zqXUzyLa;hGuGQ&hJuo9$n6E56`tmjcCYvTq6r;rBF&yB@*+z@4TXY&olLG+cjE19u zcU&6)?adZDAkDvV>I5r{4%jEmo_5vf%`@P8QHQZX7MC?Uc?d5Hi8Q zv@s8i^aK)SPtvdOQcjun z>0q7PwgRg2ABPH%d$+(KzdB8jU_J0w2*wSgEKCPy3;19eOq|+~fwnr(+4m2~fW*WA z*1s8G<@DQ<9XR17CP8A0-&5@e?m*>X`R6|aqd<$3(~s*Fw*4;;7uPc zVNPQ(@9nT$;Xa2Z`B&%giAqOxN;Mtf3g(|o4cOMFlLf_e!tU6V`i zqcf=e)GYFK)f*~1=_NJhN6oW!$4$RA`HdHi-!_&QUNkuMSM>Mm=l(No3YM`I91?!|`^ z5bK~ZNWI+eBp7gxb(Wt9In!k~1Ly|5tF7s@mjODe8`4=d{?2N|FSdOB3ohfno~Uk1 zI}u9TI$dc7BsL9YTqpO+!2trEjdet5PU@yBq#TH!MzcWf_Z^2WNfS)M^ros5Usg7lHj+h{<5GOh;~)^| zd0yOLYtjR^@p8ZFS178;lf((>DWb$@3$nlYM9?DXxk8EDo+7T0%Mds}$Lc3FMN2O0o U#5Ze2*0@K9yrSC2mQRoQFBQUR;Q#;t literal 0 HcmV?d00001 diff --git a/vocode/streaming/action/dtmf.py b/vocode/streaming/action/dtmf.py index 392b8eef6..505e4a66b 100644 --- a/vocode/streaming/action/dtmf.py +++ b/vocode/streaming/action/dtmf.py @@ -9,6 +9,7 @@ ) from vocode.streaming.models.actions import ActionConfig as VocodeActionConfig from vocode.streaming.models.actions import ActionInput, ActionOutput +from vocode.streaming.utils.dtmf import KeypadEntry from vocode.streaming.utils.state_manager import ( TwilioPhoneConversationStateManager, VonagePhoneConversationStateManager, @@ -76,8 +77,19 @@ def __init__(self, action_config: DTMFVocodeActionConfig): ) async def run(self, action_input: ActionInput[DTMFParameters]) -> ActionOutput[DTMFResponse]: - logger.error("DTMF not yet supported with Twilio") - return ActionOutput( - action_type=action_input.action_config.type, - response=DTMFResponse(success=False), - ) + buttons = action_input.params.buttons + if not all(button in KeypadEntry for button in buttons): + logger.warning(f"Invalid DTMF buttons: {buttons}") + return ActionOutput( + action_type=action_input.action_config.type, + response=DTMFResponse(success=False), + ) + else: + keypad_entries = [KeypadEntry(button) for button in buttons] + self.conversation_state_manager._twilio_phone_conversation.output_device.send_dtmf_tones( + keypad_entries=keypad_entries + ) + return ActionOutput( + action_type=action_input.action_config.type, + response=DTMFResponse(success=False), + ) diff --git a/vocode/streaming/output_device/twilio_output_device.py b/vocode/streaming/output_device/twilio_output_device.py index 00926e9b0..451c6639f 100644 --- a/vocode/streaming/output_device/twilio_output_device.py +++ b/vocode/streaming/output_device/twilio_output_device.py @@ -1,10 +1,10 @@ from __future__ import annotations import asyncio +import audioop import base64 import json -import uuid -from typing import Optional, Union +from typing import List, Optional, Union from fastapi import WebSocket from fastapi.websockets import WebSocketState @@ -15,6 +15,7 @@ from vocode.streaming.output_device.audio_chunk import AudioChunk, ChunkState from vocode.streaming.telephony.constants import DEFAULT_AUDIO_ENCODING, DEFAULT_SAMPLING_RATE from vocode.streaming.utils.create_task import asyncio_create_task +from vocode.streaming.utils.dtmf import KeypadEntry, generate_dtmf_tone from vocode.streaming.utils.worker import InterruptibleEvent @@ -55,6 +56,17 @@ def interrupt(self): def enqueue_mark_message(self, mark_message: MarkMessage): self._mark_message_queue.put_nowait(mark_message) + def send_dtmf_tones(self, keypad_entries: List[KeypadEntry]): + for keypad_entry in keypad_entries: + dtmf_tone_pcm = generate_dtmf_tone(keypad_entry, sampling_rate=self.sampling_rate) + dtmf_tone_mulaw = audioop.lin2ulaw(dtmf_tone_pcm, 2) + dtmf_message = { + "event": "media", + "streamSid": self.stream_sid, + "media": {"payload": base64.b64encode(dtmf_tone_mulaw).decode("utf-8")}, + } + self._twilio_events_queue.put_nowait(json.dumps(dtmf_message)) + async def _send_twilio_messages(self): while True: try: diff --git a/vocode/streaming/utils/dtmf.py b/vocode/streaming/utils/dtmf.py new file mode 100644 index 000000000..8ca30ced4 --- /dev/null +++ b/vocode/streaming/utils/dtmf.py @@ -0,0 +1,40 @@ +from enum import Enum + +import numpy as np + + +class KeypadEntry(str, Enum): + ONE = "1" + TWO = "2" + THREE = "3" + FOUR = "4" + FIVE = "5" + SIX = "6" + SEVEN = "7" + EIGHT = "8" + NINE = "9" + ZERO = "0" + + +DTMF_FREQUENCIES = { + KeypadEntry.ONE: (697, 1209), + KeypadEntry.TWO: (697, 1336), + KeypadEntry.THREE: (697, 1477), + KeypadEntry.FOUR: (770, 1209), + KeypadEntry.FIVE: (770, 1336), + KeypadEntry.SIX: (770, 1477), + KeypadEntry.SEVEN: (852, 1209), + KeypadEntry.EIGHT: (852, 1336), + KeypadEntry.NINE: (852, 1477), + KeypadEntry.ZERO: (941, 1336), +} + + +def generate_dtmf_tone( + keypad_entry: KeypadEntry, sampling_rate: int, duration_seconds: float = 0.3 +) -> bytes: + f1, f2 = DTMF_FREQUENCIES[keypad_entry] + t = np.linspace(0, duration_seconds, int(sampling_rate * duration_seconds), endpoint=False) + tone = np.sin(2 * np.pi * f1 * t) + np.sin(2 * np.pi * f2 * t) + tone = tone * 32767 / np.max(np.abs(tone)) # Normalize to [-1, 1] + return tone.tobytes() From 0336b7c497ffdbbe9900a66a109a5e3209f45168 Mon Sep 17 00:00:00 2001 From: Ajay Raj Date: Mon, 8 Jul 2024 18:44:00 -0700 Subject: [PATCH 2/9] make it work --- vocode/streaming/action/dtmf.py | 25 ++++++++++--------- .../output_device/twilio_output_device.py | 2 +- .../utils/{dtmf.py => dtmf_utils.py} | 4 +-- 3 files changed, 16 insertions(+), 15 deletions(-) rename vocode/streaming/utils/{dtmf.py => dtmf_utils.py} (88%) diff --git a/vocode/streaming/action/dtmf.py b/vocode/streaming/action/dtmf.py index 505e4a66b..14c481854 100644 --- a/vocode/streaming/action/dtmf.py +++ b/vocode/streaming/action/dtmf.py @@ -1,4 +1,4 @@ -from typing import Type +from typing import List, Type from loguru import logger from pydantic.v1 import BaseModel, Field @@ -9,7 +9,7 @@ ) from vocode.streaming.models.actions import ActionConfig as VocodeActionConfig from vocode.streaming.models.actions import ActionInput, ActionOutput -from vocode.streaming.utils.dtmf import KeypadEntry +from vocode.streaming.utils.dtmf_utils import KeypadEntry from vocode.streaming.utils.state_manager import ( TwilioPhoneConversationStateManager, VonagePhoneConversationStateManager, @@ -78,18 +78,19 @@ def __init__(self, action_config: DTMFVocodeActionConfig): async def run(self, action_input: ActionInput[DTMFParameters]) -> ActionOutput[DTMFResponse]: buttons = action_input.params.buttons - if not all(button in KeypadEntry for button in buttons): - logger.warning(f"Invalid DTMF buttons: {buttons}") - return ActionOutput( - action_type=action_input.action_config.type, - response=DTMFResponse(success=False), - ) - else: + keypad_entries: List[KeypadEntry] + try: keypad_entries = [KeypadEntry(button) for button in buttons] - self.conversation_state_manager._twilio_phone_conversation.output_device.send_dtmf_tones( - keypad_entries=keypad_entries - ) + except ValueError: + logger.warning(f"Invalid DTMF buttons: {buttons}") return ActionOutput( action_type=action_input.action_config.type, response=DTMFResponse(success=False), ) + self.conversation_state_manager._twilio_phone_conversation.output_device.send_dtmf_tones( + keypad_entries=keypad_entries + ) + return ActionOutput( + action_type=action_input.action_config.type, + response=DTMFResponse(success=False), + ) diff --git a/vocode/streaming/output_device/twilio_output_device.py b/vocode/streaming/output_device/twilio_output_device.py index 451c6639f..4682042e6 100644 --- a/vocode/streaming/output_device/twilio_output_device.py +++ b/vocode/streaming/output_device/twilio_output_device.py @@ -15,7 +15,7 @@ from vocode.streaming.output_device.audio_chunk import AudioChunk, ChunkState from vocode.streaming.telephony.constants import DEFAULT_AUDIO_ENCODING, DEFAULT_SAMPLING_RATE from vocode.streaming.utils.create_task import asyncio_create_task -from vocode.streaming.utils.dtmf import KeypadEntry, generate_dtmf_tone +from vocode.streaming.utils.dtmf_utils import KeypadEntry, generate_dtmf_tone from vocode.streaming.utils.worker import InterruptibleEvent diff --git a/vocode/streaming/utils/dtmf.py b/vocode/streaming/utils/dtmf_utils.py similarity index 88% rename from vocode/streaming/utils/dtmf.py rename to vocode/streaming/utils/dtmf_utils.py index 8ca30ced4..28ad06eb2 100644 --- a/vocode/streaming/utils/dtmf.py +++ b/vocode/streaming/utils/dtmf_utils.py @@ -36,5 +36,5 @@ def generate_dtmf_tone( f1, f2 = DTMF_FREQUENCIES[keypad_entry] t = np.linspace(0, duration_seconds, int(sampling_rate * duration_seconds), endpoint=False) tone = np.sin(2 * np.pi * f1 * t) + np.sin(2 * np.pi * f2 * t) - tone = tone * 32767 / np.max(np.abs(tone)) # Normalize to [-1, 1] - return tone.tobytes() + tone = tone / np.max(np.abs(tone)) # Normalize to [-1, 1] + return (tone * 32767).astype(np.int16).tobytes() From 9e0959269d97850e8e885fd8ce2c2d9a576ddd33 Mon Sep 17 00:00:00 2001 From: Ajay Raj Date: Mon, 8 Jul 2024 18:45:58 -0700 Subject: [PATCH 3/9] move onus of audio encoding into utils --- .../output_device/twilio_output_device.py | 7 ++++--- vocode/streaming/utils/dtmf_utils.py | 16 ++++++++++++++-- 2 files changed, 18 insertions(+), 5 deletions(-) diff --git a/vocode/streaming/output_device/twilio_output_device.py b/vocode/streaming/output_device/twilio_output_device.py index 4682042e6..bb1cd2734 100644 --- a/vocode/streaming/output_device/twilio_output_device.py +++ b/vocode/streaming/output_device/twilio_output_device.py @@ -58,12 +58,13 @@ def enqueue_mark_message(self, mark_message: MarkMessage): def send_dtmf_tones(self, keypad_entries: List[KeypadEntry]): for keypad_entry in keypad_entries: - dtmf_tone_pcm = generate_dtmf_tone(keypad_entry, sampling_rate=self.sampling_rate) - dtmf_tone_mulaw = audioop.lin2ulaw(dtmf_tone_pcm, 2) + dtmf_tone = generate_dtmf_tone( + keypad_entry, sampling_rate=self.sampling_rate, audio_encoding=self.audio_encoding + ) dtmf_message = { "event": "media", "streamSid": self.stream_sid, - "media": {"payload": base64.b64encode(dtmf_tone_mulaw).decode("utf-8")}, + "media": {"payload": base64.b64encode(dtmf_tone).decode("utf-8")}, } self._twilio_events_queue.put_nowait(json.dumps(dtmf_message)) diff --git a/vocode/streaming/utils/dtmf_utils.py b/vocode/streaming/utils/dtmf_utils.py index 28ad06eb2..51f92c396 100644 --- a/vocode/streaming/utils/dtmf_utils.py +++ b/vocode/streaming/utils/dtmf_utils.py @@ -1,7 +1,12 @@ +import audioop from enum import Enum import numpy as np +from vocode.streaming.models.audio import AudioEncoding + +DEFAULT_DTMF_TONE_LENGTH_SECONDS = 0.3 + class KeypadEntry(str, Enum): ONE = "1" @@ -31,10 +36,17 @@ class KeypadEntry(str, Enum): def generate_dtmf_tone( - keypad_entry: KeypadEntry, sampling_rate: int, duration_seconds: float = 0.3 + keypad_entry: KeypadEntry, + sampling_rate: int, + audio_encoding: AudioEncoding, + duration_seconds: float = DEFAULT_DTMF_TONE_LENGTH_SECONDS, ) -> bytes: f1, f2 = DTMF_FREQUENCIES[keypad_entry] t = np.linspace(0, duration_seconds, int(sampling_rate * duration_seconds), endpoint=False) tone = np.sin(2 * np.pi * f1 * t) + np.sin(2 * np.pi * f2 * t) tone = tone / np.max(np.abs(tone)) # Normalize to [-1, 1] - return (tone * 32767).astype(np.int16).tobytes() + pcm = (tone * 32767).astype(np.int16).tobytes() + if audio_encoding == AudioEncoding.MULAW: + return audioop.lin2ulaw(pcm, 2) + else: + return pcm From 6d731d577257d0d4cb516288f4ac59b0b36af744 Mon Sep 17 00:00:00 2001 From: Ajay Raj Date: Mon, 8 Jul 2024 19:01:08 -0700 Subject: [PATCH 4/9] adds tests --- .coverage | Bin 77824 -> 0 bytes tests/streaming/action/test_dtmf.py | 79 ++++++++++++++++++++++++++- vocode/streaming/action/dtmf.py | 2 +- vocode/streaming/utils/dtmf_utils.py | 3 +- 4 files changed, 79 insertions(+), 5 deletions(-) delete mode 100644 .coverage diff --git a/.coverage b/.coverage deleted file mode 100644 index cd4158d3ecec6dac5523e64651fb6a13e315cdd2..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 77824 zcmeHw349aRwfBrP+BGxAfH$x$7GqY6#bGme0Sp05%)S{s9!X{o3z)O-h!wq>rS*P1~0yAt7BJX+wadS)8OPkho#D_0HTGNh8Uo1daM$ z`bIzSnf3nf`QLNTz30xoGiz4WxPqLe(dYNDLCXv*1H*Bw(qh3d%n1L7!vDxm2OC6W z2P7`p9&FQym9*bQ((5s^_Ujm3Pu9{?%ny^}OusdI40oCK>sW&wPAETx0fhmD0fmA8 z-3)|QnRVH@x%j@_LDuHxf_~P{`Gvpa!qwGPYpX45s}|H$TZChl@kR^$rcAL^S^T~Z zODpHMG`ifJ#pQLl>}=5Gby|W=T;%Gu0O#NvIyeSR9e-g7x}V=-Tn@M^$T=a2R=>-` z`a3NxT<2tTfru95wg-g+kO=2;dig6X^Vbm46Ksd9@yrF{uaZw1$}6rfx>5| z!5{N&L-e}r!b1FxZOBk~dKjB47>I5d;*2JGco5U!&kt@ld@YGL622C>-4kGibA`3kNfus0lFBqMOE0u2z>lVg$a=fXqlNEm7#= zOej*41Gv%1DG$d6+WZkmaxEtrxg+K!BXvYP1L{oCX|tvf3j72gaCse9ts(l2QRPWV(ktV5{oe4$8E>}cYWk`r2^MQo4@x;&hP zzXW9l8CC}yu-N?^oP@S?Lb3WNo6P`N>9sI6G`djsRx}WG) zXy4a1Yc6WOr70y|Cz{otsc%;osk&7Q@DsQnr?E#Os42?UVgd_s)^!0`2x4WM*-k&( zYz_GB)^?xW=isLBs~G%V)*abrTV0V&SV75&+txr3R&aP=J*73sxw+ORpSRO$ce^-m z5U%jTx-Ul9gXJUaI6zz?BVt?7*&*0Kr`OK-Iarn$09F8q z8w)_SG6Kce3v!-TH!Oqh}dmFyrLK7R|Mu*CxD*U6YZtQKo+3o;I_-DT&jY@5U7W9&_BUS65@LIB&A zKCnn(#pMBT9R+~9LPlHfg&f$O}XFDZY zxnK~CjkULMUI(btIt+m1G)2J}MCWH?1MqeSWFEp1K(r49L^&G`x2v6NaRvFc@v&0M zl)02d{-7!J2JC(pyf=Y2xgmfer^$&OmLRn546M}!pTxt|X8=q(Tpr&vLHZ3eq+cy# z4HeXom~%aq9t}{b@EVXZYakZ4hYJMwR}%n&IXwu_v*KTb00^835acu_A!(~17*r+z z<1hk@oc>FMK~-JY29#z4plp)Sq(l@xet?L9b-S4;)>b`WRmhL!2znYpl^Pu+pPzE_ z{L*wtZKW1cD@r*vN!?nlffOg)c#1&?SpwEZwuR&00AO|TtpZCxnkhtLLYmPYfoOve z0S2I7EYRAZ28a}#&L9xb2sHAiK#5uvAdE?s7-@pI90@WImk+6dKN-a;_^YBP@`3tL z`Vo2}^$__t@&{(h+(s@jpU~f?KW}o?nCLN;n7h0wfX?LxO@qgY9BGe&g$w&(R@5KMPP9n4_UC@$#P)_QB#6VTQf(YfOp1c_U>sAq=qSRB9)Gh5AA~Yd| z6oo)KKK|FF7>NxMob!hx@jr3>6-o$ZX)h6CQqVIg9+h@s99$#Y<_Bs4Z=)354+Dosem(gYPC_0C_N}Z(slR8ShKpm!@qW+C~l==qc zr)sHc%0dmNjO07y8{|>)1@am4AbB5o2iZX~WDPl*97gKQ=gcR}zcfE(e$;%A*==5K zE;pB&$CADA9AebaP@X@_ZzsnnEZJa2r*c-;7F<9CcU<7(pq<6Pq?qtU1` z^cvnV{K4>w;ZDO&gUirhm}?kgFzMgWckA!d->nbmoAlYbur8o$)UDO2wV!F<1ErLo z!hphn!hpiS|1t*5%>-`9Ykv8O-X|{X+HH36d#;HGj`i;PKutHno~5bzQfJSr7vA07 zf2r1~!407T=ksT~9+~~!V}HJ4!!gVP|M{Oj1G7|qi0$m!gJG4la7_3c%8>4%?QZy6 ziTx_05fDZHj=%y`8OIo!u=gFZtQGB^NmU#8=-u4_cYMVc;z)uY(G z4|sFCiT##Wnf(drvFV=|CLEwRz6z34128|FE;;!rqGBdB=|IuBf#kPHd-cuQ=vB-S~>y0C}kJ%Eq3zuJ%54&-VkprY+y_rM#+ ze*YTtm2#fd#g!O=??%dD*y}p9F%=_*Z-t8|Z$0$D$$}>$#gl{Wn!W`NH*Ps~32X;LkVYe-$rEB!968)XP)1Lu08F-eeXz-t?Pe2+&2rKdhT-HCu&%m`J|3tpoxJoHmDaJ-+mv_MSQga zqaVe)%6DU3*xS6y2>NxkMYooRT07TcTEi#2#(1vcu@^KMt9jhjhj)JBKJnSJ-|PSM zIdT;o8M&&YET}m%~Qa;BL`;fy?Qo$CEOPdhtI%f8pdW=$qkU%gu-G zg;H+qDxZ%VmcW5wOPZhhB|da9Y>r>N{=V1yi^BaUo(cC~?kOy&hJ)qRhaoA;+u?h1 z_iX%n7i@1Y?br4lD8(vAE`sxO7acl>{h1EW`)g13@4`pkf1r2bk>YUZh&q2EoSnDu z%5q~KV$V+H=dyAg(yplKd&(992Jo89428(*T_d?Dp zZ`(OU4_O3{z<-uJV~AQi57MUQX?0~hSlL7Bg1N9!KKD>)@1Bdtjvje$=R4{rU%Bw& z)=T|An{?kxg*x*bIG;D?Gix?nxw?4;T?&_2N{>%J@a`_v%odJ$=s; zr+?Aadth(HrGl(!aHM3~^D~_PVLtYG&!w|RE|lZXRnHqb6^@UedVKa3=WEW-Cv}aj zc%auj+zJOLTMx44!t;eEX4M^XUzt`{p*5Aj;hd7qm(Fasve!HXcJij&e_-ziM~~v; zC&S+A$p-@lCcLb^>+Hl=vaxgfo>60;6!!J+r!HPH;MiPnU#DjbnFLpko%E*WJDZ4$ zTXz5M!bf=4L^xD3@f11j!qsq2s zc(~Afvv*~(8zHam0cN`Vf}AD`PWzf_`-7S4kA-f1& zE*#0oo$k`+zy_6r>$3R+*`Jkbv-ks9xPCau%;66m!*#>>Q^W2br5(zj8ag*W6Xd+i z_YMC-)Zk zXk7-MR))VnhlY*mbpJ%W0xQ7EYksA=_drF#C*fWlNx_+s)XNL+^qkA>&Nq^G# zY{m>-mKnB7%m=yG9)5Vvi>FIX1*adLF{4wZHNkPpl$mJ+InH?53@ z3~;Q#Ff-#5b*r*<5 zKhbL72&I`$6Cf?b%sYGgW`5N7n+szMYB(@VJss=&npp*#d8#w!w?D-i8Z|g!Fi<%E z{J)tTiNU-7{zm_m{(wGB|B?PJ{WASa`Z*X0I7t6L`ce8p`d<2O`Zjtuj0yPYCYq%; z(`)D&x|%MhXVFvX3G`@s1U-xfRTA5-sAZ^3xMYt)NWH}&t-)6`?s|D_(G z?x*gd?xaH04k|#oDUPbAHc%_6rPKndj4Gw3P~)gVDu)_E87Vb+mHeDML-vqwlYb(A zN4`S-iu^fwh-HODPElAYWA+dQg5}P(5v2i038#W-ZemxTF)*-QWEfQ)kEEXgR3y~-&Kq5aMiIF3b7%>8gygVdwbCJl&K_WXF ziL5Lnh7U(#*f1o94n-m}6Nw>1kjOwGK2FnUkD`zuNhHi>BupkGj7B631|;-)By>6? zv|1!I8YBn;3AGvtm5P7?0QCGHLxa}WSt*pu6b2Lq6b2Lq6b2Lq6b2Lq6b2Lq6b2Lq z6b2LqzH|)m&;OP9|4UbFN`@5%6b2Lq6b2Lq6b2Lq6b2Lq6b2Lq6b2Lqz7PZa`hO+< z|3Z{07b^@X3@8jJ3@8jJ3@8jJ3@8jJ3@8jJ3@8kI=@{Uj{}*AGG5SfGp>gU*)N=Ae zvXeBL|Brc`>9A>o@vQN7V~*i&!w~%s^<}!(bvEsJ?cLh(n&X-~HTlFLVx{^6^&RSw zsvoLu!e7E|*kwG3ZRTm%LWIgv{8nN!5!#SK5{aLH6ut?Iz5ody@s#_U(BkJo(dT&^ zIX`1(-ER28;>Ilq^6Gs1p= z@pIeS;Cr2%9~7J>_gA8$pq#xe$T11vfH*Zp^BYu6uY-ZldAD=^02_4qyZ{BSyh`i$ zV_igOT^b53`IDvP|Cmqvrm0Q|CK3BiaheQ(wFSUS*rlI(E&HGTgRuuiYQDgJIM(Cx zIujjQDK<}uMJoaGwQ^q874_&#Q|$Cz14;P!Jg+8{ zr+gB2yF)61mp+Hp&2^y^H0??7MvdwI#pKZ1V{C2AlY)nC39YUzcKHKVw zY(lswG(Z6>1KuB+00#6o>P^@ftV6A@F&fnyRA*J+P!;RS$rkM!BxQaW>%f1FbJ`9q zrTLL&Iq?_bc4CzJS=|rlhv~7@GuT_^arz%owWcLTx8ZK%8-{H14~DC%i`Ov-;vhoH zQ{W{jv~s)JxfWN@3RDGxEcC!YwV@#$v2Tc+C~-};^|~9=8ioi}$(aw2&%wE&{KeL0 zWQsK}iOop*$anHonq?2`P=6qybzbCXAVLe}BueNBIDI~ct&{IWR;1YeCov@e*qhiO z<7^EwkrG*GCqhf*S0t{FF<CNALG!TGIr)(?T4i)G^d)8-+RJ%dpaz|^|$Nh%9fM$*-s zmtqu|gdpM18b4UH-baL3iE3Z8n|w_u)Cx9h^1lR^q~u6>>MOLIsP0Mcbvt5P$8I zv$Q0(ZWS1dflAH+fw^)wh?3-`QEN_$!;?w%JKi8gXVTQ>Z2_e$9YkoGoO~vLMVSFeBd-?7T_mvKTbn(^88wh z2`t3j_-IE`F$W`2hgIm3!q^D^oQ4thfDd*YATE&+QFzc5fJZfME{LA>I*R~ejqDIC zR26&N6LW4X0M*I}6k{*Qd0O4@EP;;@cZ~s<-D!al3n|(FMl%pAF#BjgUoIoMP+)d= zz6OsXven20_oSun0}lqiS&&`ch+7AVB69MHkdne6aelD8F9rR zM$$w;QQ7hVWvToqQm0uz5-{X62T>E$pz}J6NRJND>H`?9d4Q2t*DChvqOP?w7Z5vS zEE!3`|DA40V-7&CN)xms!r`(3Cp{ilI*~|zI`Yl%GqcP^!UM%DRU_$4B80ev)m9sk<;YF4oi@4{$jIO>N5Z)9WIZb z;{oodfrj*}Wvros8j`wMPo+l#R4Tj%q|6$K#T^^|Z7`Um}d~cmzGP0%e*FfU-$OlM+$*rxp+~!0cXeo+@B%g)g{TdOsY;)EMdQJ7W}|6raaAk7pav81Uwl8GQ9tqp2` zNWtk00wH-^U#kLyF{u(GO%RtOK}pf^@qZEa9gO}8T}=IqS^=~F?;s1!Pnye3FPT;v z-#6|tW*UBISg3znAJXUQ4(e*OA8Pk#^EE%!+(LXt+(nF4A5pJYeX6=$H3C0~kAkFR z`GvGK1h#L;jZ>Et`^U#b*ZuH5WLypz$-VK6Bw9&)KKQfDb!Vn!85DmCfns7_gds$f zhuDxFYC6GBG%W?Nb@C}L0ZVxB-+VJ5$?2-2(VEcxi|&F}_9cKNXWuWHxHUF6*0~r! z8MLUp`GQxiwp);WNahOVk@m! z{3H;`g2m>W04IgUIH}+z9q+`6CNcVZ^aSe1 zR1JBO^pk4yedaNypP6npzGn;=!iGByI{mlw1-c*U=4)TjuGO5^d`&Znc$QeAKA~=f zo0Xr!z&`^6Y%^3t89R$YE##Y=XoTo;0YnNeGO_)VKqPh;o0|Yi#*Ul>DyjYQI|0ij zqXUzyLa;hGuGQ&hJuo9$n6E56`tmjcCYvTq6r;rBF&yB@*+z@4TXY&olLG+cjE19u zcU&6)?adZDAkDvV>I5r{4%jEmo_5vf%`@P8QHQZX7MC?Uc?d5Hi8Q zv@s8i^aK)SPtvdOQcjun z>0q7PwgRg2ABPH%d$+(KzdB8jU_J0w2*wSgEKCPy3;19eOq|+~fwnr(+4m2~fW*WA z*1s8G<@DQ<9XR17CP8A0-&5@e?m*>X`R6|aqd<$3(~s*Fw*4;;7uPc zVNPQ(@9nT$;Xa2Z`B&%giAqOxN;Mtf3g(|o4cOMFlLf_e!tU6V`i zqcf=e)GYFK)f*~1=_NJhN6oW!$4$RA`HdHi-!_&QUNkuMSM>Mm=l(No3YM`I91?!|`^ z5bK~ZNWI+eBp7gxb(Wt9In!k~1Ly|5tF7s@mjODe8`4=d{?2N|FSdOB3ohfno~Uk1 zI}u9TI$dc7BsL9YTqpO+!2trEjdet5PU@yBq#TH!MzcWf_Z^2WNfS)M^ros5Usg7lHj+h{<5GOh;~)^| zd0yOLYtjR^@p8ZFS178;lf((>DWb$@3$nlYM9?DXxk8EDo+7T0%Mds}$Lc3FMN2O0o U#5Ze2*0@K9yrSC2mQRoQFBQUR;Q#;t diff --git a/tests/streaming/action/test_dtmf.py b/tests/streaming/action/test_dtmf.py index 24050c052..e4207a44b 100644 --- a/tests/streaming/action/test_dtmf.py +++ b/tests/streaming/action/test_dtmf.py @@ -1,5 +1,10 @@ +import asyncio +import base64 +import json + import pytest from aioresponses import aioresponses +from pytest_mock import MockerFixture from tests.fakedata.id import generate_uuid from vocode.streaming.action.dtmf import ( @@ -12,9 +17,15 @@ TwilioPhoneConversationActionInput, VonagePhoneConversationActionInput, ) +from vocode.streaming.models.audio import AudioEncoding from vocode.streaming.models.telephony import VonageConfig +from vocode.streaming.output_device.twilio_output_device import TwilioOutputDevice from vocode.streaming.utils import create_conversation_id -from vocode.streaming.utils.state_manager import VonagePhoneConversationStateManager +from vocode.streaming.utils.dtmf_utils import generate_dtmf_tone +from vocode.streaming.utils.state_manager import ( + TwilioPhoneConversationStateManager, + VonagePhoneConversationStateManager, +) @pytest.mark.asyncio @@ -59,12 +70,74 @@ async def test_vonage_dtmf_press_digits(mocker, mock_env): assert action_output.response.success is True +@pytest.fixture +def mock_twilio_output_device(mocker: MockerFixture): + output_device = TwilioOutputDevice() + output_device.ws = mocker.AsyncMock() + output_device.stream_sid = "stream_sid" + return output_device + + +@pytest.fixture +def mock_twilio_phone_conversation( + mocker: MockerFixture, mock_twilio_output_device: TwilioOutputDevice +): + twilio_phone_conversation_mock = mocker.MagicMock() + twilio_phone_conversation_mock.output_device = mock_twilio_output_device + return twilio_phone_conversation_mock + + @pytest.mark.asyncio -async def test_twilio_dtmf_press_digits(mocker, mock_env): +async def test_twilio_dtmf_press_digits( + mocker, mock_env, mock_twilio_phone_conversation, mock_twilio_output_device: TwilioOutputDevice +): action = TwilioDTMF(action_config=DTMFVocodeActionConfig()) digits = "1234" twilio_sid = "twilio_sid" + action.attach_conversation_state_manager( + TwilioPhoneConversationStateManager(mock_twilio_phone_conversation) + ) + + action_output = await action.run( + action_input=TwilioPhoneConversationActionInput( + action_config=DTMFVocodeActionConfig(), + conversation_id=create_conversation_id(), + params=DTMFParameters(buttons=digits), + twilio_sid=twilio_sid, + ) + ) + + mock_twilio_output_device.start() + while not mock_twilio_output_device._twilio_events_queue.empty(): + await asyncio.sleep(0.1) + + assert action_output.response.success + mock_twilio_output_device.terminate() + + for digit, call in zip(digits, mock_twilio_output_device.ws.send_text.call_args_list): + expected_dtmf = generate_dtmf_tone( + digit, sampling_rate=8000, audio_encoding=AudioEncoding.MULAW + ) + media_message = json.loads(call[0][0]) + assert media_message["streamSid"] == mock_twilio_output_device.stream_sid + assert media_message["media"] == { + "payload": base64.b64encode(expected_dtmf).decode("utf-8") + } + + +@pytest.mark.asyncio +async def test_twilio_dtmf_failure( + mocker, mock_env, mock_twilio_phone_conversation, mock_twilio_output_device: TwilioOutputDevice +): + action = TwilioDTMF(action_config=DTMFVocodeActionConfig()) + digits = "****" + twilio_sid = "twilio_sid" + + action.attach_conversation_state_manager( + TwilioPhoneConversationStateManager(mock_twilio_phone_conversation) + ) + action_output = await action.run( action_input=TwilioPhoneConversationActionInput( action_config=DTMFVocodeActionConfig(), @@ -74,4 +147,4 @@ async def test_twilio_dtmf_press_digits(mocker, mock_env): ) ) - assert action_output.response.success is False # Twilio does not support DTMF + assert not action_output.response.success diff --git a/vocode/streaming/action/dtmf.py b/vocode/streaming/action/dtmf.py index 14c481854..ff274e924 100644 --- a/vocode/streaming/action/dtmf.py +++ b/vocode/streaming/action/dtmf.py @@ -92,5 +92,5 @@ async def run(self, action_input: ActionInput[DTMFParameters]) -> ActionOutput[D ) return ActionOutput( action_type=action_input.action_config.type, - response=DTMFResponse(success=False), + response=DTMFResponse(success=True), ) diff --git a/vocode/streaming/utils/dtmf_utils.py b/vocode/streaming/utils/dtmf_utils.py index 51f92c396..fc601bc56 100644 --- a/vocode/streaming/utils/dtmf_utils.py +++ b/vocode/streaming/utils/dtmf_utils.py @@ -6,6 +6,7 @@ from vocode.streaming.models.audio import AudioEncoding DEFAULT_DTMF_TONE_LENGTH_SECONDS = 0.3 +MAX_INT = 32767 class KeypadEntry(str, Enum): @@ -45,7 +46,7 @@ def generate_dtmf_tone( t = np.linspace(0, duration_seconds, int(sampling_rate * duration_seconds), endpoint=False) tone = np.sin(2 * np.pi * f1 * t) + np.sin(2 * np.pi * f2 * t) tone = tone / np.max(np.abs(tone)) # Normalize to [-1, 1] - pcm = (tone * 32767).astype(np.int16).tobytes() + pcm = (tone * MAX_INT).astype(np.int16).tobytes() if audio_encoding == AudioEncoding.MULAW: return audioop.lin2ulaw(pcm, 2) else: From 8adbef05040fa34891378a24ec2cb1e4c444e11a Mon Sep 17 00:00:00 2001 From: Ajay Raj Date: Tue, 9 Jul 2024 11:35:36 -0700 Subject: [PATCH 5/9] adds log for pressing buttons --- vocode/streaming/output_device/twilio_output_device.py | 1 + 1 file changed, 1 insertion(+) diff --git a/vocode/streaming/output_device/twilio_output_device.py b/vocode/streaming/output_device/twilio_output_device.py index bb1cd2734..303e26988 100644 --- a/vocode/streaming/output_device/twilio_output_device.py +++ b/vocode/streaming/output_device/twilio_output_device.py @@ -58,6 +58,7 @@ def enqueue_mark_message(self, mark_message: MarkMessage): def send_dtmf_tones(self, keypad_entries: List[KeypadEntry]): for keypad_entry in keypad_entries: + logger.info(f"Sending DTMF tone {keypad_entry}") dtmf_tone = generate_dtmf_tone( keypad_entry, sampling_rate=self.sampling_rate, audio_encoding=self.audio_encoding ) From a9b52cc683a16f0919f7fd0ebe44bbafe26a4c22 Mon Sep 17 00:00:00 2001 From: Ajay Raj Date: Tue, 9 Jul 2024 11:45:11 -0700 Subject: [PATCH 6/9] adds test for dtmftonegenerator --- tests/streaming/action/test_dtmf.py | 4 +- .../test_twilio_output_device.py | 16 +++++++ vocode/streaming/action/dtmf.py | 2 +- .../output_device/twilio_output_device.py | 7 +-- vocode/streaming/utils/dtmf_utils.py | 43 ++++++++++++------- 5 files changed, 51 insertions(+), 21 deletions(-) diff --git a/tests/streaming/action/test_dtmf.py b/tests/streaming/action/test_dtmf.py index e4207a44b..f9a8b4ecc 100644 --- a/tests/streaming/action/test_dtmf.py +++ b/tests/streaming/action/test_dtmf.py @@ -21,7 +21,7 @@ from vocode.streaming.models.telephony import VonageConfig from vocode.streaming.output_device.twilio_output_device import TwilioOutputDevice from vocode.streaming.utils import create_conversation_id -from vocode.streaming.utils.dtmf_utils import generate_dtmf_tone +from vocode.streaming.utils.dtmf_utils import DTMFToneGenerator from vocode.streaming.utils.state_manager import ( TwilioPhoneConversationStateManager, VonagePhoneConversationStateManager, @@ -116,7 +116,7 @@ async def test_twilio_dtmf_press_digits( mock_twilio_output_device.terminate() for digit, call in zip(digits, mock_twilio_output_device.ws.send_text.call_args_list): - expected_dtmf = generate_dtmf_tone( + expected_dtmf = DTMFToneGenerator().generate( digit, sampling_rate=8000, audio_encoding=AudioEncoding.MULAW ) media_message = json.loads(call[0][0]) diff --git a/tests/streaming/output_device/test_twilio_output_device.py b/tests/streaming/output_device/test_twilio_output_device.py index a2c4ca3e5..e8cd69f3d 100644 --- a/tests/streaming/output_device/test_twilio_output_device.py +++ b/tests/streaming/output_device/test_twilio_output_device.py @@ -10,6 +10,8 @@ ChunkFinishedMarkMessage, TwilioOutputDevice, ) +from vocode.streaming.utils.dtmf_utils import DTMFToneGenerator, KeypadEntry +from vocode.streaming.utils.singleton import SingletonMeta from vocode.streaming.utils.worker import InterruptibleEvent @@ -137,3 +139,17 @@ def on_interrupt(): assert audio_chunk.state == ChunkState.INTERRUPTED twilio_output_device.ws.send_text.assert_not_called() + + +def test_dtmf_tone_generator_caches( + twilio_output_device: TwilioOutputDevice, mocker: MockerFixture +): + del SingletonMeta._instances[DTMFToneGenerator] + lin2ulaw_mock = mocker.patch( + "audioop.lin2ulaw", + return_value=b"ulaw_encoded", + ) + + twilio_output_device.send_dtmf_tones([KeypadEntry.ONE, KeypadEntry.ONE]) + + lin2ulaw_mock.assert_called_once() diff --git a/vocode/streaming/action/dtmf.py b/vocode/streaming/action/dtmf.py index ff274e924..ae3552c82 100644 --- a/vocode/streaming/action/dtmf.py +++ b/vocode/streaming/action/dtmf.py @@ -9,7 +9,7 @@ ) from vocode.streaming.models.actions import ActionConfig as VocodeActionConfig from vocode.streaming.models.actions import ActionInput, ActionOutput -from vocode.streaming.utils.dtmf_utils import KeypadEntry +from vocode.streaming.utils.dtmf_utils import DTMFToneGenerator, KeypadEntry from vocode.streaming.utils.state_manager import ( TwilioPhoneConversationStateManager, VonagePhoneConversationStateManager, diff --git a/vocode/streaming/output_device/twilio_output_device.py b/vocode/streaming/output_device/twilio_output_device.py index 303e26988..a7d1917ca 100644 --- a/vocode/streaming/output_device/twilio_output_device.py +++ b/vocode/streaming/output_device/twilio_output_device.py @@ -15,7 +15,7 @@ from vocode.streaming.output_device.audio_chunk import AudioChunk, ChunkState from vocode.streaming.telephony.constants import DEFAULT_AUDIO_ENCODING, DEFAULT_SAMPLING_RATE from vocode.streaming.utils.create_task import asyncio_create_task -from vocode.streaming.utils.dtmf_utils import KeypadEntry, generate_dtmf_tone +from vocode.streaming.utils.dtmf_utils import DTMFToneGenerator, KeypadEntry from vocode.streaming.utils.worker import InterruptibleEvent @@ -57,9 +57,10 @@ def enqueue_mark_message(self, mark_message: MarkMessage): self._mark_message_queue.put_nowait(mark_message) def send_dtmf_tones(self, keypad_entries: List[KeypadEntry]): + tone_generator = DTMFToneGenerator() for keypad_entry in keypad_entries: - logger.info(f"Sending DTMF tone {keypad_entry}") - dtmf_tone = generate_dtmf_tone( + logger.info(f"Sending DTMF tone {keypad_entry.value}") + dtmf_tone = tone_generator.generate( keypad_entry, sampling_rate=self.sampling_rate, audio_encoding=self.audio_encoding ) dtmf_message = { diff --git a/vocode/streaming/utils/dtmf_utils.py b/vocode/streaming/utils/dtmf_utils.py index fc601bc56..5a77b12f8 100644 --- a/vocode/streaming/utils/dtmf_utils.py +++ b/vocode/streaming/utils/dtmf_utils.py @@ -1,9 +1,12 @@ import audioop +from ast import Tuple from enum import Enum +from typing import Dict import numpy as np from vocode.streaming.models.audio import AudioEncoding +from vocode.streaming.utils.singleton import Singleton DEFAULT_DTMF_TONE_LENGTH_SECONDS = 0.3 MAX_INT = 32767 @@ -36,18 +39,28 @@ class KeypadEntry(str, Enum): } -def generate_dtmf_tone( - keypad_entry: KeypadEntry, - sampling_rate: int, - audio_encoding: AudioEncoding, - duration_seconds: float = DEFAULT_DTMF_TONE_LENGTH_SECONDS, -) -> bytes: - f1, f2 = DTMF_FREQUENCIES[keypad_entry] - t = np.linspace(0, duration_seconds, int(sampling_rate * duration_seconds), endpoint=False) - tone = np.sin(2 * np.pi * f1 * t) + np.sin(2 * np.pi * f2 * t) - tone = tone / np.max(np.abs(tone)) # Normalize to [-1, 1] - pcm = (tone * MAX_INT).astype(np.int16).tobytes() - if audio_encoding == AudioEncoding.MULAW: - return audioop.lin2ulaw(pcm, 2) - else: - return pcm +class DTMFToneGenerator(Singleton): + + def __init__(self): + self.tone_cache: Dict[Tuple[KeypadEntry, int, AudioEncoding], bytes] = {} + + def generate( + self, + keypad_entry: KeypadEntry, + sampling_rate: int, + audio_encoding: AudioEncoding, + duration_seconds: float = DEFAULT_DTMF_TONE_LENGTH_SECONDS, + ) -> bytes: + if (keypad_entry, sampling_rate, audio_encoding) in self.tone_cache: + return self.tone_cache[(keypad_entry, sampling_rate, audio_encoding)] + f1, f2 = DTMF_FREQUENCIES[keypad_entry] + t = np.linspace(0, duration_seconds, int(sampling_rate * duration_seconds), endpoint=False) + tone = np.sin(2 * np.pi * f1 * t) + np.sin(2 * np.pi * f2 * t) + tone = tone / np.max(np.abs(tone)) # Normalize to [-1, 1] + pcm = (tone * MAX_INT).astype(np.int16).tobytes() + if audio_encoding == AudioEncoding.MULAW: + output = audioop.lin2ulaw(pcm, 2) + else: + output = pcm + self.tone_cache[(keypad_entry, sampling_rate, audio_encoding)] = output + return output From 766259dc727d2ec78ea57daaf7ae7f2173767e8f Mon Sep 17 00:00:00 2001 From: Ajay Raj Date: Tue, 9 Jul 2024 11:48:09 -0700 Subject: [PATCH 7/9] fix types --- vocode/streaming/utils/dtmf_utils.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/vocode/streaming/utils/dtmf_utils.py b/vocode/streaming/utils/dtmf_utils.py index 5a77b12f8..34732dbce 100644 --- a/vocode/streaming/utils/dtmf_utils.py +++ b/vocode/streaming/utils/dtmf_utils.py @@ -1,7 +1,6 @@ import audioop -from ast import Tuple from enum import Enum -from typing import Dict +from typing import Dict, Tuple import numpy as np From 6a39c4b70e235f8fe556edfa1d93f86c3a2bc49a Mon Sep 17 00:00:00 2001 From: Ajay Raj Date: Tue, 9 Jul 2024 14:36:22 -0700 Subject: [PATCH 8/9] change response message --- tests/streaming/action/test_dtmf.py | 1 + vocode/streaming/action/dtmf.py | 14 +++++++++++--- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/tests/streaming/action/test_dtmf.py b/tests/streaming/action/test_dtmf.py index f9a8b4ecc..bef0f62e6 100644 --- a/tests/streaming/action/test_dtmf.py +++ b/tests/streaming/action/test_dtmf.py @@ -148,3 +148,4 @@ async def test_twilio_dtmf_failure( ) assert not action_output.response.success + assert action_output.response.message == "Invalid DTMF buttons, can only accept 0-9" diff --git a/vocode/streaming/action/dtmf.py b/vocode/streaming/action/dtmf.py index ae3552c82..33c3a23db 100644 --- a/vocode/streaming/action/dtmf.py +++ b/vocode/streaming/action/dtmf.py @@ -1,4 +1,4 @@ -from typing import List, Type +from typing import List, Optional, Type from loguru import logger from pydantic.v1 import BaseModel, Field @@ -22,6 +22,7 @@ class DTMFParameters(BaseModel): class DTMFResponse(BaseModel): success: bool + message: Optional[str] = None class DTMFVocodeActionConfig(VocodeActionConfig, type="action_dtmf"): # type: ignore @@ -32,7 +33,12 @@ def action_attempt_to_string(self, input: ActionInput) -> str: def action_result_to_string(self, input: ActionInput, output: ActionOutput) -> str: assert isinstance(input.params, DTMFParameters) assert isinstance(output.response, DTMFResponse) - return f"Pressed numbers {list(input.params.buttons)} successfully" + if output.response.success: + return f"Pressed numbers {list(input.params.buttons)} successfully" + else: + return ( + f"Failed to press numbers {list(input.params.buttons)}: {output.response.message}" + ) FUNCTION_DESCRIPTION = "Presses a string numbers using DTMF tones." @@ -85,7 +91,9 @@ async def run(self, action_input: ActionInput[DTMFParameters]) -> ActionOutput[D logger.warning(f"Invalid DTMF buttons: {buttons}") return ActionOutput( action_type=action_input.action_config.type, - response=DTMFResponse(success=False), + response=DTMFResponse( + success=False, message="Invalid DTMF buttons, can only accept 0-9" + ), ) self.conversation_state_manager._twilio_phone_conversation.output_device.send_dtmf_tones( keypad_entries=keypad_entries From 19e191b61e201789c02385854f00cc6deedeff54 Mon Sep 17 00:00:00 2001 From: Ajay Raj Date: Tue, 9 Jul 2024 15:11:10 -0700 Subject: [PATCH 9/9] change while condition --- tests/streaming/action/test_dtmf.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tests/streaming/action/test_dtmf.py b/tests/streaming/action/test_dtmf.py index bef0f62e6..026779c0a 100644 --- a/tests/streaming/action/test_dtmf.py +++ b/tests/streaming/action/test_dtmf.py @@ -109,8 +109,13 @@ async def test_twilio_dtmf_press_digits( ) mock_twilio_output_device.start() - while not mock_twilio_output_device._twilio_events_queue.empty(): + max_wait_seconds = 1 + waited_seconds = 0 + while mock_twilio_output_device.ws.send_text.call_count < len(digits): await asyncio.sleep(0.1) + waited_seconds += 0.1 + if waited_seconds > max_wait_seconds: + assert False, "Timed out waiting for DTMF tones to be sent" assert action_output.response.success mock_twilio_output_device.terminate()