diff --git a/poetry.lock b/poetry.lock index 08b5eb058..86492715f 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,4 +1,4 @@ -# This file is automatically @generated by Poetry 1.5.1 and should not be changed by hand. +# This file is automatically @generated by Poetry 1.6.1 and should not be changed by hand. [[package]] name = "aiohttp" @@ -839,12 +839,12 @@ files = [ google-auth = ">=2.14.1,<3.0.dev0" googleapis-common-protos = ">=1.56.2,<2.0.dev0" grpcio = [ - {version = ">=1.33.2,<2.0dev", optional = true, markers = "extra == \"grpc\""}, {version = ">=1.49.1,<2.0dev", optional = true, markers = "python_version >= \"3.11\" and extra == \"grpc\""}, + {version = ">=1.33.2,<2.0dev", optional = true, markers = "python_version < \"3.11\" and extra == \"grpc\""}, ] grpcio-status = [ - {version = ">=1.33.2,<2.0.dev0", optional = true, markers = "extra == \"grpc\""}, {version = ">=1.49.1,<2.0.dev0", optional = true, markers = "python_version >= \"3.11\" and extra == \"grpc\""}, + {version = ">=1.33.2,<2.0.dev0", optional = true, markers = "python_version < \"3.11\" and extra == \"grpc\""}, ] protobuf = ">=3.19.5,<3.20.0 || >3.20.0,<3.20.1 || >3.20.1,<4.21.0 || >4.21.0,<4.21.1 || >4.21.1,<4.21.2 || >4.21.2,<4.21.3 || >4.21.3,<4.21.4 || >4.21.4,<4.21.5 || >4.21.5,<5.0.0.dev0" requests = ">=2.18.0,<3.0.0.dev0" @@ -932,8 +932,8 @@ google-api-core = {version = ">=1.31.5,<2.0.dev0 || >2.3.0,<3.0.0dev", extras = google-cloud-core = ">=1.6.0,<3.0.0dev" google-resumable-media = ">=0.6.0,<3.0dev" grpcio = [ - {version = ">=1.47.0,<2.0dev", markers = "python_version < \"3.11\""}, {version = ">=1.49.1,<2.0dev", markers = "python_version >= \"3.11\""}, + {version = ">=1.47.0,<2.0dev", markers = "python_version < \"3.11\""}, ] packaging = ">=20.0.0" proto-plus = ">=1.15.0,<2.0.0dev" @@ -984,8 +984,8 @@ files = [ google-api-core = {version = ">=1.34.0,<2.0.dev0 || >=2.11.dev0,<3.0.0dev", extras = ["grpc"]} grpc-google-iam-v1 = ">=0.12.4,<1.0.0dev" proto-plus = [ - {version = ">=1.22.0,<2.0.0dev", markers = "python_version < \"3.11\""}, {version = ">=1.22.2,<2.0.0dev", markers = "python_version >= \"3.11\""}, + {version = ">=1.22.0,<2.0.0dev", markers = "python_version < \"3.11\""}, ] protobuf = ">=3.19.5,<3.20.0 || >3.20.0,<3.20.1 || >3.20.1,<4.21.0 || >4.21.0,<4.21.1 || >4.21.1,<4.21.2 || >4.21.2,<4.21.3 || >4.21.3,<4.21.4 || >4.21.4,<4.21.5 || >4.21.5,<5.0.0dev" @@ -1003,8 +1003,8 @@ files = [ [package.dependencies] google-api-core = {version = ">=1.34.0,<2.0.dev0 || >=2.11.dev0,<3.0.0dev", extras = ["grpc"]} proto-plus = [ - {version = ">=1.22.0,<2.0.0dev", markers = "python_version < \"3.11\""}, {version = ">=1.22.2,<2.0.0dev", markers = "python_version >= \"3.11\""}, + {version = ">=1.22.0,<2.0.0dev", markers = "python_version < \"3.11\""}, ] protobuf = ">=3.19.5,<3.20.0 || >3.20.0,<3.20.1 || >3.20.1,<4.21.0 || >4.21.0,<4.21.1 || >4.21.1,<4.21.2 || >4.21.2,<4.21.3 || >4.21.3,<4.21.4 || >4.21.4,<4.21.5 || >4.21.5,<5.0.0dev" @@ -1043,8 +1043,8 @@ files = [ [package.dependencies] google-api-core = {version = ">=1.34.0,<2.0.dev0 || >=2.11.dev0,<3.0.0dev", extras = ["grpc"]} proto-plus = [ - {version = ">=1.22.0,<2.0.0dev", markers = "python_version < \"3.11\""}, {version = ">=1.22.2,<2.0.0dev", markers = "python_version >= \"3.11\""}, + {version = ">=1.22.0,<2.0.0dev", markers = "python_version < \"3.11\""}, ] protobuf = ">=3.19.5,<3.20.0 || >3.20.0,<3.20.1 || >3.20.1,<4.21.0 || >4.21.0,<4.21.1 || >4.21.1,<4.21.2 || >4.21.2,<4.21.3 || >4.21.3,<4.21.4 || >4.21.4,<4.21.5 || >4.21.5,<5.0.0dev" @@ -1176,6 +1176,7 @@ files = [ {file = "greenlet-2.0.2-cp27-cp27m-win32.whl", hash = "sha256:6c3acb79b0bfd4fe733dff8bc62695283b57949ebcca05ae5c129eb606ff2d74"}, {file = "greenlet-2.0.2-cp27-cp27m-win_amd64.whl", hash = "sha256:283737e0da3f08bd637b5ad058507e578dd462db259f7f6e4c5c365ba4ee9343"}, {file = "greenlet-2.0.2-cp27-cp27mu-manylinux2010_x86_64.whl", hash = "sha256:d27ec7509b9c18b6d73f2f5ede2622441de812e7b1a80bbd446cb0633bd3d5ae"}, + {file = "greenlet-2.0.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:d967650d3f56af314b72df7089d96cda1083a7fc2da05b375d2bc48c82ab3f3c"}, {file = "greenlet-2.0.2-cp310-cp310-macosx_11_0_x86_64.whl", hash = "sha256:30bcf80dda7f15ac77ba5af2b961bdd9dbc77fd4ac6105cee85b0d0a5fcf74df"}, {file = "greenlet-2.0.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:26fbfce90728d82bc9e6c38ea4d038cba20b7faf8a0ca53a9c07b67318d46088"}, {file = "greenlet-2.0.2-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9190f09060ea4debddd24665d6804b995a9c122ef5917ab26e1566dcc712ceeb"}, @@ -1184,6 +1185,7 @@ files = [ {file = "greenlet-2.0.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:76ae285c8104046b3a7f06b42f29c7b73f77683df18c49ab5af7983994c2dd91"}, {file = "greenlet-2.0.2-cp310-cp310-win_amd64.whl", hash = "sha256:2d4686f195e32d36b4d7cf2d166857dbd0ee9f3d20ae349b6bf8afc8485b3645"}, {file = "greenlet-2.0.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:c4302695ad8027363e96311df24ee28978162cdcdd2006476c43970b384a244c"}, + {file = "greenlet-2.0.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:d4606a527e30548153be1a9f155f4e283d109ffba663a15856089fb55f933e47"}, {file = "greenlet-2.0.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c48f54ef8e05f04d6eff74b8233f6063cb1ed960243eacc474ee73a2ea8573ca"}, {file = "greenlet-2.0.2-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a1846f1b999e78e13837c93c778dcfc3365902cfb8d1bdb7dd73ead37059f0d0"}, {file = "greenlet-2.0.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3a06ad5312349fec0ab944664b01d26f8d1f05009566339ac6f63f56589bc1a2"}, @@ -1213,6 +1215,7 @@ files = [ {file = "greenlet-2.0.2-cp37-cp37m-win32.whl", hash = "sha256:3f6ea9bd35eb450837a3d80e77b517ea5bc56b4647f5502cd28de13675ee12f7"}, {file = "greenlet-2.0.2-cp37-cp37m-win_amd64.whl", hash = "sha256:7492e2b7bd7c9b9916388d9df23fa49d9b88ac0640db0a5b4ecc2b653bf451e3"}, {file = "greenlet-2.0.2-cp38-cp38-macosx_10_15_x86_64.whl", hash = "sha256:b864ba53912b6c3ab6bcb2beb19f19edd01a6bfcbdfe1f37ddd1778abfe75a30"}, + {file = "greenlet-2.0.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:1087300cf9700bbf455b1b97e24db18f2f77b55302a68272c56209d5587c12d1"}, {file = "greenlet-2.0.2-cp38-cp38-manylinux2010_x86_64.whl", hash = "sha256:ba2956617f1c42598a308a84c6cf021a90ff3862eddafd20c3333d50f0edb45b"}, {file = "greenlet-2.0.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fc3a569657468b6f3fb60587e48356fe512c1754ca05a564f11366ac9e306526"}, {file = "greenlet-2.0.2-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8eab883b3b2a38cc1e050819ef06a7e6344d4a990d24d45bc6f2cf959045a45b"}, @@ -1221,6 +1224,7 @@ files = [ {file = "greenlet-2.0.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:b0ef99cdbe2b682b9ccbb964743a6aca37905fda5e0452e5ee239b1654d37f2a"}, {file = "greenlet-2.0.2-cp38-cp38-win32.whl", hash = "sha256:b80f600eddddce72320dbbc8e3784d16bd3fb7b517e82476d8da921f27d4b249"}, {file = "greenlet-2.0.2-cp38-cp38-win_amd64.whl", hash = "sha256:4d2e11331fc0c02b6e84b0d28ece3a36e0548ee1a1ce9ddde03752d9b79bba40"}, + {file = "greenlet-2.0.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:8512a0c38cfd4e66a858ddd1b17705587900dd760c6003998e9472b77b56d417"}, {file = "greenlet-2.0.2-cp39-cp39-macosx_11_0_x86_64.whl", hash = "sha256:88d9ab96491d38a5ab7c56dd7a3cc37d83336ecc564e4e8816dbed12e5aaefc8"}, {file = "greenlet-2.0.2-cp39-cp39-manylinux2010_x86_64.whl", hash = "sha256:561091a7be172ab497a3527602d467e2b3fbe75f9e783d8b8ce403fa414f71a6"}, {file = "greenlet-2.0.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:971ce5e14dc5e73715755d0ca2975ac88cfdaefcaab078a284fea6cfabf866df"}, @@ -1646,6 +1650,16 @@ files = [ {file = "MarkupSafe-2.1.3-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:5bbe06f8eeafd38e5d0a4894ffec89378b6c6a625ff57e3028921f8ff59318ac"}, {file = "MarkupSafe-2.1.3-cp311-cp311-win32.whl", hash = "sha256:dd15ff04ffd7e05ffcb7fe79f1b98041b8ea30ae9234aed2a9168b5797c3effb"}, {file = "MarkupSafe-2.1.3-cp311-cp311-win_amd64.whl", hash = "sha256:134da1eca9ec0ae528110ccc9e48041e0828d79f24121a1a146161103c76e686"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:f698de3fd0c4e6972b92290a45bd9b1536bffe8c6759c62471efaa8acb4c37bc"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:aa57bd9cf8ae831a362185ee444e15a93ecb2e344c8e52e4d721ea3ab6ef1823"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ffcc3f7c66b5f5b7931a5aa68fc9cecc51e685ef90282f4a82f0f5e9b704ad11"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:47d4f1c5f80fc62fdd7777d0d40a2e9dda0a05883ab11374334f6c4de38adffd"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1f67c7038d560d92149c060157d623c542173016c4babc0c1913cca0564b9939"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:9aad3c1755095ce347e26488214ef77e0485a3c34a50c5a5e2471dff60b9dd9c"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:14ff806850827afd6b07a5f32bd917fb7f45b046ba40c57abdb636674a8b559c"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8f9293864fe09b8149f0cc42ce56e3f0e54de883a9de90cd427f191c346eb2e1"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-win32.whl", hash = "sha256:715d3562f79d540f251b99ebd6d8baa547118974341db04f5ad06d5ea3eb8007"}, + {file = "MarkupSafe-2.1.3-cp312-cp312-win_amd64.whl", hash = "sha256:1b8dd8c3fd14349433c79fa8abeb573a55fc0fdd769133baac1f5e07abf54aeb"}, {file = "MarkupSafe-2.1.3-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:8e254ae696c88d98da6555f5ace2279cf7cd5b3f52be2b5cf97feafe883b58d2"}, {file = "MarkupSafe-2.1.3-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cb0932dc158471523c9637e807d9bfb93e06a95cbf010f1a38b98623b929ef2b"}, {file = "MarkupSafe-2.1.3-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9402b03f1a1b4dc4c19845e5c749e3ab82d5078d16a2a4c2cd2df62d57bb0707"}, @@ -2569,6 +2583,7 @@ files = [ {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:69b023b2b4daa7548bcfbd4aa3da05b3a74b772db9e23b982788168117739938"}, {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:81e0b275a9ecc9c0c0c07b4b90ba548307583c125f54d5b6946cfee6360c733d"}, {file = "PyYAML-6.0.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ba336e390cd8e4d1739f42dfe9bb83a3cc2e80f567d8805e11b46f4a943f5515"}, + {file = "PyYAML-6.0.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:326c013efe8048858a6d312ddd31d56e468118ad4cdeda36c719bf5bb6192290"}, {file = "PyYAML-6.0.1-cp310-cp310-win32.whl", hash = "sha256:bd4af7373a854424dabd882decdc5579653d7868b8fb26dc7d0e99f823aa5924"}, {file = "PyYAML-6.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:fd1592b3fdf65fff2ad0004b5e363300ef59ced41c2e6b3a99d4089fa8c5435d"}, {file = "PyYAML-6.0.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6965a7bc3cf88e5a1c3bd2e0b5c22f8d677dc88a455344035f03399034eb3007"}, @@ -2576,8 +2591,15 @@ files = [ {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:42f8152b8dbc4fe7d96729ec2b99c7097d656dc1213a3229ca5383f973a5ed6d"}, {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:062582fca9fabdd2c8b54a3ef1c978d786e0f6b3a1510e0ac93ef59e0ddae2bc"}, {file = "PyYAML-6.0.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d2b04aac4d386b172d5b9692e2d2da8de7bfb6c387fa4f801fbf6fb2e6ba4673"}, + {file = "PyYAML-6.0.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:e7d73685e87afe9f3b36c799222440d6cf362062f78be1013661b00c5c6f678b"}, {file = "PyYAML-6.0.1-cp311-cp311-win32.whl", hash = "sha256:1635fd110e8d85d55237ab316b5b011de701ea0f29d07611174a1b42f1444741"}, {file = "PyYAML-6.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34"}, + {file = "PyYAML-6.0.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28"}, + {file = "PyYAML-6.0.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9"}, + {file = "PyYAML-6.0.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0"}, + {file = "PyYAML-6.0.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4"}, + {file = "PyYAML-6.0.1-cp312-cp312-win32.whl", hash = "sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54"}, + {file = "PyYAML-6.0.1-cp312-cp312-win_amd64.whl", hash = "sha256:0d3304d8c0adc42be59c5f8a4d9e3d7379e6955ad754aa9d6ab7a398b59dd1df"}, {file = "PyYAML-6.0.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:50550eb667afee136e9a77d6dc71ae76a44df8b3e51e41b77f6de2932bfe0f47"}, {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1fe35611261b29bd1de0070f0b2f47cb6ff71fa6595c077e42bd0c419fa27b98"}, {file = "PyYAML-6.0.1-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:704219a11b772aea0d8ecd7058d0082713c3562b4e271b849ad7dc4a5c90c13c"}, @@ -2594,6 +2616,7 @@ files = [ {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a0cd17c15d3bb3fa06978b4e8958dcdc6e0174ccea823003a106c7d4d7899ac5"}, {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:28c119d996beec18c05208a8bd78cbe4007878c6dd15091efb73a30e90539696"}, {file = "PyYAML-6.0.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7e07cbde391ba96ab58e532ff4803f79c4129397514e1413a7dc761ccd755735"}, + {file = "PyYAML-6.0.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:49a183be227561de579b4a36efbb21b3eab9651dd81b1858589f796549873dd6"}, {file = "PyYAML-6.0.1-cp38-cp38-win32.whl", hash = "sha256:184c5108a2aca3c5b3d3bf9395d50893a7ab82a38004c8f61c258d4428e80206"}, {file = "PyYAML-6.0.1-cp38-cp38-win_amd64.whl", hash = "sha256:1e2722cc9fbb45d9b87631ac70924c11d3a401b2d7f410cc0e3bbf249f2dca62"}, {file = "PyYAML-6.0.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9eb6caa9a297fc2c2fb8862bc5370d0303ddba53ba97e71f08023b6cd73d16a8"}, @@ -2601,6 +2624,7 @@ files = [ {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5773183b6446b2c99bb77e77595dd486303b4faab2b086e7b17bc6bef28865f6"}, {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b786eecbdf8499b9ca1d697215862083bd6d2a99965554781d0d8d1ad31e13a0"}, {file = "PyYAML-6.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bc1bf2925a1ecd43da378f4db9e4f799775d6367bdb94671027b73b393a7c42c"}, + {file = "PyYAML-6.0.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:04ac92ad1925b2cff1db0cfebffb6ffc43457495c9b3c39d3fcae417d7125dc5"}, {file = "PyYAML-6.0.1-cp39-cp39-win32.whl", hash = "sha256:faca3bdcf85b2fc05d06ff3fbc1f83e1391b3e724afa3feba7d13eeab355484c"}, {file = "PyYAML-6.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:510c9deebc5c0225e8c96813043e62b680ba2f9c50a08d3724c7f28a747d1486"}, {file = "PyYAML-6.0.1.tar.gz", hash = "sha256:bfdf460b1736c775f2ba9f6a92bca30bc2095067b8a9d77876d1fad6cc3b4a43"}, @@ -2993,7 +3017,7 @@ files = [ ] [package.dependencies] -greenlet = {version = "!=0.4.17", markers = "platform_machine == \"win32\" or platform_machine == \"WIN32\" or platform_machine == \"AMD64\" or platform_machine == \"amd64\" or platform_machine == \"x86_64\" or platform_machine == \"ppc64le\" or platform_machine == \"aarch64\""} +greenlet = {version = "!=0.4.17", markers = "platform_machine == \"aarch64\" or platform_machine == \"ppc64le\" or platform_machine == \"x86_64\" or platform_machine == \"amd64\" or platform_machine == \"AMD64\" or platform_machine == \"win32\" or platform_machine == \"WIN32\""} typing-extensions = ">=4.2.0" [package.extras] @@ -3387,6 +3411,19 @@ files = [ {file = "wcwidth-0.2.6.tar.gz", hash = "sha256:a5220780a404dbe3353789870978e472cfe477761f06ee55077256e509b156d0"}, ] +[[package]] +name = "webrtcvad" +version = "2.0.10" +description = "Python interface to the Google WebRTC Voice Activity Detector (VAD)" +optional = false +python-versions = "*" +files = [ + {file = "webrtcvad-2.0.10.tar.gz", hash = "sha256:f1bed2fb25b63fb7b1a55d64090c993c9c9167b28485ae0bcdd81cf6ede96aea"}, +] + +[package.extras] +dev = ["check-manifest", "memory_profiler", "nose", "psutil", "unittest2", "zest.releaser"] + [[package]] name = "websocket-client" version = "0.59.0" @@ -3677,4 +3714,4 @@ transcribers = ["google-cloud-speech"] [metadata] lock-version = "2.0" python-versions = ">=3.8.1,<3.12" -content-hash = "60ee4d2395b06922d238968909c009a25d709310e7eee66ddc4f5fb16463e74e" +content-hash = "3d1650349cfb2d26aaad1e1d7dc1783e275d6a215c150869d2fe8860943805c8" diff --git a/pyproject.toml b/pyproject.toml index 8236c0e5e..005a60e80 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -42,6 +42,7 @@ langchain = "^0.0.198" google-cloud-aiplatform = {version = "^1.26.0", optional = true} miniaudio = "^1.59" boto3 = "^1.28.28" +webrtcvad = "^2.0.10" [tool.poetry.group.lint.dependencies] diff --git a/vocode/streaming/models/telephony.py b/vocode/streaming/models/telephony.py index 27521711a..2522cce5c 100644 --- a/vocode/streaming/models/telephony.py +++ b/vocode/streaming/models/telephony.py @@ -12,6 +12,7 @@ PunctuationEndpointingConfig, TranscriberConfig, ) +from vocode.streaming.report.base_call_report import CallReporterConfig from vocode.streaming.telephony.constants import ( DEFAULT_AUDIO_ENCODING, DEFAULT_CHUNK_SIZE, @@ -97,6 +98,7 @@ class BaseCallConfig(TypedModel, type=CallConfigType.BASE.value): transcriber_config: TranscriberConfig agent_config: AgentConfig synthesizer_config: SynthesizerConfig + call_reporter_config: Optional[CallReporterConfig] = None from_phone: str to_phone: str diff --git a/vocode/streaming/models/transcriber.py b/vocode/streaming/models/transcriber.py index 9fc6ed5ef..1d4906117 100644 --- a/vocode/streaming/models/transcriber.py +++ b/vocode/streaming/models/transcriber.py @@ -13,6 +13,7 @@ from .audio_encoding import AudioEncoding from .model import TypedModel from vocode.utils.context_tracker import BaseContextTrackerConfig +from ...utils.voice_activity_detection import BaseVoiceActivityDetectorConfig AZURE_DEFAULT_LANGUAGE = "en-US" @@ -44,8 +45,8 @@ class TimeEndpointingConfig(EndpointingConfig, type=EndpointingType.TIME_BASED): class PunctuationEndpointingConfig( EndpointingConfig, type=EndpointingType.PUNCTUATION_BASED -): - time_cutoff_seconds: float = 0.4 +): + time_cutoff_seconds: float = 0.4 class TranscriberConfig(TypedModel, type=TranscriberType.BASE.value): @@ -57,9 +58,10 @@ class TranscriberConfig(TypedModel, type=TranscriberType.BASE.value): min_interrupt_confidence: Optional[float] = None mute_during_speech: bool = False context_tracker_config: Optional[BaseContextTrackerConfig] = None + voice_activity_detector_config: Optional[BaseVoiceActivityDetectorConfig] = None interrupt_on_blockers: bool = False skip_on_back_track_audio: bool = False - minimum_speaking_duration_to_interupt: float = 0 + minimum_speaking_duration_to_interrupt: float = 0 @validator("min_interrupt_confidence") def min_interrupt_confidence_must_be_between_0_and_1(cls, v): diff --git a/vocode/streaming/models/transcript.py b/vocode/streaming/models/transcript.py index 311514ab9..092e00f46 100644 --- a/vocode/streaming/models/transcript.py +++ b/vocode/streaming/models/transcript.py @@ -1,10 +1,10 @@ import time -from typing import Any, Dict, List, Optional, Union +from typing import List, Optional + from pydantic import BaseModel, Field -from enum import Enum + from vocode.streaming.models.actions import ActionInput, ActionOutput from vocode.streaming.models.events import ActionEvent, Sender, Event, EventType - from vocode.streaming.utils.events_manager import EventsManager diff --git a/vocode/utils/context_tracker/base_context_tracker.py b/vocode/streaming/report/__init__.py similarity index 100% rename from vocode/utils/context_tracker/base_context_tracker.py rename to vocode/streaming/report/__init__.py diff --git a/vocode/streaming/report/api_call_reporter.py b/vocode/streaming/report/api_call_reporter.py new file mode 100644 index 000000000..5388a3091 --- /dev/null +++ b/vocode/streaming/report/api_call_reporter.py @@ -0,0 +1,27 @@ +import logging +from typing import Optional + +import requests + +from vocode.streaming.models.transcript import Transcript +from vocode.streaming.report.base_call_report import BaseCallReporter, CallReporterType, CallReporterConfig + + +class ApiCallReporterConfig(CallReporterConfig, type=CallReporterType.API.value): + url: str + + +class ApiCallReporter(BaseCallReporter[ApiCallReporterConfig]): + def __init__(self, config: ApiCallReporterConfig, logger: Optional[logging.Logger] = None): + super().__init__(config, logger) + + def report(self, conversation_id: str, transcript: Transcript): + logs = self.get_event_logs(transcript) + data = { + "conversation_id": conversation_id, + "logs": logs, + "start_time": transcript.start_time, + } + self.logger.debug(f"Data to call reporter: {data}") + response = requests.post(self.config.url, json=data) + self.logger.debug(f"Response from call reporter: {response}") diff --git a/vocode/streaming/report/base_call_report.py b/vocode/streaming/report/base_call_report.py new file mode 100644 index 000000000..5abaafe1c --- /dev/null +++ b/vocode/streaming/report/base_call_report.py @@ -0,0 +1,44 @@ +import logging +from enum import Enum +from typing import TypeVar, Generic, Optional + +from vocode.streaming.models.model import TypedModel +from vocode.streaming.models.transcript import Transcript, Message + + +class CallReporterType(str, Enum): + BASE = "base_call_report" + API = "api_call_report" + + +class CallReporterConfig(TypedModel, type=CallReporterType.BASE.value): + pass + + +CallReporterConfigType = TypeVar("CallReporterConfigType", bound=CallReporterConfig) + + +class BaseCallReporter(Generic[CallReporterConfigType]): + def __init__(self, config: CallReporterConfigType, logger: Optional[logging.Logger] = None): + self.logger: logging.Logger = logger or logging.getLogger(__name__) + self.config = config + + def get_config(self) -> CallReporterConfig: + return self.config + + def report(self, conversation_id: str, transcript: Transcript): + raise NotImplementedError + + @staticmethod + def get_event_logs(transcript): + logs = [] + for event_log in transcript.event_logs: + if isinstance(event_log, Message): + log = { + 'text': event_log.text, + 'sender': event_log.sender.value, + 'timestamp': event_log.timestamp, + 'confidence': event_log.confidence, + } + logs.append(log) + return logs diff --git a/vocode/streaming/report/factory.py b/vocode/streaming/report/factory.py new file mode 100644 index 000000000..e9fcf1ac1 --- /dev/null +++ b/vocode/streaming/report/factory.py @@ -0,0 +1,12 @@ +import logging +from typing import Optional + +from vocode.streaming.report.api_call_reporter import ApiCallReporterConfig, ApiCallReporter +from vocode.streaming.report.base_call_report import CallReporterConfig + + +class CallReporterFactory: + @staticmethod + def create_call_reporter(config: CallReporterConfig, logger: Optional[logging.Logger] = None): + if isinstance(config, ApiCallReporterConfig): + return ApiCallReporter(config, logger) diff --git a/vocode/streaming/streaming_conversation.py b/vocode/streaming/streaming_conversation.py index 8f044a1ac..018d7260f 100644 --- a/vocode/streaming/streaming_conversation.py +++ b/vocode/streaming/streaming_conversation.py @@ -39,6 +39,7 @@ TranscriptCompleteEvent, ) from vocode.streaming.output_device.base_output_device import BaseOutputDevice +from vocode.streaming.report.base_call_report import CallReporterConfig, BaseCallReporter from vocode.streaming.response_worker.random_response import RandomAudioManager from vocode.streaming.synthesizer.base_synthesizer import ( BaseSynthesizer, @@ -47,7 +48,7 @@ from vocode.streaming.telephony.noise_canceler.base_noise_canceler import BaseNoiseCanceler from vocode.streaming.transcriber.base_transcriber import ( Transcription, - BaseTranscriber, + BaseTranscriber, HUMAN_ACTIVITY_DETECTED, ) from vocode.streaming.utils import create_conversation_id, get_chunk_size_per_second from vocode.streaming.utils.conversation_logger_adapter import wrap_logger @@ -138,7 +139,9 @@ async def process(self, transcription: Transcription): ) self.conversation.is_human_speaking = not transcription.is_final if transcription.is_final: - # we use getattr here to avoid the dependency cycle between VonageCall and StreamingConversation + if self.conversation.transcriber.transcriber_config.voice_activity_detector_config and \ + transcription.message == HUMAN_ACTIVITY_DETECTED: + return event = self.interruptable_event_factory.create_interruptable_event( TranscriptionAgentInput( transcription=transcription, @@ -185,7 +188,6 @@ async def process(self, item: InterruptableAgentResponseEvent[AgentResponse]): return try: agent_response = item.payload - self.conversation.logger.debug("Got agent response: {}".format(agent_response)) if isinstance(agent_response, AgentResponseFillerAudio): self.conversation.random_audio_manager.sync_send_filler_audio(item.agent_response_tracker) return @@ -296,6 +298,7 @@ def __init__( agent: BaseAgent, synthesizer: BaseSynthesizer, noise_canceler: Optional[BaseNoiseCanceler] = None, + call_reporter: Optional[BaseCallReporter] = None, conversation_id: Optional[str] = None, per_chunk_allowance_seconds: float = PER_CHUNK_ALLOWANCE_SECONDS, events_manager: Optional[EventsManager] = None, @@ -307,6 +310,7 @@ def __init__( logger or logging.getLogger(__name__), conversation_id=self.id, ) + self.call_reporter = call_reporter self.output_device = output_device self.transcriber = transcriber self.agent = agent @@ -494,9 +498,12 @@ def broadcast_interrupt(self): return num_interrupts > 0 def is_interrupt(self, transcription: Transcription): - return transcription.confidence >= ( - self.transcriber.get_transcriber_config().min_interrupt_confidence or 0 - ) + interrupt_by_confidence = transcription.confidence >= ( + self.transcriber.get_transcriber_config().min_interrupt_confidence or 0) + interrupt_by_vad = ( + self.transcriber.transcriber_config.voice_activity_detector_config and + transcription.message == HUMAN_ACTIVITY_DETECTED) + return interrupt_by_confidence or interrupt_by_vad async def send_speech_to_output( self, @@ -579,6 +586,10 @@ async def send_speech_to_output( def mark_terminated(self): self.active = False + def report_call(self): + if self.call_reporter: + self.call_reporter.report(self.id, self.transcript) + async def terminate(self): self.mark_terminated() self.broadcast_interrupt() @@ -612,6 +623,7 @@ async def terminate(self): self.logger.debug("Terminating speech transcriber") self.transcriber.terminate() self.logger.debug("Terminating transcriptions worker") + self.report_call() self.transcriptions_worker.terminate() self.logger.debug("Terminating final transcriptions worker") self.agent_responses_worker.terminate() diff --git a/vocode/streaming/synthesizer/eleven_labs_synthesizer.py b/vocode/streaming/synthesizer/eleven_labs_synthesizer.py index 3cc019923..ec86e4d11 100644 --- a/vocode/streaming/synthesizer/eleven_labs_synthesizer.py +++ b/vocode/streaming/synthesizer/eleven_labs_synthesizer.py @@ -174,9 +174,6 @@ async def get_audio_data_from_cache_or_download(self, phrase: BaseMessage, base_ ) filler_audio_path = os.path.join(base_path, f"{cache_key}.wav") if not os.path.exists(filler_audio_path): - print('$#@$!#@$!@#'*10) - print(filler_audio_path) - print('$#@$!#@$!@#'*10) self.logger.debug(f"Generating cached audio for {phrase.text}") audio_data = await self.download_filler_audio_data(phrase) diff --git a/vocode/streaming/telephony/conversation/call.py b/vocode/streaming/telephony/conversation/call.py index de4aa8366..cd808e112 100644 --- a/vocode/streaming/telephony/conversation/call.py +++ b/vocode/streaming/telephony/conversation/call.py @@ -14,6 +14,8 @@ ) from vocode.streaming.output_device.twilio_output_device import TwilioOutputDevice from vocode.streaming.output_device.vonage_output_device import VonageOutputDevice +from vocode.streaming.report.base_call_report import CallReporterConfig +from vocode.streaming.report.factory import CallReporterFactory from vocode.streaming.streaming_conversation import StreamingConversation from vocode.streaming.synthesizer.factory import SynthesizerFactory from vocode.streaming.telephony.config_manager.base_config_manager import ( @@ -47,9 +49,11 @@ def __init__( agent_factory: AgentFactory = AgentFactory(), synthesizer_factory: SynthesizerFactory = SynthesizerFactory(), noise_canceler_factory: NoiseCancelerFactory = NoiseCancelerFactory(), + call_reporter_factory: CallReporterFactory = CallReporterFactory(), events_manager: Optional[EventsManager] = None, logger: Optional[logging.Logger] = None, noise_canceling_config: Optional[NoiseCancelingConfig] = None, + call_reporter_config: Optional[CallReporterConfig] = None, ): conversation_id = conversation_id or create_conversation_id() logger = wrap_logger( @@ -67,6 +71,7 @@ def __init__( agent_factory.create_agent(agent_config, logger=logger), synthesizer_factory.create_synthesizer(synthesizer_config, logger=logger), noise_canceler_factory.create_noise_canceler(noise_canceling_config, logger=logger), + call_reporter_factory.create_call_reporter(call_reporter_config, logger=logger), conversation_id=conversation_id, per_chunk_allowance_seconds=0.01, events_manager=events_manager, diff --git a/vocode/streaming/telephony/conversation/outbound_call.py b/vocode/streaming/telephony/conversation/outbound_call.py index d43f20d21..c3dbcc35d 100644 --- a/vocode/streaming/telephony/conversation/outbound_call.py +++ b/vocode/streaming/telephony/conversation/outbound_call.py @@ -16,6 +16,7 @@ from vocode.streaming.models.transcriber import ( TranscriberConfig, ) +from vocode.streaming.report.base_call_report import CallReporterConfig from vocode.streaming.telephony.client.base_telephony_client import BaseTelephonyClient from vocode.streaming.telephony.client.twilio_client import TwilioClient from vocode.streaming.telephony.client.vonage_client import VonageClient @@ -27,24 +28,25 @@ class OutboundCall: def __init__( - self, - base_url: str, - to_phone: str, - from_phone: str, - config_manager: BaseConfigManager, - agent_config: AgentConfig, - twilio_config: Optional[TwilioConfig] = None, - vonage_config: Optional[VonageConfig] = None, - transcriber_config: Optional[TranscriberConfig] = None, - synthesizer_config: Optional[SynthesizerConfig] = None, - conversation_id: Optional[str] = None, - logger: Optional[logging.Logger] = None, - mobile_only: bool = True, - digits: Optional[ - str - ] = None, # Keys to press when the call connects, see send_digits https://www.twilio.com/docs/voice/api/call-resource#create-a-call-resource - output_to_speaker: bool = False, - ): + self, + base_url: str, + to_phone: str, + from_phone: str, + config_manager: BaseConfigManager, + agent_config: AgentConfig, + twilio_config: Optional[TwilioConfig] = None, + vonage_config: Optional[VonageConfig] = None, + transcriber_config: Optional[TranscriberConfig] = None, + synthesizer_config: Optional[SynthesizerConfig] = None, + conversation_id: Optional[str] = None, + logger: Optional[logging.Logger] = None, + mobile_only: bool = True, + digits: Optional[ + str + ] = None, + # Keys to press when the call connects, see send_digits https://www.twilio.com/docs/voice/api/call-resource#create-a-call-resource + output_to_speaker: bool = False, + call_reporter_config: [CallReporterConfig] = None): self.base_url = base_url self.to_phone = to_phone self.digits = digits @@ -72,6 +74,7 @@ def __init__( self.synthesizer_config = self.create_synthesizer_config(synthesizer_config) self.telephony_id = None self.output_to_speaker = output_to_speaker + self.call_reporter_config = call_reporter_config def create_telephony_client(self) -> BaseTelephonyClient: if self.twilio_config is not None: @@ -86,7 +89,7 @@ def create_telephony_client(self) -> BaseTelephonyClient: raise ValueError("No telephony config provided") def create_transcriber_config( - self, transcriber_config_override: Optional[TranscriberConfig] + self, transcriber_config_override: Optional[TranscriberConfig] ) -> TranscriberConfig: if transcriber_config_override is not None: return transcriber_config_override @@ -98,7 +101,7 @@ def create_transcriber_config( raise ValueError("No telephony config provided") def create_synthesizer_config( - self, synthesizer_config_override: Optional[SynthesizerConfig] + self, synthesizer_config_override: Optional[SynthesizerConfig] ) -> SynthesizerConfig: if synthesizer_config_override is not None: return synthesizer_config_override @@ -132,6 +135,7 @@ async def start(self): twilio_sid=self.telephony_id, from_phone=self.from_phone, to_phone=self.to_phone, + call_reporter_config=self.call_reporter_config, ) elif isinstance(self.telephony_client, VonageClient): call_config = VonageCallConfig( @@ -143,6 +147,7 @@ async def start(self): from_phone=self.from_phone, to_phone=self.to_phone, output_to_speaker=self.output_to_speaker, + call_reporter_config=self.call_reporter_config, ) else: raise ValueError("Unknown telephony client") diff --git a/vocode/streaming/telephony/conversation/twilio_call.py b/vocode/streaming/telephony/conversation/twilio_call.py index 94af3350f..b4f53bae1 100644 --- a/vocode/streaming/telephony/conversation/twilio_call.py +++ b/vocode/streaming/telephony/conversation/twilio_call.py @@ -18,6 +18,7 @@ from vocode.streaming.models.transcriber import ( TranscriberConfig, ) +from vocode.streaming.report.base_call_report import CallReporterConfig from vocode.streaming.synthesizer.factory import SynthesizerFactory from vocode.streaming.telephony.client.twilio_client import TwilioClient from vocode.streaming.telephony.config_manager.base_config_manager import ( @@ -51,6 +52,7 @@ def __init__( synthesizer_factory: SynthesizerFactory = SynthesizerFactory(), events_manager: Optional[EventsManager] = None, logger: Optional[logging.Logger] = None, + call_reporter_config: Optional[CallReporterConfig] = None, ): noise_canceling_config = twilio_config.noise_canceling_config if twilio_config else None super().__init__( @@ -69,6 +71,7 @@ def __init__( synthesizer_factory=synthesizer_factory, logger=logger, noise_canceling_config=noise_canceling_config, + call_reporter_config=call_reporter_config, ) self.base_url = base_url self.config_manager = config_manager diff --git a/vocode/streaming/telephony/conversation/vonage_call.py b/vocode/streaming/telephony/conversation/vonage_call.py index 251c39950..ea4a99b88 100644 --- a/vocode/streaming/telephony/conversation/vonage_call.py +++ b/vocode/streaming/telephony/conversation/vonage_call.py @@ -14,6 +14,7 @@ from vocode.streaming.models.transcriber import ( TranscriberConfig, ) +from vocode.streaming.report.base_call_report import CallReporterConfig from vocode.streaming.synthesizer.factory import SynthesizerFactory from vocode.streaming.telephony.client.vonage_client import VonageClient from vocode.streaming.telephony.config_manager.base_config_manager import ( @@ -33,23 +34,25 @@ class VonageCall(Call[VonageOutputDevice]): def __init__( - self, - from_phone: str, - to_phone: str, - base_url: str, - config_manager: BaseConfigManager, - agent_config: AgentConfig, - transcriber_config: TranscriberConfig, - synthesizer_config: SynthesizerConfig, - vonage_uuid: str, - vonage_config: Optional[VonageConfig] = None, - conversation_id: Optional[str] = None, - transcriber_factory: TranscriberFactory = TranscriberFactory(), - agent_factory: AgentFactory = AgentFactory(), - synthesizer_factory: SynthesizerFactory = SynthesizerFactory(), - events_manager: Optional[EventsManager] = None, - output_to_speaker: bool = False, - logger: Optional[logging.Logger] = None, + self, + from_phone: str, + to_phone: str, + base_url: str, + config_manager: BaseConfigManager, + agent_config: AgentConfig, + transcriber_config: TranscriberConfig, + synthesizer_config: SynthesizerConfig, + vonage_uuid: str, + vonage_config: Optional[VonageConfig] = None, + conversation_id: Optional[str] = None, + transcriber_factory: TranscriberFactory = TranscriberFactory(), + agent_factory: AgentFactory = AgentFactory(), + synthesizer_factory: SynthesizerFactory = SynthesizerFactory(), + events_manager: Optional[EventsManager] = None, + output_to_speaker: bool = False, + logger: Optional[logging.Logger] = None, + call_reporter_config: Optional[CallReporterConfig] = None, + ): super().__init__( from_phone, @@ -66,6 +69,7 @@ def __init__( agent_factory=agent_factory, synthesizer_factory=synthesizer_factory, logger=logger, + call_reporter_config=call_reporter_config ) self.output_to_speaker = output_to_speaker self.base_url = base_url diff --git a/vocode/streaming/telephony/server/router/calls.py b/vocode/streaming/telephony/server/router/calls.py index f62338531..84eeeb9f3 100644 --- a/vocode/streaming/telephony/server/router/calls.py +++ b/vocode/streaming/telephony/server/router/calls.py @@ -23,14 +23,14 @@ class CallsRouter(BaseRouter): def __init__( - self, - base_url: str, - config_manager: BaseConfigManager, - transcriber_factory: TranscriberFactory = TranscriberFactory(), - agent_factory: AgentFactory = AgentFactory(), - synthesizer_factory: SynthesizerFactory = SynthesizerFactory(), - events_manager: Optional[EventsManager] = None, - logger: Optional[logging.Logger] = None, + self, + base_url: str, + config_manager: BaseConfigManager, + transcriber_factory: TranscriberFactory = TranscriberFactory(), + agent_factory: AgentFactory = AgentFactory(), + synthesizer_factory: SynthesizerFactory = SynthesizerFactory(), + events_manager: Optional[EventsManager] = None, + logger: Optional[logging.Logger] = None, ): super().__init__() self.base_url = base_url @@ -44,16 +44,16 @@ def __init__( self.router.websocket("/connect_call/{id}")(self.connect_call) def _from_call_config( - self, - base_url: str, - call_config: BaseCallConfig, - config_manager: BaseConfigManager, - conversation_id: str, - logger: logging.Logger, - transcriber_factory: TranscriberFactory = TranscriberFactory(), - agent_factory: AgentFactory = AgentFactory(), - synthesizer_factory: SynthesizerFactory = SynthesizerFactory(), - events_manager: Optional[EventsManager] = None, + self, + base_url: str, + call_config: BaseCallConfig, + config_manager: BaseConfigManager, + conversation_id: str, + logger: logging.Logger, + transcriber_factory: TranscriberFactory = TranscriberFactory(), + agent_factory: AgentFactory = AgentFactory(), + synthesizer_factory: SynthesizerFactory = SynthesizerFactory(), + events_manager: Optional[EventsManager] = None, ): if isinstance(call_config, TwilioCallConfig): return TwilioCall( @@ -72,6 +72,7 @@ def _from_call_config( agent_factory=agent_factory, synthesizer_factory=synthesizer_factory, events_manager=events_manager, + call_reporter_config=call_config.call_reporter_config, ) elif isinstance(call_config, VonageCallConfig): return VonageCall( @@ -91,6 +92,7 @@ def _from_call_config( synthesizer_factory=synthesizer_factory, events_manager=events_manager, output_to_speaker=call_config.output_to_speaker, + call_reporter_config=call_config.call_reporter_config, ) else: raise ValueError(f"Unknown call config type {call_config.type}") diff --git a/vocode/streaming/transcriber/base_transcriber.py b/vocode/streaming/transcriber/base_transcriber.py index 7251cedb2..5b0f45802 100644 --- a/vocode/streaming/transcriber/base_transcriber.py +++ b/vocode/streaming/transcriber/base_transcriber.py @@ -15,9 +15,12 @@ from vocode.streaming.utils.worker import AsyncWorker, ThreadAsyncWorker from vocode.utils.context_tracker import BaseContextTracker from vocode.utils.context_tracker.factory import ContextTrackerFactory +from vocode.utils.voice_activity_detection import BaseVoiceActivityDetector +from vocode.utils.voice_activity_detection.factory import VoiceActivityDetectorFactory tracer = trace.get_tracer(__name__) meter = metrics.get_meter(__name__) +HUMAN_ACTIVITY_DETECTED = "human_activity_detected" class Transcription(BaseModel): @@ -87,6 +90,11 @@ def __init__( if transcriber_config.context_tracker_config: self.context_tracker = context_tracker_factory.create_context_tracker( transcriber_config.context_tracker_config, logger) + vad_factory = VoiceActivityDetectorFactory() + self.voice_activity_detector: Optional[BaseVoiceActivityDetector] = None + if transcriber_config.voice_activity_detector_config: + self.voice_activity_detector = vad_factory.create_voice_activity_detector( + transcriber_config.voice_activity_detector_config, logger) async def _run_loop(self): raise NotImplementedError diff --git a/vocode/streaming/transcriber/deepgram_transcriber.py b/vocode/streaming/transcriber/deepgram_transcriber.py index 4d31451f5..3c4099175 100644 --- a/vocode/streaming/transcriber/deepgram_transcriber.py +++ b/vocode/streaming/transcriber/deepgram_transcriber.py @@ -19,7 +19,7 @@ from vocode.streaming.transcriber.base_transcriber import ( BaseAsyncTranscriber, Transcription, - meter, + meter, HUMAN_ACTIVITY_DETECTED, ) PUNCTUATION_TERMINATORS = [".", "!", "?"] @@ -158,27 +158,16 @@ async def is_speech_final( ): is_finished = (( - transcript - and deepgram_response["speech_final"] - and transcript.strip()[-1] in PUNCTUATION_TERMINATORS - ) or ( - not transcript - and current_buffer - and (time_silent + deepgram_response["duration"]) - > self.transcriber_config.endpointing_config.time_cutoff_seconds - )) and deepgram_response["duration"] > self.transcriber_config.minimum_speaking_duration_to_interupt - - - - # print('_silece_'*5) - # print(time_silent) - # print('_silece_'*5) - - - if is_finished: - print("Transcriber Duration"*3) - print(deepgram_response["duration"]) - print("Transcriber Duration"*3) + transcript + and deepgram_response["speech_final"] + and transcript.strip()[-1] in PUNCTUATION_TERMINATORS + ) or ( + not transcript + and current_buffer + and (time_silent + deepgram_response["duration"]) + > self.transcriber_config.endpointing_config.time_cutoff_seconds + )) and deepgram_response[ + "duration"] > self.transcriber_config.minimum_speaking_duration_to_interrupt if is_finished and self.transcriber_config.skip_on_back_track_audio: is_interrupt_task = asyncio.create_task( @@ -212,6 +201,16 @@ async def sender(ws: WebSocketClientProtocol): # sends audio to websocket while not self._ended: try: data = await asyncio.wait_for(self.input_queue.get(), 5) + if self.transcriber_config.voice_activity_detector_config: + try: + if self.voice_activity_detector.should_interrupt(data): + self.output_queue.put_nowait(Transcription( + message=HUMAN_ACTIVITY_DETECTED, + confidence=1, + is_final=True, + )) + except Exception as e: + self.logger.debug(f"Error in voice activity detector: {repr(e)}") except asyncio.exceptions.TimeoutError: break num_channels = 1 diff --git a/vocode/utils/context_tracker/factory.py b/vocode/utils/context_tracker/factory.py index da0d917b3..6ff8d9bb2 100644 --- a/vocode/utils/context_tracker/factory.py +++ b/vocode/utils/context_tracker/factory.py @@ -1,7 +1,7 @@ import logging from typing import Optional -from vocode.utils.context_tracker.context_tracker import BaseContextTrackerConfig, BaseContextTracker +from vocode.utils.context_tracker.context_tracker import BaseContextTrackerConfig from vocode.utils.context_tracker.open_ai_context_tracker import OpenAIContextTrackerConfig, OpenAIContextTracker diff --git a/vocode/utils/voice_activity_detection/__init__.py b/vocode/utils/voice_activity_detection/__init__.py new file mode 100644 index 000000000..70328b4f2 --- /dev/null +++ b/vocode/utils/voice_activity_detection/__init__.py @@ -0,0 +1,3 @@ +from .vad import BaseVoiceActivityDetectorConfig, BaseVoiceActivityDetector +from .silero_vad import SileroVoiceActivityDetectorConfig, SileroVoiceActivityDetector +from .webrtc_vad import WebRTCVoiceActivityDetectorConfig, WebRTCVoiceActivityDetector diff --git a/vocode/utils/voice_activity_detection/factory.py b/vocode/utils/voice_activity_detection/factory.py new file mode 100644 index 000000000..d51dc1d3f --- /dev/null +++ b/vocode/utils/voice_activity_detection/factory.py @@ -0,0 +1,20 @@ +import logging +from typing import Optional + +from vocode.utils.voice_activity_detection.silero_vad import SileroVoiceActivityDetectorConfig, \ + SileroVoiceActivityDetector +from vocode.utils.voice_activity_detection.vad import BaseVoiceActivityDetectorConfig +from vocode.utils.voice_activity_detection.webrtc_vad import WebRTCVoiceActivityDetectorConfig, \ + WebRTCVoiceActivityDetector + + +class VoiceActivityDetectorFactory: + @staticmethod + def create_voice_activity_detector( + vad_config: Optional[BaseVoiceActivityDetectorConfig], + logger: Optional[logging.Logger] = None, + ): + if isinstance(vad_config, WebRTCVoiceActivityDetectorConfig): + return WebRTCVoiceActivityDetector(vad_config, logger=logger) + if isinstance(vad_config, SileroVoiceActivityDetectorConfig): + return SileroVoiceActivityDetector(vad_config, logger=logger) diff --git a/vocode/utils/voice_activity_detection/silero_vad.py b/vocode/utils/voice_activity_detection/silero_vad.py new file mode 100644 index 000000000..2ff26b30e --- /dev/null +++ b/vocode/utils/voice_activity_detection/silero_vad.py @@ -0,0 +1,49 @@ +import logging +from typing import Optional + +import numpy as np + +from vocode.utils.voice_activity_detection.vad import BaseVoiceActivityDetector, BaseVoiceActivityDetectorConfig, \ + VoiceActivityDetectorType + + +class SileroVoiceActivityDetectorConfig(BaseVoiceActivityDetectorConfig, type=VoiceActivityDetectorType.SILERO.value): + model_name: str = "silero_vad" + USE_ONNX: bool = False + repo_or_dir: str = 'snakers4/silero-vad' + force_reload: str = True + num_threads: int = 1 + threshold: float = .05 + model_save_path: str = "silero_vad.onnx" + + +class SileroVoiceActivityDetector(BaseVoiceActivityDetector[SileroVoiceActivityDetectorConfig]): + def __init__(self, config: SileroVoiceActivityDetectorConfig, logger: Optional[logging.Logger] = None): + import torch + + super().__init__(config, logger) + if self.config.USE_ONNX: + import onnxruntime + + self.torch = torch + self.torch.set_num_threads(self.config.num_threads) + self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + self.model, self.utils = torch.hub.load( + repo_or_dir=self.config.repo_or_dir, + model=self.config.model_name, + force_reload=self.config.force_reload, + onnx=self.config.USE_ONNX, + ) + (self.get_speech_timestamps, + _, + _, + self.VADIterator, + _) = self.utils + self.vad_iterator = self.VADIterator(self.model) + self.model.to(self.device) + + def is_voice_active(self, frame: bytes) -> bool: + np_frame = np.frombuffer(frame, dtype=np.int8) + speech_timestamps = self.get_speech_timestamps(np_frame, self.model, sampling_rate=self.config.frame_rate) + self.logger.debug(f"speech_timestamps = {speech_timestamps}") + return np.mean(speech_timestamps) > self.config.threshold diff --git a/vocode/utils/voice_activity_detection/vad.py b/vocode/utils/voice_activity_detection/vad.py new file mode 100644 index 000000000..4e4dff0fb --- /dev/null +++ b/vocode/utils/voice_activity_detection/vad.py @@ -0,0 +1,63 @@ +import logging +from datetime import datetime, timedelta +from enum import Enum +from typing import Generic, TypeVar, Optional + +from vocode.streaming.models.model import TypedModel + + +class VoiceActivityDetectorType(str, Enum): + BASE = "base_voice_activity_detector" + WEB_RTC = "web_rtc_voice_activity_detector" + SILERO = "silero_voice_activity_detector" + LIB_ROSA = "librosa_voice_activity_detector" + + +class BaseVoiceActivityDetectorConfig(TypedModel, type=VoiceActivityDetectorType.BASE.value): + frame_rate: int = 16000 + min_activity_duration: timedelta = timedelta(milliseconds=400) + speach_ratio: float = .8 + + +VoiceActivityDetectorConfigType = TypeVar("VoiceActivityDetectorConfigType", bound=BaseVoiceActivityDetectorConfig) + + +class BaseVoiceActivityDetector(Generic[VoiceActivityDetectorConfigType]): + def __init__(self, config: VoiceActivityDetectorConfigType, logger: Optional[logging.Logger] = None): + self.logger: logging.Logger = logger or logging.getLogger(__name__) + self.config = config + self.speach_start_timestamp = None + self.activity_state = {True: 0, False: 0} + self.is_speaking = False + self.is_interrupted = False + + def is_voice_active(self, frame: bytes) -> bool: + raise NotImplementedError + + def get_config(self) -> BaseVoiceActivityDetectorConfig: + return self.config + + def should_interrupt(self, frame: bytes) -> bool: + now = datetime.now() + is_voice_active = self.is_voice_active(frame) + + self.activity_state[is_voice_active] += 1 + + if is_voice_active and self.speach_start_timestamp is None: + self.activity_state = {True: 0, False: 0} + self.speach_start_timestamp = now + + if self.speach_start_timestamp is None: + return False + + if (now - self.speach_start_timestamp) > self.config.min_activity_duration: + speach_ratio = self.activity_state[True] / (self.activity_state[True] + self.activity_state[False]) + if speach_ratio > self.config.speach_ratio: + if self.is_interrupted: + return False + self.is_interrupted = True + return True + else: + self.is_interrupted = False + self.speach_start_timestamp = None + return False diff --git a/vocode/utils/voice_activity_detection/webrtc_vad.py b/vocode/utils/voice_activity_detection/webrtc_vad.py new file mode 100644 index 000000000..51797293c --- /dev/null +++ b/vocode/utils/voice_activity_detection/webrtc_vad.py @@ -0,0 +1,19 @@ +import logging +from typing import Optional + +from vocode.utils.voice_activity_detection.vad import BaseVoiceActivityDetector, BaseVoiceActivityDetectorConfig, \ + VoiceActivityDetectorType + + +class WebRTCVoiceActivityDetectorConfig(BaseVoiceActivityDetectorConfig, type=VoiceActivityDetectorType.WEB_RTC.value): + mode: int = 3 + + +class WebRTCVoiceActivityDetector(BaseVoiceActivityDetector[WebRTCVoiceActivityDetectorConfig]): + def __init__(self, config: WebRTCVoiceActivityDetectorConfig, logger: Optional[logging.Logger] = None): + import webrtcvad + super().__init__(config, logger) + self.vad = webrtcvad.Vad(self.config.mode) + + def is_voice_active(self, frame: bytes) -> bool: + return self.vad.is_speech(frame, self.config.frame_rate)