From 1d3d4cca0623fa879bc49d7c299279dd1de90d11 Mon Sep 17 00:00:00 2001 From: brianjlai Date: Tue, 7 May 2024 14:42:46 -0700 Subject: [PATCH] [source-hubspot] bump cdk, implement rfr for contacts_form_submissions, contact_list_memberships, contact_merged_audit, and mock server tests --- .../connectors/source-hubspot/poetry.lock | 353 ++++++++++++++++-- .../source-hubspot/source_hubspot/streams.py | 136 +++---- .../unit_tests/integrations/__init__.py | 26 ++ .../integrations/request_builders/streams.py | 34 ++ .../response_builder/pagination.py | 6 + .../integrations/response_builder/streams.py | 7 +- .../test_contacts_form_submissions.py | 60 +++ .../test_contacts_list_memberships.py | 60 +++ .../test_contacts_merged_audit.py | 60 +++ .../integrations/test_engagements_calls.py | 3 +- .../integrations/test_owners_archived.py | 3 +- .../test_web_analytics_streams.py | 3 +- .../response/contacts_form_submissions.json | 186 +++++++++ .../response/contacts_list_memberships.json | 154 ++++++++ .../http/response/contacts_merged_audit.json | 240 ++++++++++++ .../source-hubspot/unit_tests/test_streams.py | 20 +- 16 files changed, 1242 insertions(+), 109 deletions(-) create mode 100644 airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/test_contacts_form_submissions.py create mode 100644 airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/test_contacts_list_memberships.py create mode 100644 airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/test_contacts_merged_audit.py create mode 100644 airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/contacts_form_submissions.json create mode 100644 airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/contacts_list_memberships.json create mode 100644 airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/contacts_merged_audit.json diff --git a/airbyte-integrations/connectors/source-hubspot/poetry.lock b/airbyte-integrations/connectors/source-hubspot/poetry.lock index 841fcdc13383..37b72e5e02de 100644 --- a/airbyte-integrations/connectors/source-hubspot/poetry.lock +++ b/airbyte-integrations/connectors/source-hubspot/poetry.lock @@ -2,19 +2,20 @@ [[package]] name = "airbyte-cdk" -version = "0.78.6" +version = "0.87.0" description = "A framework for writing Airbyte Connectors." optional = false python-versions = "<4.0,>=3.9" files = [ - {file = "airbyte_cdk-0.78.6-py3-none-any.whl", hash = "sha256:e5f44c6da6d5b5d6f3f6a7f41a3f4a5e2dfc6fefb4c6823af6302c34c6fb4a87"}, - {file = "airbyte_cdk-0.78.6.tar.gz", hash = "sha256:0178f3cefa705f600d51f09e1313024a89cd1c99f2f1f796e8e0181d8e02ad2f"}, + {file = "airbyte_cdk-0.87.0-py3-none-any.whl", hash = "sha256:e11aeae3d758b04086c584ccd00f3f11b1808a36a28f6b3f6da8c51e58ed0ab9"}, + {file = "airbyte_cdk-0.87.0.tar.gz", hash = "sha256:eaa354a3e8fa41340fde492e09553cc87a8228d96e6a684d41b493f1ac379889"}, ] [package.dependencies] -airbyte-protocol-models = "0.5.1" +airbyte-protocol-models = ">=0.9.0,<1.0" backoff = "*" cachetools = "*" +cryptography = ">=42.0.5,<43.0.0" Deprecated = ">=1.2,<1.3" dpath = ">=2.0.1,<2.1.0" genson = "1.2.2" @@ -22,8 +23,10 @@ isodate = ">=0.6.1,<0.7.0" Jinja2 = ">=3.1.2,<3.2.0" jsonref = ">=0.2,<0.3" jsonschema = ">=3.2.0,<3.3.0" +langchain_core = "0.1.42" pendulum = "<3.0.0" pydantic = ">=1.10.8,<2.0.0" +pyjwt = ">=2.8.0,<3.0.0" pyrate-limiter = ">=3.1.0,<3.2.0" python-dateutil = "*" PyYAML = ">=6.0.1,<7.0.0" @@ -34,17 +37,17 @@ wcmatch = "8.4" [package.extras] file-based = ["avro (>=1.11.2,<1.12.0)", "fastavro (>=1.8.0,<1.9.0)", "markdown", "pdf2image (==1.16.3)", "pdfminer.six (==20221105)", "pyarrow (>=15.0.0,<15.1.0)", "pytesseract (==0.3.10)", "unstructured.pytesseract (>=0.3.12)", "unstructured[docx,pptx] (==0.10.27)"] sphinx-docs = ["Sphinx (>=4.2,<4.3)", "sphinx-rtd-theme (>=1.0,<1.1)"] -vector-db-based = ["cohere (==4.21)", "langchain (==0.0.271)", "openai[embeddings] (==0.27.9)", "tiktoken (==0.4.0)"] +vector-db-based = ["cohere (==4.21)", "langchain (==0.1.16)", "openai[embeddings] (==0.27.9)", "tiktoken (==0.4.0)"] [[package]] name = "airbyte-protocol-models" -version = "0.5.1" +version = "0.9.0" description = "Declares the Airbyte Protocol." optional = false python-versions = ">=3.8" files = [ - {file = "airbyte_protocol_models-0.5.1-py3-none-any.whl", hash = "sha256:dfe84e130e51ce2ae81a06d5aa36f6c5ce3152b9e36e6f0195fad6c3dab0927e"}, - {file = "airbyte_protocol_models-0.5.1.tar.gz", hash = "sha256:7c8b16c7c1c7956b1996052e40585a3a93b1e44cb509c4e97c1ee4fe507ea086"}, + {file = "airbyte_protocol_models-0.9.0-py3-none-any.whl", hash = "sha256:e972e140b5efd1edad5a338bcae8fdee9fc12545caf2c321e0f61b151c163a9b"}, + {file = "airbyte_protocol_models-0.9.0.tar.gz", hash = "sha256:40b69c33df23fe82d7078e84beb123bd604480e4d73cb277a890fcc92aedc8d2"}, ] [package.dependencies] @@ -148,6 +151,70 @@ files = [ {file = "certifi-2024.2.2.tar.gz", hash = "sha256:0569859f95fc761b18b45ef421b1290a0f65f147e92a1e5eb3e635f9a5e4e66f"}, ] +[[package]] +name = "cffi" +version = "1.16.0" +description = "Foreign Function Interface for Python calling C code." +optional = false +python-versions = ">=3.8" +files = [ + {file = "cffi-1.16.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:6b3d6606d369fc1da4fd8c357d026317fbb9c9b75d36dc16e90e84c26854b088"}, + {file = "cffi-1.16.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:ac0f5edd2360eea2f1daa9e26a41db02dd4b0451b48f7c318e217ee092a213e9"}, + {file = "cffi-1.16.0-cp310-cp310-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7e61e3e4fa664a8588aa25c883eab612a188c725755afff6289454d6362b9673"}, + {file = "cffi-1.16.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a72e8961a86d19bdb45851d8f1f08b041ea37d2bd8d4fd19903bc3083d80c896"}, + {file = "cffi-1.16.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5b50bf3f55561dac5438f8e70bfcdfd74543fd60df5fa5f62d94e5867deca684"}, + {file = "cffi-1.16.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:7651c50c8c5ef7bdb41108b7b8c5a83013bfaa8a935590c5d74627c047a583c7"}, + {file = "cffi-1.16.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e4108df7fe9b707191e55f33efbcb2d81928e10cea45527879a4749cbe472614"}, + {file = "cffi-1.16.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:32c68ef735dbe5857c810328cb2481e24722a59a2003018885514d4c09af9743"}, + {file = "cffi-1.16.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:673739cb539f8cdaa07d92d02efa93c9ccf87e345b9a0b556e3ecc666718468d"}, + {file = "cffi-1.16.0-cp310-cp310-win32.whl", hash = "sha256:9f90389693731ff1f659e55c7d1640e2ec43ff725cc61b04b2f9c6d8d017df6a"}, + {file = "cffi-1.16.0-cp310-cp310-win_amd64.whl", hash = "sha256:e6024675e67af929088fda399b2094574609396b1decb609c55fa58b028a32a1"}, + {file = "cffi-1.16.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:b84834d0cf97e7d27dd5b7f3aca7b6e9263c56308ab9dc8aae9784abb774d404"}, + {file = "cffi-1.16.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:1b8ebc27c014c59692bb2664c7d13ce7a6e9a629be20e54e7271fa696ff2b417"}, + {file = "cffi-1.16.0-cp311-cp311-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ee07e47c12890ef248766a6e55bd38ebfb2bb8edd4142d56db91b21ea68b7627"}, + {file = "cffi-1.16.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d8a9d3ebe49f084ad71f9269834ceccbf398253c9fac910c4fd7053ff1386936"}, + {file = "cffi-1.16.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e70f54f1796669ef691ca07d046cd81a29cb4deb1e5f942003f401c0c4a2695d"}, + {file = "cffi-1.16.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:5bf44d66cdf9e893637896c7faa22298baebcd18d1ddb6d2626a6e39793a1d56"}, + {file = "cffi-1.16.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7b78010e7b97fef4bee1e896df8a4bbb6712b7f05b7ef630f9d1da00f6444d2e"}, + {file = "cffi-1.16.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:c6a164aa47843fb1b01e941d385aab7215563bb8816d80ff3a363a9f8448a8dc"}, + {file = "cffi-1.16.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:e09f3ff613345df5e8c3667da1d918f9149bd623cd9070c983c013792a9a62eb"}, + {file = "cffi-1.16.0-cp311-cp311-win32.whl", hash = "sha256:2c56b361916f390cd758a57f2e16233eb4f64bcbeee88a4881ea90fca14dc6ab"}, + {file = "cffi-1.16.0-cp311-cp311-win_amd64.whl", hash = "sha256:db8e577c19c0fda0beb7e0d4e09e0ba74b1e4c092e0e40bfa12fe05b6f6d75ba"}, + {file = "cffi-1.16.0-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:fa3a0128b152627161ce47201262d3140edb5a5c3da88d73a1b790a959126956"}, + {file = "cffi-1.16.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:68e7c44931cc171c54ccb702482e9fc723192e88d25a0e133edd7aff8fcd1f6e"}, + {file = "cffi-1.16.0-cp312-cp312-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:abd808f9c129ba2beda4cfc53bde801e5bcf9d6e0f22f095e45327c038bfe68e"}, + {file = "cffi-1.16.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:88e2b3c14bdb32e440be531ade29d3c50a1a59cd4e51b1dd8b0865c54ea5d2e2"}, + {file = "cffi-1.16.0-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:fcc8eb6d5902bb1cf6dc4f187ee3ea80a1eba0a89aba40a5cb20a5087d961357"}, + {file = "cffi-1.16.0-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b7be2d771cdba2942e13215c4e340bfd76398e9227ad10402a8767ab1865d2e6"}, + {file = "cffi-1.16.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e715596e683d2ce000574bae5d07bd522c781a822866c20495e52520564f0969"}, + {file = "cffi-1.16.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:2d92b25dbf6cae33f65005baf472d2c245c050b1ce709cc4588cdcdd5495b520"}, + {file = "cffi-1.16.0-cp312-cp312-win32.whl", hash = "sha256:b2ca4e77f9f47c55c194982e10f058db063937845bb2b7a86c84a6cfe0aefa8b"}, + {file = "cffi-1.16.0-cp312-cp312-win_amd64.whl", hash = "sha256:68678abf380b42ce21a5f2abde8efee05c114c2fdb2e9eef2efdb0257fba1235"}, + {file = "cffi-1.16.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:0c9ef6ff37e974b73c25eecc13952c55bceed9112be2d9d938ded8e856138bcc"}, + {file = "cffi-1.16.0-cp38-cp38-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a09582f178759ee8128d9270cd1344154fd473bb77d94ce0aeb2a93ebf0feaf0"}, + {file = "cffi-1.16.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e760191dd42581e023a68b758769e2da259b5d52e3103c6060ddc02c9edb8d7b"}, + {file = "cffi-1.16.0-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:80876338e19c951fdfed6198e70bc88f1c9758b94578d5a7c4c91a87af3cf31c"}, + {file = "cffi-1.16.0-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a6a14b17d7e17fa0d207ac08642c8820f84f25ce17a442fd15e27ea18d67c59b"}, + {file = "cffi-1.16.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6602bc8dc6f3a9e02b6c22c4fc1e47aa50f8f8e6d3f78a5e16ac33ef5fefa324"}, + {file = "cffi-1.16.0-cp38-cp38-win32.whl", hash = "sha256:131fd094d1065b19540c3d72594260f118b231090295d8c34e19a7bbcf2e860a"}, + {file = "cffi-1.16.0-cp38-cp38-win_amd64.whl", hash = "sha256:31d13b0f99e0836b7ff893d37af07366ebc90b678b6664c955b54561fc36ef36"}, + {file = "cffi-1.16.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:582215a0e9adbe0e379761260553ba11c58943e4bbe9c36430c4ca6ac74b15ed"}, + {file = "cffi-1.16.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:b29ebffcf550f9da55bec9e02ad430c992a87e5f512cd63388abb76f1036d8d2"}, + {file = "cffi-1.16.0-cp39-cp39-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:dc9b18bf40cc75f66f40a7379f6a9513244fe33c0e8aa72e2d56b0196a7ef872"}, + {file = "cffi-1.16.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9cb4a35b3642fc5c005a6755a5d17c6c8b6bcb6981baf81cea8bfbc8903e8ba8"}, + {file = "cffi-1.16.0-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b86851a328eedc692acf81fb05444bdf1891747c25af7529e39ddafaf68a4f3f"}, + {file = "cffi-1.16.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c0f31130ebc2d37cdd8e44605fb5fa7ad59049298b3f745c74fa74c62fbfcfc4"}, + {file = "cffi-1.16.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8f8e709127c6c77446a8c0a8c8bf3c8ee706a06cd44b1e827c3e6a2ee6b8c098"}, + {file = "cffi-1.16.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:748dcd1e3d3d7cd5443ef03ce8685043294ad6bd7c02a38d1bd367cfd968e000"}, + {file = "cffi-1.16.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:8895613bcc094d4a1b2dbe179d88d7fb4a15cee43c052e8885783fac397d91fe"}, + {file = "cffi-1.16.0-cp39-cp39-win32.whl", hash = "sha256:ed86a35631f7bfbb28e108dd96773b9d5a6ce4811cf6ea468bb6a359b256b1e4"}, + {file = "cffi-1.16.0-cp39-cp39-win_amd64.whl", hash = "sha256:3686dffb02459559c74dd3d81748269ffb0eb027c39a6fc99502de37d501faa8"}, + {file = "cffi-1.16.0.tar.gz", hash = "sha256:bcb3ef43e58665bbda2fb198698fcae6776483e0c4a631aa5647806c25e02cc0"}, +] + +[package.dependencies] +pycparser = "*" + [[package]] name = "charset-normalizer" version = "3.3.2" @@ -258,6 +325,60 @@ files = [ {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, ] +[[package]] +name = "cryptography" +version = "42.0.7" +description = "cryptography is a package which provides cryptographic recipes and primitives to Python developers." +optional = false +python-versions = ">=3.7" +files = [ + {file = "cryptography-42.0.7-cp37-abi3-macosx_10_12_universal2.whl", hash = "sha256:a987f840718078212fdf4504d0fd4c6effe34a7e4740378e59d47696e8dfb477"}, + {file = "cryptography-42.0.7-cp37-abi3-macosx_10_12_x86_64.whl", hash = "sha256:bd13b5e9b543532453de08bcdc3cc7cebec6f9883e886fd20a92f26940fd3e7a"}, + {file = "cryptography-42.0.7-cp37-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a79165431551042cc9d1d90e6145d5d0d3ab0f2d66326c201d9b0e7f5bf43604"}, + {file = "cryptography-42.0.7-cp37-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a47787a5e3649008a1102d3df55424e86606c9bae6fb77ac59afe06d234605f8"}, + {file = "cryptography-42.0.7-cp37-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:02c0eee2d7133bdbbc5e24441258d5d2244beb31da5ed19fbb80315f4bbbff55"}, + {file = "cryptography-42.0.7-cp37-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:5e44507bf8d14b36b8389b226665d597bc0f18ea035d75b4e53c7b1ea84583cc"}, + {file = "cryptography-42.0.7-cp37-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:7f8b25fa616d8b846aef64b15c606bb0828dbc35faf90566eb139aa9cff67af2"}, + {file = "cryptography-42.0.7-cp37-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:93a3209f6bb2b33e725ed08ee0991b92976dfdcf4e8b38646540674fc7508e13"}, + {file = "cryptography-42.0.7-cp37-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:e6b8f1881dac458c34778d0a424ae5769de30544fc678eac51c1c8bb2183e9da"}, + {file = "cryptography-42.0.7-cp37-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:3de9a45d3b2b7d8088c3fbf1ed4395dfeff79d07842217b38df14ef09ce1d8d7"}, + {file = "cryptography-42.0.7-cp37-abi3-win32.whl", hash = "sha256:789caea816c6704f63f6241a519bfa347f72fbd67ba28d04636b7c6b7da94b0b"}, + {file = "cryptography-42.0.7-cp37-abi3-win_amd64.whl", hash = "sha256:8cb8ce7c3347fcf9446f201dc30e2d5a3c898d009126010cbd1f443f28b52678"}, + {file = "cryptography-42.0.7-cp39-abi3-macosx_10_12_universal2.whl", hash = "sha256:a3a5ac8b56fe37f3125e5b72b61dcde43283e5370827f5233893d461b7360cd4"}, + {file = "cryptography-42.0.7-cp39-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:779245e13b9a6638df14641d029add5dc17edbef6ec915688f3acb9e720a5858"}, + {file = "cryptography-42.0.7-cp39-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0d563795db98b4cd57742a78a288cdbdc9daedac29f2239793071fe114f13785"}, + {file = "cryptography-42.0.7-cp39-abi3-manylinux_2_28_aarch64.whl", hash = "sha256:31adb7d06fe4383226c3e963471f6837742889b3c4caa55aac20ad951bc8ffda"}, + {file = "cryptography-42.0.7-cp39-abi3-manylinux_2_28_x86_64.whl", hash = "sha256:efd0bf5205240182e0f13bcaea41be4fdf5c22c5129fc7ced4a0282ac86998c9"}, + {file = "cryptography-42.0.7-cp39-abi3-musllinux_1_1_aarch64.whl", hash = "sha256:a9bc127cdc4ecf87a5ea22a2556cab6c7eda2923f84e4f3cc588e8470ce4e42e"}, + {file = "cryptography-42.0.7-cp39-abi3-musllinux_1_1_x86_64.whl", hash = "sha256:3577d029bc3f4827dd5bf8bf7710cac13527b470bbf1820a3f394adb38ed7d5f"}, + {file = "cryptography-42.0.7-cp39-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:2e47577f9b18723fa294b0ea9a17d5e53a227867a0a4904a1a076d1646d45ca1"}, + {file = "cryptography-42.0.7-cp39-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:1a58839984d9cb34c855197043eaae2c187d930ca6d644612843b4fe8513c886"}, + {file = "cryptography-42.0.7-cp39-abi3-win32.whl", hash = "sha256:e6b79d0adb01aae87e8a44c2b64bc3f3fe59515280e00fb6d57a7267a2583cda"}, + {file = "cryptography-42.0.7-cp39-abi3-win_amd64.whl", hash = "sha256:16268d46086bb8ad5bf0a2b5544d8a9ed87a0e33f5e77dd3c3301e63d941a83b"}, + {file = "cryptography-42.0.7-pp310-pypy310_pp73-macosx_10_12_x86_64.whl", hash = "sha256:2954fccea107026512b15afb4aa664a5640cd0af630e2ee3962f2602693f0c82"}, + {file = "cryptography-42.0.7-pp310-pypy310_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:362e7197754c231797ec45ee081f3088a27a47c6c01eff2ac83f60f85a50fe60"}, + {file = "cryptography-42.0.7-pp310-pypy310_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:4f698edacf9c9e0371112792558d2f705b5645076cc0aaae02f816a0171770fd"}, + {file = "cryptography-42.0.7-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:5482e789294854c28237bba77c4c83be698be740e31a3ae5e879ee5444166582"}, + {file = "cryptography-42.0.7-pp39-pypy39_pp73-macosx_10_12_x86_64.whl", hash = "sha256:e9b2a6309f14c0497f348d08a065d52f3020656f675819fc405fb63bbcd26562"}, + {file = "cryptography-42.0.7-pp39-pypy39_pp73-manylinux_2_28_aarch64.whl", hash = "sha256:d8e3098721b84392ee45af2dd554c947c32cc52f862b6a3ae982dbb90f577f14"}, + {file = "cryptography-42.0.7-pp39-pypy39_pp73-manylinux_2_28_x86_64.whl", hash = "sha256:c65f96dad14f8528a447414125e1fc8feb2ad5a272b8f68477abbcc1ea7d94b9"}, + {file = "cryptography-42.0.7-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:36017400817987670037fbb0324d71489b6ead6231c9604f8fc1f7d008087c68"}, + {file = "cryptography-42.0.7.tar.gz", hash = "sha256:ecbfbc00bf55888edda9868a4cf927205de8499e7fabe6c050322298382953f2"}, +] + +[package.dependencies] +cffi = {version = ">=1.12", markers = "platform_python_implementation != \"PyPy\""} + +[package.extras] +docs = ["sphinx (>=5.3.0)", "sphinx-rtd-theme (>=1.1.1)"] +docstest = ["pyenchant (>=1.6.11)", "readme-renderer", "sphinxcontrib-spelling (>=4.0.1)"] +nox = ["nox"] +pep8test = ["check-sdist", "click", "mypy", "ruff"] +sdist = ["build"] +ssh = ["bcrypt (>=3.1.5)"] +test = ["certifi", "pretend", "pytest (>=6.2.0)", "pytest-benchmark", "pytest-cov", "pytest-xdist"] +test-randomorder = ["pytest-randomly"] + [[package]] name = "deprecated" version = "1.2.14" @@ -288,13 +409,13 @@ files = [ [[package]] name = "exceptiongroup" -version = "1.2.0" +version = "1.2.1" description = "Backport of PEP 654 (exception groups)" optional = false python-versions = ">=3.7" files = [ - {file = "exceptiongroup-1.2.0-py3-none-any.whl", hash = "sha256:4bfd3996ac73b41e9b9628b04e079f193850720ea5945fc96a08633c66912f14"}, - {file = "exceptiongroup-1.2.0.tar.gz", hash = "sha256:91f5c769735f051a4290d52edd0858999b57e5876e9f85937691bd4c9fa3ed68"}, + {file = "exceptiongroup-1.2.1-py3-none-any.whl", hash = "sha256:5258b9ed329c5bbdd31a309f53cbfb0b155341807f6ff7606a1e801a891b29ad"}, + {file = "exceptiongroup-1.2.1.tar.gz", hash = "sha256:a4785e48b045528f5bfe627b6ad554ff32def154f42372786903b7abcfe1aa16"}, ] [package.extras] @@ -327,13 +448,13 @@ files = [ [[package]] name = "idna" -version = "3.6" +version = "3.7" description = "Internationalized Domain Names in Applications (IDNA)" optional = false python-versions = ">=3.5" files = [ - {file = "idna-3.6-py3-none-any.whl", hash = "sha256:c05567e9c24a6b9faaa835c4821bad0590fbb9d5779e7caa6e1cc4978e7eb24f"}, - {file = "idna-3.6.tar.gz", hash = "sha256:9ecdbbd083b06798ae1e86adcbfe8ab1479cf864e4ee30fe4e46a003d12491ca"}, + {file = "idna-3.7-py3-none-any.whl", hash = "sha256:82fee1fc78add43492d3a1898bfa6d8a904cc97d8427f683ed8e798d07761aa0"}, + {file = "idna-3.7.tar.gz", hash = "sha256:028ff3aadf0609c1fd278d8ea3089299412a7a8b9bd005dd08b9f8285bcb5cfc"}, ] [[package]] @@ -363,13 +484,13 @@ six = "*" [[package]] name = "jinja2" -version = "3.1.3" +version = "3.1.4" description = "A very fast and expressive template engine." optional = false python-versions = ">=3.7" files = [ - {file = "Jinja2-3.1.3-py3-none-any.whl", hash = "sha256:7d6d50dd97d52cbc355597bd845fabfbac3f551e1f99619e39a35ce8c370b5fa"}, - {file = "Jinja2-3.1.3.tar.gz", hash = "sha256:ac8bd6544d4bb2c9792bf3a159e80bba8fda7f07e81bc3aed565432d5925ba90"}, + {file = "jinja2-3.1.4-py3-none-any.whl", hash = "sha256:bc5dd2abb727a5319567b7a813e6a2e7318c39f4f487cfe6c89c6f9c7d25197d"}, + {file = "jinja2-3.1.4.tar.gz", hash = "sha256:4a3aee7acbbe7303aede8e9648d13b8bf88a429282aa6122a993f0ac800cb369"}, ] [package.dependencies] @@ -378,6 +499,31 @@ MarkupSafe = ">=2.0" [package.extras] i18n = ["Babel (>=2.7)"] +[[package]] +name = "jsonpatch" +version = "1.33" +description = "Apply JSON-Patches (RFC 6902)" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*, !=3.6.*" +files = [ + {file = "jsonpatch-1.33-py2.py3-none-any.whl", hash = "sha256:0ae28c0cd062bbd8b8ecc26d7d164fbbea9652a1a3693f3b956c1eae5145dade"}, + {file = "jsonpatch-1.33.tar.gz", hash = "sha256:9fcd4009c41e6d12348b4a0ff2563ba56a2923a7dfee731d004e212e1ee5030c"}, +] + +[package.dependencies] +jsonpointer = ">=1.9" + +[[package]] +name = "jsonpointer" +version = "2.4" +description = "Identify specific nodes in a JSON document (RFC 6901)" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*, !=3.6.*" +files = [ + {file = "jsonpointer-2.4-py2.py3-none-any.whl", hash = "sha256:15d51bba20eea3165644553647711d150376234112651b4f1811022aecad7d7a"}, + {file = "jsonpointer-2.4.tar.gz", hash = "sha256:585cee82b70211fa9e6043b7bb89db6e1aa49524340dde8ad6b63206ea689d88"}, +] + [[package]] name = "jsonref" version = "0.2" @@ -410,6 +556,44 @@ six = ">=1.11.0" format = ["idna", "jsonpointer (>1.13)", "rfc3987", "strict-rfc3339", "webcolors"] format-nongpl = ["idna", "jsonpointer (>1.13)", "rfc3339-validator", "rfc3986-validator (>0.1.0)", "webcolors"] +[[package]] +name = "langchain-core" +version = "0.1.42" +description = "Building applications with LLMs through composability" +optional = false +python-versions = "<4.0,>=3.8.1" +files = [ + {file = "langchain_core-0.1.42-py3-none-any.whl", hash = "sha256:c5653ffa08a44f740295c157a24c0def4a753333f6a2c41f76bf431cd00be8b5"}, + {file = "langchain_core-0.1.42.tar.gz", hash = "sha256:40751bf60ea5d8e2b2efe65290db434717ee3834870c002e40e2811f09d814e6"}, +] + +[package.dependencies] +jsonpatch = ">=1.33,<2.0" +langsmith = ">=0.1.0,<0.2.0" +packaging = ">=23.2,<24.0" +pydantic = ">=1,<3" +PyYAML = ">=5.3" +tenacity = ">=8.1.0,<9.0.0" + +[package.extras] +extended-testing = ["jinja2 (>=3,<4)"] + +[[package]] +name = "langsmith" +version = "0.1.54" +description = "Client library to connect to the LangSmith LLM Tracing and Evaluation Platform." +optional = false +python-versions = "<4.0,>=3.8.1" +files = [ + {file = "langsmith-0.1.54-py3-none-any.whl", hash = "sha256:e8ba2758dbdff0fccb35337c28a5ab641dd980b22e178d390b72a15c9ae9caff"}, + {file = "langsmith-0.1.54.tar.gz", hash = "sha256:86f5a90e48303de897f37a893f8bb635eabdaf23e674099e8bc0f2e9ca2f8faf"}, +] + +[package.dependencies] +orjson = ">=3.9.14,<4.0.0" +pydantic = ">=1,<3" +requests = ">=2,<3" + [[package]] name = "markupsafe" version = "2.1.5" @@ -495,15 +679,70 @@ build = ["blurb", "twine", "wheel"] docs = ["sphinx"] test = ["pytest", "pytest-cov"] +[[package]] +name = "orjson" +version = "3.10.3" +description = "Fast, correct Python JSON library supporting dataclasses, datetimes, and numpy" +optional = false +python-versions = ">=3.8" +files = [ + {file = "orjson-3.10.3-cp310-cp310-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:9fb6c3f9f5490a3eb4ddd46fc1b6eadb0d6fc16fb3f07320149c3286a1409dd8"}, + {file = "orjson-3.10.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:252124b198662eee80428f1af8c63f7ff077c88723fe206a25df8dc57a57b1fa"}, + {file = "orjson-3.10.3-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9f3e87733823089a338ef9bbf363ef4de45e5c599a9bf50a7a9b82e86d0228da"}, + {file = "orjson-3.10.3-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c8334c0d87103bb9fbbe59b78129f1f40d1d1e8355bbed2ca71853af15fa4ed3"}, + {file = "orjson-3.10.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1952c03439e4dce23482ac846e7961f9d4ec62086eb98ae76d97bd41d72644d7"}, + {file = "orjson-3.10.3-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:c0403ed9c706dcd2809f1600ed18f4aae50be263bd7112e54b50e2c2bc3ebd6d"}, + {file = "orjson-3.10.3-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:382e52aa4270a037d41f325e7d1dfa395b7de0c367800b6f337d8157367bf3a7"}, + {file = "orjson-3.10.3-cp310-none-win32.whl", hash = "sha256:be2aab54313752c04f2cbaab4515291ef5af8c2256ce22abc007f89f42f49109"}, + {file = "orjson-3.10.3-cp310-none-win_amd64.whl", hash = "sha256:416b195f78ae461601893f482287cee1e3059ec49b4f99479aedf22a20b1098b"}, + {file = "orjson-3.10.3-cp311-cp311-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:73100d9abbbe730331f2242c1fc0bcb46a3ea3b4ae3348847e5a141265479700"}, + {file = "orjson-3.10.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:544a12eee96e3ab828dbfcb4d5a0023aa971b27143a1d35dc214c176fdfb29b3"}, + {file = "orjson-3.10.3-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:520de5e2ef0b4ae546bea25129d6c7c74edb43fc6cf5213f511a927f2b28148b"}, + {file = "orjson-3.10.3-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:ccaa0a401fc02e8828a5bedfd80f8cd389d24f65e5ca3954d72c6582495b4bcf"}, + {file = "orjson-3.10.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9a7bc9e8bc11bac40f905640acd41cbeaa87209e7e1f57ade386da658092dc16"}, + {file = "orjson-3.10.3-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:3582b34b70543a1ed6944aca75e219e1192661a63da4d039d088a09c67543b08"}, + {file = "orjson-3.10.3-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:1c23dfa91481de880890d17aa7b91d586a4746a4c2aa9a145bebdbaf233768d5"}, + {file = "orjson-3.10.3-cp311-none-win32.whl", hash = "sha256:1770e2a0eae728b050705206d84eda8b074b65ee835e7f85c919f5705b006c9b"}, + {file = "orjson-3.10.3-cp311-none-win_amd64.whl", hash = "sha256:93433b3c1f852660eb5abdc1f4dd0ced2be031ba30900433223b28ee0140cde5"}, + {file = "orjson-3.10.3-cp312-cp312-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:a39aa73e53bec8d410875683bfa3a8edf61e5a1c7bb4014f65f81d36467ea098"}, + {file = "orjson-3.10.3-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0943a96b3fa09bee1afdfccc2cb236c9c64715afa375b2af296c73d91c23eab2"}, + {file = "orjson-3.10.3-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e852baafceff8da3c9defae29414cc8513a1586ad93e45f27b89a639c68e8176"}, + {file = "orjson-3.10.3-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:18566beb5acd76f3769c1d1a7ec06cdb81edc4d55d2765fb677e3eaa10fa99e0"}, + {file = "orjson-3.10.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1bd2218d5a3aa43060efe649ec564ebedec8ce6ae0a43654b81376216d5ebd42"}, + {file = "orjson-3.10.3-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:cf20465e74c6e17a104ecf01bf8cd3b7b252565b4ccee4548f18b012ff2f8069"}, + {file = "orjson-3.10.3-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:ba7f67aa7f983c4345eeda16054a4677289011a478ca947cd69c0a86ea45e534"}, + {file = "orjson-3.10.3-cp312-none-win32.whl", hash = "sha256:17e0713fc159abc261eea0f4feda611d32eabc35708b74bef6ad44f6c78d5ea0"}, + {file = "orjson-3.10.3-cp312-none-win_amd64.whl", hash = "sha256:4c895383b1ec42b017dd2c75ae8a5b862fc489006afde06f14afbdd0309b2af0"}, + {file = "orjson-3.10.3-cp38-cp38-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:be2719e5041e9fb76c8c2c06b9600fe8e8584e6980061ff88dcbc2691a16d20d"}, + {file = "orjson-3.10.3-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cb0175a5798bdc878956099f5c54b9837cb62cfbf5d0b86ba6d77e43861bcec2"}, + {file = "orjson-3.10.3-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:978be58a68ade24f1af7758626806e13cff7748a677faf95fbb298359aa1e20d"}, + {file = "orjson-3.10.3-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:16bda83b5c61586f6f788333d3cf3ed19015e3b9019188c56983b5a299210eb5"}, + {file = "orjson-3.10.3-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4ad1f26bea425041e0a1adad34630c4825a9e3adec49079b1fb6ac8d36f8b754"}, + {file = "orjson-3.10.3-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:9e253498bee561fe85d6325ba55ff2ff08fb5e7184cd6a4d7754133bd19c9195"}, + {file = "orjson-3.10.3-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:0a62f9968bab8a676a164263e485f30a0b748255ee2f4ae49a0224be95f4532b"}, + {file = "orjson-3.10.3-cp38-none-win32.whl", hash = "sha256:8d0b84403d287d4bfa9bf7d1dc298d5c1c5d9f444f3737929a66f2fe4fb8f134"}, + {file = "orjson-3.10.3-cp38-none-win_amd64.whl", hash = "sha256:8bc7a4df90da5d535e18157220d7915780d07198b54f4de0110eca6b6c11e290"}, + {file = "orjson-3.10.3-cp39-cp39-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:9059d15c30e675a58fdcd6f95465c1522b8426e092de9fff20edebfdc15e1cb0"}, + {file = "orjson-3.10.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8d40c7f7938c9c2b934b297412c067936d0b54e4b8ab916fd1a9eb8f54c02294"}, + {file = "orjson-3.10.3-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d4a654ec1de8fdaae1d80d55cee65893cb06494e124681ab335218be6a0691e7"}, + {file = "orjson-3.10.3-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:831c6ef73f9aa53c5f40ae8f949ff7681b38eaddb6904aab89dca4d85099cb78"}, + {file = "orjson-3.10.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:99b880d7e34542db89f48d14ddecbd26f06838b12427d5a25d71baceb5ba119d"}, + {file = "orjson-3.10.3-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:2e5e176c994ce4bd434d7aafb9ecc893c15f347d3d2bbd8e7ce0b63071c52e25"}, + {file = "orjson-3.10.3-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:b69a58a37dab856491bf2d3bbf259775fdce262b727f96aafbda359cb1d114d8"}, + {file = "orjson-3.10.3-cp39-none-win32.whl", hash = "sha256:b8d4d1a6868cde356f1402c8faeb50d62cee765a1f7ffcfd6de732ab0581e063"}, + {file = "orjson-3.10.3-cp39-none-win_amd64.whl", hash = "sha256:5102f50c5fc46d94f2033fe00d392588564378260d64377aec702f21a7a22912"}, + {file = "orjson-3.10.3.tar.gz", hash = "sha256:2b166507acae7ba2f7c315dcf185a9111ad5e992ac81f2d507aac39193c2c818"}, +] + [[package]] name = "packaging" -version = "24.0" +version = "23.2" description = "Core utilities for Python packages" optional = false python-versions = ">=3.7" files = [ - {file = "packaging-24.0-py3-none-any.whl", hash = "sha256:2ddfb553fdf02fb784c234c7ba6ccc288296ceabec964ad2eae3777778130bc5"}, - {file = "packaging-24.0.tar.gz", hash = "sha256:eb82c5e3e56209074766e6885bb04b8c38a0c015d0a30036ebe7ece34c9989e9"}, + {file = "packaging-23.2-py3-none-any.whl", hash = "sha256:8c491190033a9af7e1d931d0b5dacc2ef47509b34dd0de67ed209b5203fc88c7"}, + {file = "packaging-23.2.tar.gz", hash = "sha256:048fb0e9405036518eaaf48a55953c750c11e1a1b68e0dd1a9d62ed0c092cfc5"}, ] [[package]] @@ -542,28 +781,29 @@ pytzdata = ">=2020.1" [[package]] name = "platformdirs" -version = "4.2.0" -description = "A small Python package for determining appropriate platform-specific dirs, e.g. a \"user data dir\"." +version = "4.2.1" +description = "A small Python package for determining appropriate platform-specific dirs, e.g. a `user data dir`." optional = false python-versions = ">=3.8" files = [ - {file = "platformdirs-4.2.0-py3-none-any.whl", hash = "sha256:0614df2a2f37e1a662acbd8e2b25b92ccf8632929bc6d43467e17fe89c75e068"}, - {file = "platformdirs-4.2.0.tar.gz", hash = "sha256:ef0cc731df711022c174543cb70a9b5bd22e5a9337c8624ef2c2ceb8ddad8768"}, + {file = "platformdirs-4.2.1-py3-none-any.whl", hash = "sha256:17d5a1161b3fd67b390023cb2d3b026bbd40abde6fdb052dfbd3a29c3ba22ee1"}, + {file = "platformdirs-4.2.1.tar.gz", hash = "sha256:031cd18d4ec63ec53e82dceaac0417d218a6863f7745dfcc9efe7793b7039bdf"}, ] [package.extras] docs = ["furo (>=2023.9.10)", "proselint (>=0.13)", "sphinx (>=7.2.6)", "sphinx-autodoc-typehints (>=1.25.2)"] test = ["appdirs (==1.4.4)", "covdefaults (>=2.3)", "pytest (>=7.4.3)", "pytest-cov (>=4.1)", "pytest-mock (>=3.12)"] +type = ["mypy (>=1.8)"] [[package]] name = "pluggy" -version = "1.4.0" +version = "1.5.0" description = "plugin and hook calling mechanisms for python" optional = false python-versions = ">=3.8" files = [ - {file = "pluggy-1.4.0-py3-none-any.whl", hash = "sha256:7db9f7b503d67d1c5b95f59773ebb58a8c1c288129a88665838012cfb07b8981"}, - {file = "pluggy-1.4.0.tar.gz", hash = "sha256:8c85c2876142a764e5b7548e7d9a0e0ddb46f5185161049a79b7e974454223be"}, + {file = "pluggy-1.5.0-py3-none-any.whl", hash = "sha256:44e1ad92c8ca002de6377e165f3e0f1be63266ab4d554740532335b9d75ea669"}, + {file = "pluggy-1.5.0.tar.gz", hash = "sha256:2cffa88e94fdc978c4c574f15f9e59b7f4201d439195c3715ca9e2486f1d0cf1"}, ] [package.extras] @@ -581,6 +821,17 @@ files = [ {file = "py-1.11.0.tar.gz", hash = "sha256:51c75c4126074b472f746a24399ad32f6053d1b34b68d2fa41e558e6f4a98719"}, ] +[[package]] +name = "pycparser" +version = "2.22" +description = "C parser in Python" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pycparser-2.22-py3-none-any.whl", hash = "sha256:c3702b6d3dd8c7abc1afa565d7e63d53a1d0bd86cdc24edd75470f4de499cfcc"}, + {file = "pycparser-2.22.tar.gz", hash = "sha256:491c8be9c040f5390f5bf44a5b07752bd07f56edf992381b05c701439eec10f6"}, +] + [[package]] name = "pydantic" version = "1.10.15" @@ -633,6 +884,23 @@ typing-extensions = ">=4.2.0" dotenv = ["python-dotenv (>=0.10.4)"] email = ["email-validator (>=1.0.3)"] +[[package]] +name = "pyjwt" +version = "2.8.0" +description = "JSON Web Token implementation in Python" +optional = false +python-versions = ">=3.7" +files = [ + {file = "PyJWT-2.8.0-py3-none-any.whl", hash = "sha256:59127c392cc44c2da5bb3192169a91f429924e17aff6534d70fdc02ab3e04320"}, + {file = "PyJWT-2.8.0.tar.gz", hash = "sha256:57e28d156e3d5c10088e0c68abb90bfac3df82b40a71bd0daa20c65ccd5c23de"}, +] + +[package.extras] +crypto = ["cryptography (>=3.4.0)"] +dev = ["coverage[toml] (==5.0.4)", "cryptography (>=3.4.0)", "pre-commit", "pytest (>=6.0.0,<7.0.0)", "sphinx (>=4.5.0,<5.0.0)", "sphinx-rtd-theme", "zope.interface"] +docs = ["sphinx (>=4.5.0,<5.0.0)", "sphinx-rtd-theme", "zope.interface"] +tests = ["coverage[toml] (==5.0.4)", "pytest (>=6.0.0,<7.0.0)"] + [[package]] name = "pyrate-limiter" version = "3.1.1" @@ -896,18 +1164,18 @@ fixture = ["fixtures"] [[package]] name = "setuptools" -version = "69.2.0" +version = "69.5.1" description = "Easily download, build, install, upgrade, and uninstall Python packages" optional = false python-versions = ">=3.8" files = [ - {file = "setuptools-69.2.0-py3-none-any.whl", hash = "sha256:c21c49fb1042386df081cb5d86759792ab89efca84cf114889191cd09aacc80c"}, - {file = "setuptools-69.2.0.tar.gz", hash = "sha256:0ff4183f8f42cd8fa3acea16c45205521a4ef28f73c6391d8a25e92893134f2e"}, + {file = "setuptools-69.5.1-py3-none-any.whl", hash = "sha256:c636ac361bc47580504644275c9ad802c50415c7522212252c033bd15f301f32"}, + {file = "setuptools-69.5.1.tar.gz", hash = "sha256:6c1fccdac05a97e598fb0ae3bbed5904ccb317337a51139dcd51453611bbb987"}, ] [package.extras] -docs = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "rst.linker (>=1.9)", "sphinx (<7.2.5)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-inline-tabs", "sphinx-lint", "sphinx-notfound-page (>=1,<2)", "sphinx-reredirects", "sphinxcontrib-towncrier"] -testing = ["build[virtualenv]", "filelock (>=3.4.0)", "importlib-metadata", "ini2toml[lite] (>=0.9)", "jaraco.develop (>=7.21)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "mypy (==1.9)", "packaging (>=23.2)", "pip (>=19.1)", "pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-home (>=0.5)", "pytest-mypy (>=0.9.1)", "pytest-perf", "pytest-ruff (>=0.2.1)", "pytest-timeout", "pytest-xdist (>=3)", "tomli", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"] +docs = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-inline-tabs", "sphinx-lint", "sphinx-notfound-page (>=1,<2)", "sphinx-reredirects", "sphinxcontrib-towncrier"] +testing = ["build[virtualenv]", "filelock (>=3.4.0)", "importlib-metadata", "ini2toml[lite] (>=0.9)", "jaraco.develop (>=7.21)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "mypy (==1.9)", "packaging (>=23.2)", "pip (>=19.1)", "pytest (>=6,!=8.1.1)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-home (>=0.5)", "pytest-mypy", "pytest-perf", "pytest-ruff (>=0.2.1)", "pytest-timeout", "pytest-xdist (>=3)", "tomli", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"] testing-integration = ["build[virtualenv] (>=1.0.3)", "filelock (>=3.4.0)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "packaging (>=23.2)", "pytest", "pytest-enabler", "pytest-xdist", "tomli", "virtualenv (>=13.0.0)", "wheel"] [[package]] @@ -921,6 +1189,21 @@ files = [ {file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"}, ] +[[package]] +name = "tenacity" +version = "8.3.0" +description = "Retry code until it succeeds" +optional = false +python-versions = ">=3.8" +files = [ + {file = "tenacity-8.3.0-py3-none-any.whl", hash = "sha256:3649f6443dbc0d9b01b9d8020a9c4ec7a1ff5f6f3c6c8a036ef371f573fe9185"}, + {file = "tenacity-8.3.0.tar.gz", hash = "sha256:953d4e6ad24357bceffbc9707bc74349aca9d245f68eb65419cf0c249a1949a2"}, +] + +[package.extras] +doc = ["reno", "sphinx"] +test = ["pytest", "tornado (>=4.5)", "typeguard"] + [[package]] name = "toml" version = "0.10.2" @@ -934,13 +1217,13 @@ files = [ [[package]] name = "typing-extensions" -version = "4.10.0" +version = "4.11.0" description = "Backported and Experimental Type Hints for Python 3.8+" optional = false python-versions = ">=3.8" files = [ - {file = "typing_extensions-4.10.0-py3-none-any.whl", hash = "sha256:69b1a937c3a517342112fb4c6df7e72fc39a38e7891a5730ed4985b5214b5475"}, - {file = "typing_extensions-4.10.0.tar.gz", hash = "sha256:b0abd7c89e8fb96f98db18d86106ff1d90ab692004eb746cf6eda2682f91b3cb"}, + {file = "typing_extensions-4.11.0-py3-none-any.whl", hash = "sha256:c1f94d72897edaf4ce775bb7558d5b79d8126906a14ea5ed1635921406c0387a"}, + {file = "typing_extensions-4.11.0.tar.gz", hash = "sha256:83f085bd5ca59c80295fc2a82ab5dac679cbe02b9f33f7d83af68e241bea51b0"}, ] [[package]] diff --git a/airbyte-integrations/connectors/source-hubspot/source_hubspot/streams.py b/airbyte-integrations/connectors/source-hubspot/source_hubspot/streams.py index 4296b864d574..aabb27fcbc11 100644 --- a/airbyte-integrations/connectors/source-hubspot/source_hubspot/streams.py +++ b/airbyte-integrations/connectors/source-hubspot/source_hubspot/streams.py @@ -18,7 +18,7 @@ from airbyte_cdk.entrypoint import logger from airbyte_cdk.models import FailureType, SyncMode from airbyte_cdk.sources import Source -from airbyte_cdk.sources.streams import IncrementalMixin, Stream +from airbyte_cdk.sources.streams import CheckpointMixin, Stream from airbyte_cdk.sources.streams.availability_strategy import AvailabilityStrategy from airbyte_cdk.sources.streams.core import StreamData from airbyte_cdk.sources.streams.http import HttpStream, HttpSubStream @@ -838,7 +838,7 @@ def availability_strategy(self) -> Optional[AvailabilityStrategy]: return HubspotAvailabilityStrategy() -class ClientSideIncrementalStream(Stream, IncrementalMixin): +class ClientSideIncrementalStream(Stream, CheckpointMixin): _cursor_value = "" @property @@ -861,7 +861,10 @@ def state(self) -> Mapping[str, Any]: @state.setter def state(self, value: Mapping[str, Any]): - self._cursor_value = value[self.cursor_field] + if value: + self._cursor_value = value[self.cursor_field] + else: + self._cursor_value = "" def filter_by_state(self, stream_state: Mapping[str, Any] = None, record: Mapping[str, Any] = None) -> bool: """ @@ -1232,7 +1235,7 @@ def stream_slices( self, *, sync_mode: SyncMode, cursor_field: List[str] = None, stream_state: Mapping[str, Any] = None ) -> Iterable[Optional[Mapping[str, Any]]]: self.set_sync(sync_mode, stream_state) - return [None] + return [{}] # I changed this from [None] since this is a more accurate depiction of what is actually being done. Sync one slice def set_sync(self, sync_mode: SyncMode, stream_state): self._sync_mode = sync_mode @@ -1361,7 +1364,7 @@ class ContactLists(IncrementalStream): unnest_fields = ["metaData"] -class ContactsAllBase(Stream): +class ContactsAllBase(Stream, CheckpointMixin): url = "/contacts/v1/lists/all/contacts/all" updated_at_field = "timestamp" more_key = "has-more" @@ -1374,6 +1377,62 @@ class ContactsAllBase(Stream): records_field = None filter_field = None filter_value = None + _state = {} + limit_field = "count" + limit = 100 + + @property + def state(self) -> MutableMapping[str, Any]: + return self._state + + @state.setter + def state(self, value: MutableMapping[str, Any]) -> None: + self._state = value + + def read_records( + self, + sync_mode: SyncMode, + cursor_field: List[str] = None, + stream_slice: Mapping[str, Any] = None, + stream_state: Mapping[str, Any] = None, + ) -> Iterable[Mapping[str, Any]]: + """ + This is a specialized read_records for resumable full refresh that only attempts to read a single page of records + at a time and updates the state w/ a synthetic cursor based on the Hubspot cursor pagination value `vidOffset` + """ + + next_page_token = stream_slice + try: + properties = self._property_wrapper + if properties and properties.too_many_properties: + records, response = self._read_stream_records( + stream_slice=stream_slice, + stream_state=stream_state, + next_page_token=next_page_token, + ) + else: + response = self.handle_request( + stream_slice=stream_slice, + stream_state=stream_state, + next_page_token=next_page_token, + properties=properties, + ) + records = self._transform(self.parse_response(response, stream_state=stream_state, stream_slice=stream_slice)) + + if self.filter_old_records: + records = self._filter_old_records(records) + yield from self.record_unnester.unnest(records) + + self.state = self.next_page_token(response) or {} + + # Always return an empty generator just in case no records were ever yielded + yield from [] + except requests.exceptions.HTTPError as e: + response = e.response + if response.status_code == HTTPStatus.UNAUTHORIZED: + raise AirbyteTracedException("The authentication to HubSpot has expired. Re-authenticate to restore access to HubSpot.") + else: + raise e def _transform(self, records: Iterable) -> Iterable: for record in super()._transform(records): @@ -1415,6 +1474,12 @@ class ContactsFormSubmissions(ContactsAllBase, ABC): filter_value = "all" +class ContactsMergedAudit(ContactsAllBase, ABC): + + records_field = "merge-audits" + unnest_fields = ["merged_from_email", "merged_to_email"] + + class Deals(CRMSearchStream): """Deals, API v3""" @@ -2101,65 +2166,6 @@ class Contacts(CRMSearchStream): scopes = {"crm.objects.contacts.read"} -class ContactsMergedAudit(Stream): - url = "/contacts/v1/contact/vids/batch/" - updated_at_field = "timestamp" - scopes = {"crm.objects.contacts.read"} - unnest_fields = ["merged_from_email", "merged_to_email"] - - def __init__(self, **kwargs): - super().__init__(**kwargs) - self.config = kwargs - - def stream_slices( - self, sync_mode: SyncMode, cursor_field: List[str] = None, stream_state: Mapping[str, Any] = None, **kwargs - ) -> Iterable[Mapping[str, Any]]: - slices = [] - - # we can query a max of 100 contacts at a time - max_contacts = 100 - slices = [] - contact_batch = [] - - contacts = Contacts(**self.config) - contacts._sync_mode = SyncMode.full_refresh - contacts.filter_old_records = False - - for contact in contacts.read_records(sync_mode=SyncMode.full_refresh): - if contact.get("properties_hs_merged_object_ids"): - contact_batch.append(contact["id"]) - - if len(contact_batch) == max_contacts: - slices.append({"vid": contact_batch}) - contact_batch = [] - - if contact_batch: - slices.append({"vid": contact_batch}) - - return slices - - def request_params( - self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None - ) -> MutableMapping[str, Any]: - return {"vid": stream_slice["vid"]} - - def parse_response( - self, - response: requests.Response, - *, - stream_state: Mapping[str, Any], - stream_slice: Mapping[str, Any] = None, - next_page_token: Mapping[str, Any] = None, - ) -> Iterable[Mapping]: - response = self._parse_response(response) - if response.get("status", None) == "error": - self.logger.warning(f"Stream `{self.name}` cannot be procced. {response.get('message')}") - return - - for contact_id in list(response.keys()): - yield from response[contact_id]["merge-audits"] - - class EngagementsCalls(CRMSearchStream): entity = "calls" last_modified_field = "hs_lastmodifieddate" @@ -2273,7 +2279,7 @@ class EmailSubscriptions(Stream): filter_old_records = False -class WebAnalyticsStream(IncrementalMixin, HttpSubStream, Stream): +class WebAnalyticsStream(CheckpointMixin, HttpSubStream, Stream): """ A base class for Web Analytics API Docs: https://developers.hubspot.com/docs/api/events/web-analytics diff --git a/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/__init__.py b/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/__init__.py index f8ebf27a08c2..b3368edfa81e 100644 --- a/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/__init__.py +++ b/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/__init__.py @@ -18,6 +18,7 @@ from .request_builders.streams import CRMStreamRequestBuilder, IncrementalCRMStreamRequestBuilder, WebAnalyticsRequestBuilder from .response_builder.helpers import RootHttpResponseBuilder from .response_builder.api import ScopesResponseBuilder +from .response_builder.pagination import HubspotCursorPaginationStrategy from .response_builder.streams import GenericResponseBuilder, HubspotStreamResponseBuilder @@ -131,3 +132,28 @@ def read_from_stream( cls, cfg, stream: str, sync_mode: SyncMode, state: Optional[List[AirbyteStateMessage]] = None, expecting_exception: bool = False ) -> EntrypointOutput: return read(SourceHubspot(), cfg, cls.catalog(stream, sync_mode), state, expecting_exception) + + +@freezegun.freeze_time("2024-05-05T00:00:00Z") +class HubspotContactsTestCase(HubspotTestCase): + CHECKPOINT_FIELD = "vid" + + @classmethod + def response_builder(cls, stream_name) -> HubspotStreamResponseBuilder: + return HubspotStreamResponseBuilder.for_stream(stream_name, "contacts", HubspotCursorPaginationStrategy()) + + @classmethod + def response(cls, stream_name, with_pagination: bool = False) -> HubspotStreamResponseBuilder: + record = cls.record_builder(stream_name, FieldPath(cls.CHECKPOINT_FIELD)).with_field( + FieldPath("id"), cls.OBJECT_ID + ) + response = cls.response_builder(stream_name=stream_name).with_record(record).with_record(record) + if with_pagination: + response = response.with_pagination() + return response + + @classmethod + def record_builder(cls, stream: str, record_cursor_path): + return create_record_builder( + find_template(stream, __file__), records_path=FieldPath("contacts"), record_id_path=None, record_cursor_path=record_cursor_path + ) diff --git a/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/request_builders/streams.py b/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/request_builders/streams.py index 10c2519080f7..a978716a104a 100644 --- a/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/request_builders/streams.py +++ b/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/request_builders/streams.py @@ -135,3 +135,37 @@ def with_page_token(self, next_page_token: Dict): def build(self): q = "&".join(filter(None, self._query_params)) return HttpRequest(self.URL, query_params=q) + + +# We only need to mock the Contacts endpoint because it services the data extracted by ListMemberships, FormSubmissions, MergedAudit +class ContactsStreamRequestBuilder(AbstractRequestBuilder): + URL = "https://api.hubapi.com/contacts/v1/lists/all/contacts/all" + + def __init__(self): + self._filters = [] + self._vid_offset = None + + @property + def _count(self): + return "count=100" + + def with_filter(self, filter_field: str, filter_value: Any): + self._filters.append(f"{filter_field}={filter_value}") + return self + + def with_vid_offset(self, vid_offset: str): + self._vid_offset = f"vidOffset={vid_offset}" + return self + + @property + def _query_params(self): + params = [ + self._count, + self._vid_offset, + ] + params.extend(self._filters) + return filter(None, params) + + def build(self): + q = "&".join(filter(None, self._query_params)) + return HttpRequest(self.URL, query_params=q) diff --git a/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/response_builder/pagination.py b/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/response_builder/pagination.py index ded25153204f..c779f836fb7a 100644 --- a/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/response_builder/pagination.py +++ b/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/response_builder/pagination.py @@ -19,3 +19,9 @@ def update(self, response: Dict[str, Any]) -> None: "link": None } } + + +class HubspotCursorPaginationStrategy(PaginationStrategy): + def update(self, response: Dict[str, Any]) -> None: + response["has-more"] = True + response["vid-offset"] = "5331889818" diff --git a/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/response_builder/streams.py b/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/response_builder/streams.py index 6b02d952e14c..35d140464015 100644 --- a/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/response_builder/streams.py +++ b/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/response_builder/streams.py @@ -3,10 +3,9 @@ import json from airbyte_cdk.test.mock_http import HttpResponse -from airbyte_cdk.test.mock_http.response_builder import FieldPath, HttpResponseBuilder, find_template +from airbyte_cdk.test.mock_http.response_builder import FieldPath, HttpResponseBuilder, PaginationStrategy, find_template from . import AbstractResponseBuilder -from .pagination import HubspotPaginationStrategy class HubspotStreamResponseBuilder(HttpResponseBuilder): @@ -15,8 +14,8 @@ def pagination_strategy(self): return self._pagination_strategy @classmethod - def for_stream(cls, stream: str): - return cls(find_template(stream, __file__), FieldPath("results"), HubspotPaginationStrategy()) + def for_stream(cls, stream: str, records_path: str, pagination_strategy: PaginationStrategy): + return cls(find_template(stream, __file__), FieldPath(records_path), pagination_strategy) class GenericResponseBuilder(AbstractResponseBuilder): diff --git a/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/test_contacts_form_submissions.py b/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/test_contacts_form_submissions.py new file mode 100644 index 000000000000..ce2a8dc1633a --- /dev/null +++ b/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/test_contacts_form_submissions.py @@ -0,0 +1,60 @@ +# Copyright (c) 2024 Airbyte, Inc., all rights reserved. + +import freezegun +from airbyte_cdk.test.mock_http import HttpMocker +from airbyte_protocol.models import AirbyteStateBlob, AirbyteStateMessage, AirbyteStateType, AirbyteStreamState, StreamDescriptor, SyncMode + +from . import HubspotContactsTestCase +from .request_builders.streams import ContactsStreamRequestBuilder + + +@freezegun.freeze_time("2024-05-04T00:00:00Z") +class TestContactsFormSubmissionsStream(HubspotContactsTestCase): + SCOPES = ["crm.objects.contacts.read"] + STREAM_NAME = "contacts_form_submissions" + + @HttpMocker() + def test_read_multiple_contact_pages(self, http_mocker: HttpMocker): + self.mock_oauth(http_mocker, self.ACCESS_TOKEN) + self.mock_scopes(http_mocker, self.ACCESS_TOKEN, self.SCOPES) + self.mock_custom_objects(http_mocker) + + self.mock_response(http_mocker, ContactsStreamRequestBuilder().with_filter("formSubmissionMode", "all").build(), self.response(stream_name=self.STREAM_NAME, with_pagination=True).build()) + self.mock_response(http_mocker, ContactsStreamRequestBuilder().with_filter("formSubmissionMode", "all").with_vid_offset("5331889818").build(), self.response(stream_name=self.STREAM_NAME).build()) + + output = self.read_from_stream(self.oauth_config(), self.STREAM_NAME, SyncMode.full_refresh) + + assert len(output.records) == 16 + assert output.state_messages[0].state.stream.stream_state.dict() == {"vidOffset": "5331889818"} + assert output.state_messages[0].state.stream.stream_descriptor.name == self.STREAM_NAME + assert output.state_messages[0].state.sourceStats.recordCount == 8 + assert output.state_messages[1].state.stream.stream_state.dict() == {} + assert output.state_messages[1].state.stream.stream_descriptor.name == self.STREAM_NAME + assert output.state_messages[1].state.sourceStats.recordCount == 8 + + @HttpMocker() + def test_read_from_incoming_state(self, http_mocker: HttpMocker): + state = [ + AirbyteStateMessage( + type=AirbyteStateType.STREAM, + stream=AirbyteStreamState( + stream_descriptor=StreamDescriptor(name=self.STREAM_NAME), + stream_state=AirbyteStateBlob(**{"vidOffset": "5331889818"}) + ) + ) + ] + + self.mock_oauth(http_mocker, self.ACCESS_TOKEN) + self.mock_scopes(http_mocker, self.ACCESS_TOKEN, self.SCOPES) + self.mock_custom_objects(http_mocker) + + # Even though we only care about the request with a vidOffset parameter, we mock this in order to pass the availability check + self.mock_response(http_mocker, ContactsStreamRequestBuilder().with_filter("formSubmissionMode", "all").build(), self.response(stream_name=self.STREAM_NAME, with_pagination=True).build()) + self.mock_response(http_mocker, ContactsStreamRequestBuilder().with_filter("formSubmissionMode", "all").with_vid_offset("5331889818").build(), self.response(stream_name=self.STREAM_NAME).build()) + + output = self.read_from_stream(cfg=self.oauth_config(), stream=self.STREAM_NAME, sync_mode=SyncMode.full_refresh, state=state) + + assert len(output.records) == 8 + assert output.state_messages[0].state.stream.stream_state.dict() == {} + assert output.state_messages[0].state.stream.stream_descriptor.name == self.STREAM_NAME + assert output.state_messages[0].state.sourceStats.recordCount == 8 diff --git a/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/test_contacts_list_memberships.py b/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/test_contacts_list_memberships.py new file mode 100644 index 000000000000..c49933ea1ffd --- /dev/null +++ b/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/test_contacts_list_memberships.py @@ -0,0 +1,60 @@ +# Copyright (c) 2024 Airbyte, Inc., all rights reserved. + +import freezegun +from airbyte_cdk.test.mock_http import HttpMocker +from airbyte_protocol.models import AirbyteStateBlob, AirbyteStateMessage, AirbyteStateType, AirbyteStreamState, StreamDescriptor, SyncMode + +from . import HubspotContactsTestCase +from .request_builders.streams import ContactsStreamRequestBuilder + + +@freezegun.freeze_time("2024-05-04T00:00:00Z") +class TestContactsListMembershipsStream(HubspotContactsTestCase): + SCOPES = ["crm.objects.contacts.read"] + STREAM_NAME = "contacts_list_memberships" + + @HttpMocker() + def test_read_multiple_contact_pages(self, http_mocker: HttpMocker): + self.mock_oauth(http_mocker, self.ACCESS_TOKEN) + self.mock_scopes(http_mocker, self.ACCESS_TOKEN, self.SCOPES) + self.mock_custom_objects(http_mocker) + + self.mock_response(http_mocker, ContactsStreamRequestBuilder().with_filter("showListMemberships", True).build(), self.response(stream_name=self.STREAM_NAME, with_pagination=True).build()) + self.mock_response(http_mocker, ContactsStreamRequestBuilder().with_filter("showListMemberships", True).with_vid_offset("5331889818").build(), self.response(stream_name=self.STREAM_NAME).build()) + + output = self.read_from_stream(self.oauth_config(), self.STREAM_NAME, SyncMode.full_refresh) + + assert len(output.records) == 12 + assert output.state_messages[0].state.stream.stream_state.dict() == {"vidOffset": "5331889818"} + assert output.state_messages[0].state.stream.stream_descriptor.name == self.STREAM_NAME + assert output.state_messages[0].state.sourceStats.recordCount == 6 + assert output.state_messages[1].state.stream.stream_state.dict() == {} + assert output.state_messages[1].state.stream.stream_descriptor.name == self.STREAM_NAME + assert output.state_messages[1].state.sourceStats.recordCount == 6 + + @HttpMocker() + def test_read_from_incoming_state(self, http_mocker: HttpMocker): + state = [ + AirbyteStateMessage( + type=AirbyteStateType.STREAM, + stream=AirbyteStreamState( + stream_descriptor=StreamDescriptor(name=self.STREAM_NAME), + stream_state=AirbyteStateBlob(**{"vidOffset": "5331889818"}) + ) + ) + ] + + self.mock_oauth(http_mocker, self.ACCESS_TOKEN) + self.mock_scopes(http_mocker, self.ACCESS_TOKEN, self.SCOPES) + self.mock_custom_objects(http_mocker) + + # Even though we only care about the request with a vidOffset parameter, we mock this in order to pass the availability check + self.mock_response(http_mocker, ContactsStreamRequestBuilder().with_filter("showListMemberships", True).build(), self.response(stream_name=self.STREAM_NAME, with_pagination=True).build()) + self.mock_response(http_mocker, ContactsStreamRequestBuilder().with_filter("showListMemberships", True).with_vid_offset("5331889818").build(), self.response(stream_name=self.STREAM_NAME).build()) + + output = self.read_from_stream(cfg=self.oauth_config(), stream=self.STREAM_NAME, sync_mode=SyncMode.full_refresh, state=state) + + assert len(output.records) == 6 + assert output.state_messages[0].state.stream.stream_state.dict() == {} + assert output.state_messages[0].state.stream.stream_descriptor.name == self.STREAM_NAME + assert output.state_messages[0].state.sourceStats.recordCount == 6 diff --git a/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/test_contacts_merged_audit.py b/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/test_contacts_merged_audit.py new file mode 100644 index 000000000000..0b119f8bf505 --- /dev/null +++ b/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/test_contacts_merged_audit.py @@ -0,0 +1,60 @@ +# Copyright (c) 2024 Airbyte, Inc., all rights reserved. + +import freezegun +from airbyte_cdk.test.mock_http import HttpMocker +from airbyte_protocol.models import AirbyteStateBlob, AirbyteStateMessage, AirbyteStateType, AirbyteStreamState, StreamDescriptor, SyncMode + +from . import HubspotContactsTestCase +from .request_builders.streams import ContactsStreamRequestBuilder + + +@freezegun.freeze_time("2024-05-04T00:00:00Z") +class TestContactsFormSubmissionsStream(HubspotContactsTestCase): + SCOPES = ["crm.objects.contacts.read"] + STREAM_NAME = "contacts_merged_audit" + + @HttpMocker() + def test_read_multiple_contact_pages(self, http_mocker: HttpMocker): + self.mock_oauth(http_mocker, self.ACCESS_TOKEN) + self.mock_scopes(http_mocker, self.ACCESS_TOKEN, self.SCOPES) + self.mock_custom_objects(http_mocker) + + self.mock_response(http_mocker, ContactsStreamRequestBuilder().build(), self.response(stream_name=self.STREAM_NAME, with_pagination=True).build()) + self.mock_response(http_mocker, ContactsStreamRequestBuilder().with_vid_offset("5331889818").build(), self.response(stream_name=self.STREAM_NAME).build()) + + output = self.read_from_stream(self.oauth_config(), self.STREAM_NAME, SyncMode.full_refresh) + + assert len(output.records) == 8 + assert output.state_messages[0].state.stream.stream_state.dict() == {"vidOffset": "5331889818"} + assert output.state_messages[0].state.stream.stream_descriptor.name == self.STREAM_NAME + assert output.state_messages[0].state.sourceStats.recordCount == 4 + assert output.state_messages[1].state.stream.stream_state.dict() == {} + assert output.state_messages[1].state.stream.stream_descriptor.name == self.STREAM_NAME + assert output.state_messages[1].state.sourceStats.recordCount == 4 + + @HttpMocker() + def test_read_from_incoming_state(self, http_mocker: HttpMocker): + state = [ + AirbyteStateMessage( + type=AirbyteStateType.STREAM, + stream=AirbyteStreamState( + stream_descriptor=StreamDescriptor(name=self.STREAM_NAME), + stream_state=AirbyteStateBlob(**{"vidOffset": "5331889818"}) + ) + ) + ] + + self.mock_oauth(http_mocker, self.ACCESS_TOKEN) + self.mock_scopes(http_mocker, self.ACCESS_TOKEN, self.SCOPES) + self.mock_custom_objects(http_mocker) + + # Even though we only care about the request with a vidOffset parameter, we mock this in order to pass the availability check + self.mock_response(http_mocker, ContactsStreamRequestBuilder().build(), self.response(stream_name=self.STREAM_NAME, with_pagination=True).build()) + self.mock_response(http_mocker, ContactsStreamRequestBuilder().with_vid_offset("5331889818").build(), self.response(stream_name=self.STREAM_NAME).build()) + + output = self.read_from_stream(cfg=self.oauth_config(), stream=self.STREAM_NAME, sync_mode=SyncMode.full_refresh, state=state) + + assert len(output.records) == 4 + assert output.state_messages[0].state.stream.stream_state.dict() == {} + assert output.state_messages[0].state.stream.stream_descriptor.name == self.STREAM_NAME + assert output.state_messages[0].state.sourceStats.recordCount == 4 diff --git a/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/test_engagements_calls.py b/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/test_engagements_calls.py index 395be501e488..d2243588b7e5 100644 --- a/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/test_engagements_calls.py +++ b/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/test_engagements_calls.py @@ -11,6 +11,7 @@ from . import HubspotTestCase from .request_builders.streams import CRMStreamRequestBuilder +from .response_builder.pagination import HubspotPaginationStrategy from .response_builder.streams import HubspotStreamResponseBuilder @@ -24,7 +25,7 @@ class TestEngagementCallsStream(HubspotTestCase): @property def response_builder(self): - return HubspotStreamResponseBuilder.for_stream(self.STREAM_NAME) + return HubspotStreamResponseBuilder.for_stream(self.STREAM_NAME, "results", HubspotPaginationStrategy()) def request(self, page_token: Optional[Dict[str, str]] = None): request_builder = CRMStreamRequestBuilder().for_entity( diff --git a/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/test_owners_archived.py b/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/test_owners_archived.py index d54a94ca6a9d..e5b932a5086d 100644 --- a/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/test_owners_archived.py +++ b/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/test_owners_archived.py @@ -7,6 +7,7 @@ from . import HubspotTestCase from .request_builders.streams import OwnersArchivedStreamRequestBuilder +from .response_builder.pagination import HubspotPaginationStrategy from .response_builder.streams import HubspotStreamResponseBuilder @@ -25,7 +26,7 @@ def request(self): @property def response_builder(self): - return HubspotStreamResponseBuilder.for_stream(self.STREAM_NAME) + return HubspotStreamResponseBuilder.for_stream(self.STREAM_NAME, "results", HubspotPaginationStrategy()) def response(self, with_pagination: bool = False): record = self.record_builder(self.STREAM_NAME, FieldPath(self.CURSOR_FIELD)).with_field( diff --git a/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/test_web_analytics_streams.py b/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/test_web_analytics_streams.py index 3f1ceff5b398..0656a2e7c8b0 100644 --- a/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/test_web_analytics_streams.py +++ b/airbyte-integrations/connectors/source-hubspot/unit_tests/integrations/test_web_analytics_streams.py @@ -14,6 +14,7 @@ from . import HubspotTestCase from .request_builders.streams import CRMStreamRequestBuilder, IncrementalCRMStreamRequestBuilder, WebAnalyticsRequestBuilder +from .response_builder.pagination import HubspotPaginationStrategy from .response_builder.streams import HubspotStreamResponseBuilder CRM_STREAMS = ( @@ -40,7 +41,7 @@ class WebAnalyticsTestCase(HubspotTestCase): @classmethod def response_builder(cls, stream): - return HubspotStreamResponseBuilder.for_stream(stream) + return HubspotStreamResponseBuilder.for_stream(stream, "results", HubspotPaginationStrategy()) @classmethod def web_analytics_request( diff --git a/airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/contacts_form_submissions.json b/airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/contacts_form_submissions.json new file mode 100644 index 000000000000..1c1fb2b1edce --- /dev/null +++ b/airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/contacts_form_submissions.json @@ -0,0 +1,186 @@ +{ + "contacts": [ + { + "vid": 5331889818, + "canonical-vid": 5331889818, + "merged-vids": [], + "portal-id": 8727216, + "is-contact": true, + "properties": { + "firstname": { + "value": "Eric" + }, + "lastmodifieddate": { + "value": "1713702782727" + }, + "lastname": { + "value": "Harris" + } + }, + "form-submissions": [ + { + "conversion-id": "2c7b8485-62e8-41e3-95a1-ac429358201c", + "timestamp": 1714546800000, + "form-id": "8372afd9-33b9-458e-ac20-58665140554b", + "portal-id": 8727216, + "title": "When Will We Land? - Barry Can't Swim", + "form-type": "HUBSPOT", + "contact-associated-by": ["EMAIL"], + "meta-data": [] + }, + { + "conversion-id": "0ee58538-4825-46bc-a25a-03d78f5fd072", + "timestamp": 1714546800000, + "form-id": "15905b83-4431-414c-8a0c-4bfd1e819290", + "portal-id": 8727216, + "title": "Favorite bending element?", + "form-type": "HUBSPOT", + "contact-associated-by": ["EMAIL"], + "meta-data": [] + }, + { + "conversion-id": "c0681fc8-352c-4017-9076-85e31fc25886", + "timestamp": 1714546800000, + "form-id": "709f8e27-21be-4d8f-9774-29311a069780", + "portal-id": 8727216, + "title": "Favorite Bakeries in San Francisco", + "form-type": "HUBSPOT", + "contact-associated-by": ["EMAIL"], + "meta-data": [] + }, + { + "conversion-id": "f8c781f4-a7e8-4bcb-b501-92f69673d30d", + "timestamp": 1714546800000, + "form-id": "bf46b791-448d-483c-bf1c-e5099c43cbff", + "portal-id": 8727216, + "title": "Brian's Test Form", + "form-type": "HUBSPOT", + "contact-associated-by": ["EMAIL"], + "meta-data": [] + } + ], + "list-memberships": [ + { + "static-list-id": 166, + "internal-list-id": 2147483643, + "timestamp": 1711049481683, + "vid": 5331889818, + "is-member": true + } + ], + "identity-profiles": [ + { + "vid": 5331889818, + "saved-at-timestamp": 1711049462684, + "deleted-changed-timestamp": 0, + "identities": [ + { + "type": "EMAIL", + "value": "eric.harris@airbyte-x.com", + "timestamp": 1711049462548, + "is-primary": true + }, + { + "type": "LEAD_GUID", + "value": "33671d61-e5fe-43c0-84ff-2ad08329b008", + "timestamp": 1711049462680 + } + ] + } + ], + "merge-audits": [], + "addedAt": 1711049462684 + }, + { + "vid": 5331890527, + "canonical-vid": 5331890527, + "merged-vids": [], + "portal-id": 8727216, + "is-contact": true, + "properties": { + "firstname": { + "value": "Reginald" + }, + "lastmodifieddate": { + "value": "1713702886064" + }, + "lastname": { + "value": "Kelly" + } + }, + "form-submissions": [ + { + "conversion-id": "7b08b0ac-23a5-4961-b09b-20da2a4a2c76", + "timestamp": 1714546800000, + "form-id": "8372afd9-33b9-458e-ac20-58665140554b", + "portal-id": 8727216, + "title": "When Will We Land? - Barry Can't Swim", + "form-type": "HUBSPOT", + "contact-associated-by": ["EMAIL"], + "meta-data": [] + }, + { + "conversion-id": "55891af1-faa6-44de-ad51-b012c6f7a892", + "timestamp": 1714546800000, + "form-id": "15905b83-4431-414c-8a0c-4bfd1e819290", + "portal-id": 8727216, + "title": "Favorite bending element?", + "form-type": "HUBSPOT", + "contact-associated-by": ["EMAIL"], + "meta-data": [] + }, + { + "conversion-id": "e81140ad-389f-4a42-83a6-bb761ba81d34", + "timestamp": 1714546800000, + "form-id": "709f8e27-21be-4d8f-9774-29311a069780", + "portal-id": 8727216, + "title": "Favorite Bakeries in San Francisco", + "form-type": "HUBSPOT", + "contact-associated-by": ["EMAIL"], + "meta-data": [] + }, + { + "conversion-id": "154a7eff-82a3-4737-a59c-85a5c96669bd", + "timestamp": 1714546800000, + "form-id": "bf46b791-448d-483c-bf1c-e5099c43cbff", + "portal-id": 8727216, + "title": "Brian's Test Form", + "form-type": "HUBSPOT", + "contact-associated-by": ["EMAIL"], + "meta-data": [] + } + ], + "list-memberships": [ + { + "static-list-id": 166, + "internal-list-id": 2147483643, + "timestamp": 1714546800000, + "vid": 5331890527, + "is-member": true + } + ], + "identity-profiles": [ + { + "vid": 5331890527, + "saved-at-timestamp": 1711049660804, + "deleted-changed-timestamp": 0, + "identities": [ + { + "type": "EMAIL", + "value": "reginald.kelly@airbyte-x.com", + "timestamp": 1711049660692, + "is-primary": true + }, + { + "type": "LEAD_GUID", + "value": "5b218fdc-f72f-4694-82e1-fc7af525c0a5", + "timestamp": 1711049660801 + } + ] + } + ], + "merge-audits": [], + "addedAt": 1711049660804 + } + ] +} diff --git a/airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/contacts_list_memberships.json b/airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/contacts_list_memberships.json new file mode 100644 index 000000000000..60503389ba45 --- /dev/null +++ b/airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/contacts_list_memberships.json @@ -0,0 +1,154 @@ +{ + "contacts": [ + { + "vid": 5331889818, + "canonical-vid": 5331889818, + "merged-vids": [], + "portal-id": 8727216, + "is-contact": true, + "properties": { + "firstname": { + "value": "Eric" + }, + "lastmodifieddate": { + "value": "1713702782727" + }, + "lastname": { + "value": "Harris" + } + }, + "form-submissions": [ + { + "conversion-id": "2c7b8485-62e8-41e3-95a1-ac429358201c", + "timestamp": 1714546800000, + "form-id": "8372afd9-33b9-458e-ac20-58665140554b", + "portal-id": 8727216, + "title": "When Will We Land? - Barry Can't Swim", + "form-type": "HUBSPOT", + "contact-associated-by": ["EMAIL"], + "meta-data": [] + } + ], + "list-memberships": [ + { + "static-list-id": 166, + "internal-list-id": 2147483643, + "timestamp": 1714546800000, + "vid": 5331890527, + "is-member": true + }, + { + "static-list-id": 167, + "internal-list-id": 2147483644, + "timestamp": 1714546800000, + "vid": 5331890528, + "is-member": true + }, + { + "static-list-id": 167, + "internal-list-id": 2147483645, + "timestamp": 1714546800000, + "vid": 5331890529, + "is-member": true + } + ], + "identity-profiles": [ + { + "vid": 5331889818, + "saved-at-timestamp": 1711049462684, + "deleted-changed-timestamp": 0, + "identities": [ + { + "type": "EMAIL", + "value": "eric.harris@airbyte-x.com", + "timestamp": 1711049462548, + "is-primary": true + }, + { + "type": "LEAD_GUID", + "value": "33671d61-e5fe-43c0-84ff-2ad08329b008", + "timestamp": 1711049462680 + } + ] + } + ], + "merge-audits": [], + "addedAt": 1711049462684 + }, + { + "vid": 5331890527, + "canonical-vid": 5331890527, + "merged-vids": [], + "portal-id": 8727216, + "is-contact": true, + "properties": { + "firstname": { + "value": "Reginald" + }, + "lastmodifieddate": { + "value": "1713702886064" + }, + "lastname": { + "value": "Kelly" + } + }, + "form-submissions": [ + { + "conversion-id": "7b08b0ac-23a5-4961-b09b-20da2a4a2c76", + "timestamp": 1714546800000, + "form-id": "8372afd9-33b9-458e-ac20-58665140554b", + "portal-id": 8727216, + "title": "When Will We Land? - Barry Can't Swim", + "form-type": "HUBSPOT", + "contact-associated-by": ["EMAIL"], + "meta-data": [] + } + ], + "list-memberships": [ + { + "static-list-id": 166, + "internal-list-id": 2147483643, + "timestamp": 1714546800000, + "vid": 5331890527, + "is-member": true + }, + { + "static-list-id": 167, + "internal-list-id": 2147483644, + "timestamp": 1714546800000, + "vid": 5331890528, + "is-member": true + }, + { + "static-list-id": 167, + "internal-list-id": 2147483645, + "timestamp": 1714546800000, + "vid": 5331890529, + "is-member": true + } + ], + "identity-profiles": [ + { + "vid": 5331890527, + "saved-at-timestamp": 1711049660804, + "deleted-changed-timestamp": 0, + "identities": [ + { + "type": "EMAIL", + "value": "reginald.kelly@airbyte-x.com", + "timestamp": 1711049660692, + "is-primary": true + }, + { + "type": "LEAD_GUID", + "value": "5b218fdc-f72f-4694-82e1-fc7af525c0a5", + "timestamp": 1711049660801 + } + ] + } + ], + "merge-audits": [], + "addedAt": 1711049660804 + } + ] +} diff --git a/airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/contacts_merged_audit.json b/airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/contacts_merged_audit.json new file mode 100644 index 000000000000..53e636097a11 --- /dev/null +++ b/airbyte-integrations/connectors/source-hubspot/unit_tests/resource/http/response/contacts_merged_audit.json @@ -0,0 +1,240 @@ +{ + "contacts": [ + { + "vid": 5331889818, + "canonical-vid": 5331889818, + "merged-vids": [], + "portal-id": 8727216, + "is-contact": true, + "properties": { + "firstname": { + "value": "Eric" + }, + "lastmodifieddate": { + "value": "1713702782727" + }, + "lastname": { + "value": "Harris" + } + }, + "form-submissions": [ + { + "conversion-id": "2c7b8485-62e8-41e3-95a1-ac429358201c", + "timestamp": 1714546800000, + "form-id": "8372afd9-33b9-458e-ac20-58665140554b", + "portal-id": 8727216, + "title": "When Will We Land? - Barry Can't Swim", + "form-type": "HUBSPOT", + "contact-associated-by": ["EMAIL"], + "meta-data": [] + } + ], + "list-memberships": [ + { + "static-list-id": 166, + "internal-list-id": 2147483643, + "timestamp": 1714546800000, + "vid": 5331890527, + "is-member": true + } + ], + "identity-profiles": [ + { + "vid": 5331889818, + "saved-at-timestamp": 1711049462684, + "deleted-changed-timestamp": 0, + "identities": [ + { + "type": "EMAIL", + "value": "eric.harris@airbyte-x.com", + "timestamp": 1711049462548, + "is-primary": true + }, + { + "type": "LEAD_GUID", + "value": "33671d61-e5fe-43c0-84ff-2ad08329b008", + "timestamp": 1711049462680 + } + ] + } + ], + "merge-audits": [ + { + "canonical-vid": 651, + "vid-to-merge": 201, + "timestamp": 1714546800000, + "entity-id": "auth:app-cookie | auth-level:app | login-id:integration-test@airbyte.io-1688758203663 | hub-id:8727216 | user-id:12282590 | origin-ip:2804:1b3:8402:b1f4:7d1b:f62e:b071:593d | correlation-id:3f139cd7-66fc-4300-8cbc-e6c1fe9ea7d1", + "user-id": 12282590, + "num-properties-moved": 45, + "merged_from_email": { + "value": "testingapis@hubspot.com", + "source-type": "API", + "source-id": null, + "source-label": null, + "updated-by-user-id": null, + "timestamp": 1714546800000, + "selected": false + }, + "merged_to_email": { + "value": "testingapicontact_1@hubspot.com", + "source-type": "API", + "source-id": null, + "source-label": null, + "updated-by-user-id": null, + "timestamp": 1714546800000, + "selected": false + }, + "first-name": "test", + "last-name": "testerson" + }, + { + "canonical-vid": 651, + "vid-to-merge": 201, + "timestamp": 1714546800000, + "entity-id": "auth:app-cookie | auth-level:app | login-id:integration-test@airbyte.io-1688758203663 | hub-id:8727216 | user-id:12282590 | origin-ip:2804:1b3:8402:b1f4:7d1b:f62e:b071:593d | correlation-id:3f139cd7-66fc-4300-8cbc-e6c1fe9ea7d1", + "user-id": 12282590, + "num-properties-moved": 45, + "merged_from_email": { + "value": "testingapis@hubspot.com", + "source-type": "API", + "source-id": null, + "source-label": null, + "updated-by-user-id": null, + "timestamp": 1714546800000, + "selected": false + }, + "merged_to_email": { + "value": "testingapicontact_1@hubspot.com", + "source-type": "API", + "source-id": null, + "source-label": null, + "updated-by-user-id": null, + "timestamp": 1714546800000, + "selected": false + }, + "first-name": "test", + "last-name": "testerson" + } + ], + "addedAt": 1711049462684 + }, + { + "vid": 5331890527, + "canonical-vid": 5331890527, + "merged-vids": [], + "portal-id": 8727216, + "is-contact": true, + "properties": { + "firstname": { + "value": "Reginald" + }, + "lastmodifieddate": { + "value": "1713702886064" + }, + "lastname": { + "value": "Kelly" + } + }, + "form-submissions": [ + { + "conversion-id": "7b08b0ac-23a5-4961-b09b-20da2a4a2c76", + "timestamp": 1714546800000, + "form-id": "8372afd9-33b9-458e-ac20-58665140554b", + "portal-id": 8727216, + "title": "When Will We Land? - Barry Can't Swim", + "form-type": "HUBSPOT", + "contact-associated-by": ["EMAIL"], + "meta-data": [] + } + ], + "list-memberships": [ + { + "static-list-id": 166, + "internal-list-id": 2147483643, + "timestamp": 1714546800000, + "vid": 5331890527, + "is-member": true + } + ], + "identity-profiles": [ + { + "vid": 5331890527, + "saved-at-timestamp": 1711049660804, + "deleted-changed-timestamp": 0, + "identities": [ + { + "type": "EMAIL", + "value": "reginald.kelly@airbyte-x.com", + "timestamp": 1711049660692, + "is-primary": true + }, + { + "type": "LEAD_GUID", + "value": "5b218fdc-f72f-4694-82e1-fc7af525c0a5", + "timestamp": 1711049660801 + } + ] + } + ], + "merge-audits": [ + { + "canonical-vid": 651, + "vid-to-merge": 201, + "timestamp": 1714546800000, + "entity-id": "auth:app-cookie | auth-level:app | login-id:integration-test@airbyte.io-1688758203663 | hub-id:8727216 | user-id:12282590 | origin-ip:2804:1b3:8402:b1f4:7d1b:f62e:b071:593d | correlation-id:3f139cd7-66fc-4300-8cbc-e6c1fe9ea7d1", + "user-id": 12282590, + "num-properties-moved": 45, + "merged_from_email": { + "value": "testingapis@hubspot.com", + "source-type": "API", + "source-id": null, + "source-label": null, + "updated-by-user-id": null, + "timestamp": 1714546800000, + "selected": false + }, + "merged_to_email": { + "value": "testingapicontact_1@hubspot.com", + "source-type": "API", + "source-id": null, + "source-label": null, + "updated-by-user-id": null, + "timestamp": 1714546800000, + "selected": false + }, + "first-name": "test", + "last-name": "testerson" + }, + { + "canonical-vid": 651, + "vid-to-merge": 201, + "timestamp": 1714546800000, + "entity-id": "auth:app-cookie | auth-level:app | login-id:integration-test@airbyte.io-1688758203663 | hub-id:8727216 | user-id:12282590 | origin-ip:2804:1b3:8402:b1f4:7d1b:f62e:b071:593d | correlation-id:3f139cd7-66fc-4300-8cbc-e6c1fe9ea7d1", + "user-id": 12282590, + "num-properties-moved": 45, + "merged_from_email": { + "value": "testingapis@hubspot.com", + "source-type": "API", + "source-id": null, + "source-label": null, + "updated-by-user-id": null, + "timestamp": 1714546800000, + "selected": false + }, + "merged_to_email": { + "value": "testingapicontact_1@hubspot.com", + "source-type": "API", + "source-id": null, + "source-label": null, + "updated-by-user-id": null, + "timestamp": 1714546800000, + "selected": false + }, + "first-name": "test", + "last-name": "testerson" + } + ], + "addedAt": 1711049660804 + } + ] +} diff --git a/airbyte-integrations/connectors/source-hubspot/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-hubspot/unit_tests/test_streams.py index 27ddfa226e16..0c4572b02a2e 100644 --- a/airbyte-integrations/connectors/source-hubspot/unit_tests/test_streams.py +++ b/airbyte-integrations/connectors/source-hubspot/unit_tests/test_streams.py @@ -129,6 +129,7 @@ def test_streams_read(stream, endpoint, cursor_value, requests_mock, common_para } } ] + properties_response = [ { "json": [ @@ -138,7 +139,8 @@ def test_streams_read(stream, endpoint, cursor_value, requests_mock, common_para "status_code": 200, } ] - contact_reponse = [ + + contact_response = [ { "json": { stream.data_field: [ @@ -147,6 +149,7 @@ def test_streams_read(stream, endpoint, cursor_value, requests_mock, common_para } } ] + read_batch_contact_v1_response = [ { "json": { @@ -155,13 +158,26 @@ def test_streams_read(stream, endpoint, cursor_value, requests_mock, common_para "status_code": 200, } ] + + contact_lists_v1_response = [ + { + "json": { + "contacts": [ + {"vid": "test_id", "merge-audits": [{"canonical-vid": 2, "vid-to-merge": 5608, "timestamp": 1653322839932}]} + ] + }, + "status_code": 200, + } + ] + is_form_submission = isinstance(stream, FormSubmissions) stream._sync_mode = SyncMode.full_refresh stream_url = stream.url + "/test_id" if is_form_submission else stream.url stream._sync_mode = None requests_mock.register_uri("GET", stream_url, responses) - requests_mock.register_uri("GET", "/crm/v3/objects/contact", contact_reponse) + requests_mock.register_uri("GET", "/crm/v3/objects/contact", contact_response) + requests_mock.register_uri("GET", "/contacts/v1/lists/all/contacts/all", contact_lists_v1_response) requests_mock.register_uri("GET", "/marketing/v3/forms", responses) requests_mock.register_uri("GET", "/email/public/v1/campaigns/test_id", responses) requests_mock.register_uri("GET", f"/properties/v2/{endpoint}/properties", properties_response)