From 73bba869cba6eefd3b7560e4282673e6691b2883 Mon Sep 17 00:00:00 2001 From: Stephen Kent Date: Thu, 13 Apr 2023 21:54:13 -0700 Subject: [PATCH 1/6] poetry add selenium webdriver-manager undetected-chromedriver --- poetry.lock | 398 ++++++++++++++++++++++++++++++++++++++++++++++++- pyproject.toml | 3 + 2 files changed, 397 insertions(+), 4 deletions(-) diff --git a/poetry.lock b/poetry.lock index 3ae3321..884a277 100644 --- a/poetry.lock +++ b/poetry.lock @@ -27,11 +27,23 @@ files = [ {file = "astor-0.8.1.tar.gz", hash = "sha256:6a6effda93f4e1ce9f618779b2dd1d9d84f1e32812c23a29b3fff6fd7f63fa5e"}, ] +[[package]] +name = "async-generator" +version = "1.10" +description = "Async generators and context managers for Python 3.5+" +category = "main" +optional = false +python-versions = ">=3.5" +files = [ + {file = "async_generator-1.10-py3-none-any.whl", hash = "sha256:01c7bf666359b4967d2cda0000cc2e4af16a0ae098cbffcb8472fb9e8ad6585b"}, + {file = "async_generator-1.10.tar.gz", hash = "sha256:6ebb3d106c12920aaae42ccb6f787ef5eefdcdd166ea3d628fa8476abe712144"}, +] + [[package]] name = "attrs" version = "22.2.0" description = "Classes Without Boilerplate" -category = "dev" +category = "main" optional = false python-versions = ">=3.6" files = [ @@ -147,6 +159,83 @@ files = [ {file = "certifi-2022.12.7.tar.gz", hash = "sha256:35824b4c3a97115964b408844d64aa14db1cc518f6562e8d7261699d1350a9e3"}, ] +[[package]] +name = "cffi" +version = "1.15.1" +description = "Foreign Function Interface for Python calling C code." +category = "main" +optional = false +python-versions = "*" +files = [ + {file = "cffi-1.15.1-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:a66d3508133af6e8548451b25058d5812812ec3798c886bf38ed24a98216fab2"}, + {file = "cffi-1.15.1-cp27-cp27m-manylinux1_i686.whl", hash = "sha256:470c103ae716238bbe698d67ad020e1db9d9dba34fa5a899b5e21577e6d52ed2"}, + {file = "cffi-1.15.1-cp27-cp27m-manylinux1_x86_64.whl", hash = "sha256:9ad5db27f9cabae298d151c85cf2bad1d359a1b9c686a275df03385758e2f914"}, + {file = "cffi-1.15.1-cp27-cp27m-win32.whl", hash = "sha256:b3bbeb01c2b273cca1e1e0c5df57f12dce9a4dd331b4fa1635b8bec26350bde3"}, + {file = "cffi-1.15.1-cp27-cp27m-win_amd64.whl", hash = "sha256:e00b098126fd45523dd056d2efba6c5a63b71ffe9f2bbe1a4fe1716e1d0c331e"}, + {file = "cffi-1.15.1-cp27-cp27mu-manylinux1_i686.whl", hash = "sha256:d61f4695e6c866a23a21acab0509af1cdfd2c013cf256bbf5b6b5e2695827162"}, + {file = "cffi-1.15.1-cp27-cp27mu-manylinux1_x86_64.whl", hash = "sha256:ed9cb427ba5504c1dc15ede7d516b84757c3e3d7868ccc85121d9310d27eed0b"}, + {file = "cffi-1.15.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:39d39875251ca8f612b6f33e6b1195af86d1b3e60086068be9cc053aa4376e21"}, + {file = "cffi-1.15.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:285d29981935eb726a4399badae8f0ffdff4f5050eaa6d0cfc3f64b857b77185"}, + {file = "cffi-1.15.1-cp310-cp310-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3eb6971dcff08619f8d91607cfc726518b6fa2a9eba42856be181c6d0d9515fd"}, + {file = "cffi-1.15.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:21157295583fe8943475029ed5abdcf71eb3911894724e360acff1d61c1d54bc"}, + {file = "cffi-1.15.1-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5635bd9cb9731e6d4a1132a498dd34f764034a8ce60cef4f5319c0541159392f"}, + {file = "cffi-1.15.1-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2012c72d854c2d03e45d06ae57f40d78e5770d252f195b93f581acf3ba44496e"}, + {file = "cffi-1.15.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dd86c085fae2efd48ac91dd7ccffcfc0571387fe1193d33b6394db7ef31fe2a4"}, + {file = "cffi-1.15.1-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:fa6693661a4c91757f4412306191b6dc88c1703f780c8234035eac011922bc01"}, + {file = "cffi-1.15.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:59c0b02d0a6c384d453fece7566d1c7e6b7bae4fc5874ef2ef46d56776d61c9e"}, + {file = "cffi-1.15.1-cp310-cp310-win32.whl", hash = "sha256:cba9d6b9a7d64d4bd46167096fc9d2f835e25d7e4c121fb2ddfc6528fb0413b2"}, + {file = "cffi-1.15.1-cp310-cp310-win_amd64.whl", hash = "sha256:ce4bcc037df4fc5e3d184794f27bdaab018943698f4ca31630bc7f84a7b69c6d"}, + {file = "cffi-1.15.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:3d08afd128ddaa624a48cf2b859afef385b720bb4b43df214f85616922e6a5ac"}, + {file = "cffi-1.15.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:3799aecf2e17cf585d977b780ce79ff0dc9b78d799fc694221ce814c2c19db83"}, + {file = "cffi-1.15.1-cp311-cp311-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a591fe9e525846e4d154205572a029f653ada1a78b93697f3b5a8f1f2bc055b9"}, + {file = "cffi-1.15.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3548db281cd7d2561c9ad9984681c95f7b0e38881201e157833a2342c30d5e8c"}, + {file = "cffi-1.15.1-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:91fc98adde3d7881af9b59ed0294046f3806221863722ba7d8d120c575314325"}, + {file = "cffi-1.15.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:94411f22c3985acaec6f83c6df553f2dbe17b698cc7f8ae751ff2237d96b9e3c"}, + {file = "cffi-1.15.1-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:03425bdae262c76aad70202debd780501fabeaca237cdfddc008987c0e0f59ef"}, + {file = "cffi-1.15.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:cc4d65aeeaa04136a12677d3dd0b1c0c94dc43abac5860ab33cceb42b801c1e8"}, + {file = "cffi-1.15.1-cp311-cp311-win32.whl", hash = "sha256:a0f100c8912c114ff53e1202d0078b425bee3649ae34d7b070e9697f93c5d52d"}, + {file = "cffi-1.15.1-cp311-cp311-win_amd64.whl", hash = "sha256:04ed324bda3cda42b9b695d51bb7d54b680b9719cfab04227cdd1e04e5de3104"}, + {file = "cffi-1.15.1-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:50a74364d85fd319352182ef59c5c790484a336f6db772c1a9231f1c3ed0cbd7"}, + {file = "cffi-1.15.1-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e263d77ee3dd201c3a142934a086a4450861778baaeeb45db4591ef65550b0a6"}, + {file = "cffi-1.15.1-cp36-cp36m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:cec7d9412a9102bdc577382c3929b337320c4c4c4849f2c5cdd14d7368c5562d"}, + {file = "cffi-1.15.1-cp36-cp36m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:4289fc34b2f5316fbb762d75362931e351941fa95fa18789191b33fc4cf9504a"}, + {file = "cffi-1.15.1-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:173379135477dc8cac4bc58f45db08ab45d228b3363adb7af79436135d028405"}, + {file = "cffi-1.15.1-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:6975a3fac6bc83c4a65c9f9fcab9e47019a11d3d2cf7f3c0d03431bf145a941e"}, + {file = "cffi-1.15.1-cp36-cp36m-win32.whl", hash = "sha256:2470043b93ff09bf8fb1d46d1cb756ce6132c54826661a32d4e4d132e1977adf"}, + {file = "cffi-1.15.1-cp36-cp36m-win_amd64.whl", hash = "sha256:30d78fbc8ebf9c92c9b7823ee18eb92f2e6ef79b45ac84db507f52fbe3ec4497"}, + {file = "cffi-1.15.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:198caafb44239b60e252492445da556afafc7d1e3ab7a1fb3f0584ef6d742375"}, + {file = "cffi-1.15.1-cp37-cp37m-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5ef34d190326c3b1f822a5b7a45f6c4535e2f47ed06fec77d3d799c450b2651e"}, + {file = "cffi-1.15.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8102eaf27e1e448db915d08afa8b41d6c7ca7a04b7d73af6514df10a3e74bd82"}, + {file = "cffi-1.15.1-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5df2768244d19ab7f60546d0c7c63ce1581f7af8b5de3eb3004b9b6fc8a9f84b"}, + {file = "cffi-1.15.1-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:a8c4917bd7ad33e8eb21e9a5bbba979b49d9a97acb3a803092cbc1133e20343c"}, + {file = "cffi-1.15.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0e2642fe3142e4cc4af0799748233ad6da94c62a8bec3a6648bf8ee68b1c7426"}, + {file = "cffi-1.15.1-cp37-cp37m-win32.whl", hash = "sha256:e229a521186c75c8ad9490854fd8bbdd9a0c9aa3a524326b55be83b54d4e0ad9"}, + {file = "cffi-1.15.1-cp37-cp37m-win_amd64.whl", hash = "sha256:a0b71b1b8fbf2b96e41c4d990244165e2c9be83d54962a9a1d118fd8657d2045"}, + {file = "cffi-1.15.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:320dab6e7cb2eacdf0e658569d2575c4dad258c0fcc794f46215e1e39f90f2c3"}, + {file = "cffi-1.15.1-cp38-cp38-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1e74c6b51a9ed6589199c787bf5f9875612ca4a8a0785fb2d4a84429badaf22a"}, + {file = "cffi-1.15.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a5c84c68147988265e60416b57fc83425a78058853509c1b0629c180094904a5"}, + {file = "cffi-1.15.1-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3b926aa83d1edb5aa5b427b4053dc420ec295a08e40911296b9eb1b6170f6cca"}, + {file = "cffi-1.15.1-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:87c450779d0914f2861b8526e035c5e6da0a3199d8f1add1a665e1cbc6fc6d02"}, + {file = "cffi-1.15.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4f2c9f67e9821cad2e5f480bc8d83b8742896f1242dba247911072d4fa94c192"}, + {file = "cffi-1.15.1-cp38-cp38-win32.whl", hash = "sha256:8b7ee99e510d7b66cdb6c593f21c043c248537a32e0bedf02e01e9553a172314"}, + {file = "cffi-1.15.1-cp38-cp38-win_amd64.whl", hash = "sha256:00a9ed42e88df81ffae7a8ab6d9356b371399b91dbdf0c3cb1e84c03a13aceb5"}, + {file = "cffi-1.15.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:54a2db7b78338edd780e7ef7f9f6c442500fb0d41a5a4ea24fff1c929d5af585"}, + {file = "cffi-1.15.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:fcd131dd944808b5bdb38e6f5b53013c5aa4f334c5cad0c72742f6eba4b73db0"}, + {file = "cffi-1.15.1-cp39-cp39-manylinux_2_12_i686.manylinux2010_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7473e861101c9e72452f9bf8acb984947aa1661a7704553a9f6e4baa5ba64415"}, + {file = "cffi-1.15.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6c9a799e985904922a4d207a94eae35c78ebae90e128f0c4e521ce339396be9d"}, + {file = "cffi-1.15.1-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3bcde07039e586f91b45c88f8583ea7cf7a0770df3a1649627bf598332cb6984"}, + {file = "cffi-1.15.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:33ab79603146aace82c2427da5ca6e58f2b3f2fb5da893ceac0c42218a40be35"}, + {file = "cffi-1.15.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5d598b938678ebf3c67377cdd45e09d431369c3b1a5b331058c338e201f12b27"}, + {file = "cffi-1.15.1-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:db0fbb9c62743ce59a9ff687eb5f4afbe77e5e8403d6697f7446e5f609976f76"}, + {file = "cffi-1.15.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:98d85c6a2bef81588d9227dde12db8a7f47f639f4a17c9ae08e773aa9c697bf3"}, + {file = "cffi-1.15.1-cp39-cp39-win32.whl", hash = "sha256:40f4774f5a9d4f5e344f31a32b5096977b5d48560c5592e2f3d2c4374bd543ee"}, + {file = "cffi-1.15.1-cp39-cp39-win_amd64.whl", hash = "sha256:70df4e3b545a17496c9b3f41f5115e69a4f2e77e94e1d2a8e1070bc0c38c8a3c"}, + {file = "cffi-1.15.1.tar.gz", hash = "sha256:d400bfb9a37b1351253cb402671cea7e89bdecc294e8016a707f6d1d8ac934f9"}, +] + +[package.dependencies] +pycparser = "*" + [[package]] name = "cfgv" version = "3.3.1" @@ -288,7 +377,7 @@ colorama = {version = "*", markers = "platform_system == \"Windows\""} name = "colorama" version = "0.4.6" description = "Cross-platform colored terminal text." -category = "dev" +category = "main" optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" files = [ @@ -441,7 +530,7 @@ files = [ name = "exceptiongroup" version = "1.1.0" description = "Backport of PEP 654 (exception groups)" -category = "dev" +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -568,6 +657,18 @@ files = [ [package.dependencies] gitdb = ">=4.0.1,<5" +[[package]] +name = "h11" +version = "0.14.0" +description = "A pure-Python, bring-your-own-I/O implementation of HTTP/1.1" +category = "main" +optional = false +python-versions = ">=3.7" +files = [ + {file = "h11-0.14.0-py3-none-any.whl", hash = "sha256:e3fe4ac4b851c468cc8363d500db52c2ead036020723024a109d37346efaa761"}, + {file = "h11-0.14.0.tar.gz", hash = "sha256:8f19fbbe99e72420ff35c00b27a34cb9937e902a8b810e2c88300c6f0a3b699d"}, +] + [[package]] name = "identify" version = "2.5.18" @@ -841,6 +942,21 @@ files = [ [package.dependencies] setuptools = "*" +[[package]] +name = "outcome" +version = "1.2.0" +description = "Capture the outcome of Python function calls." +category = "main" +optional = false +python-versions = ">=3.7" +files = [ + {file = "outcome-1.2.0-py2.py3-none-any.whl", hash = "sha256:c4ab89a56575d6d38a05aa16daeaa333109c1f96167aba8901ab18b6b5e0f7f5"}, + {file = "outcome-1.2.0.tar.gz", hash = "sha256:6f82bd3de45da303cf1f771ecafa1633750a358436a8bb60e06a1ceb745d2672"}, +] + +[package.dependencies] +attrs = ">=19.2.0" + [[package]] name = "packaging" version = "23.0" @@ -986,6 +1102,18 @@ files = [ {file = "pycodestyle-2.9.1.tar.gz", hash = "sha256:2c9607871d58c76354b697b42f5d57e1ada7d261c261efac224b664affdc5785"}, ] +[[package]] +name = "pycparser" +version = "2.21" +description = "C parser in Python" +category = "main" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +files = [ + {file = "pycparser-2.21-py2.py3-none-any.whl", hash = "sha256:8ee45429555515e1f6b185e78100aea234072576aa43ab53aefcae078162fca9"}, + {file = "pycparser-2.21.tar.gz", hash = "sha256:e644fdec12f7872f86c58ff790da456218b10f863970249516d60a5eaca77206"}, +] + [[package]] name = "pyflakes" version = "2.5.0" @@ -998,6 +1126,19 @@ files = [ {file = "pyflakes-2.5.0.tar.gz", hash = "sha256:491feb020dca48ccc562a8c0cbe8df07ee13078df59813b83959cbdada312ea3"}, ] +[[package]] +name = "pysocks" +version = "1.7.1" +description = "A Python SOCKS client module. See https://github.com/Anorov/PySocks for more information." +category = "main" +optional = false +python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" +files = [ + {file = "PySocks-1.7.1-py27-none-any.whl", hash = "sha256:08e69f092cc6dbe92a0fdd16eeb9b9ffbc13cadfe5ca4c7bd92ffb078b293299"}, + {file = "PySocks-1.7.1-py3-none-any.whl", hash = "sha256:2725bd0a9925919b9b51739eea5f9e2bae91e83288108a9ad338b2e3a4435ee5"}, + {file = "PySocks-1.7.1.tar.gz", hash = "sha256:3f8804571ebe159c380ac6de37643bb4685970655d3bba243530d6558b799aa0"}, +] + [[package]] name = "pytest" version = "7.2.1" @@ -1106,6 +1247,21 @@ files = [ [package.dependencies] six = ">=1.5" +[[package]] +name = "python-dotenv" +version = "1.0.0" +description = "Read key-value pairs from a .env file and set them as environment variables" +category = "main" +optional = false +python-versions = ">=3.8" +files = [ + {file = "python-dotenv-1.0.0.tar.gz", hash = "sha256:a8df96034aae6d2d50a4ebe8216326c61c3eb64836776504fcca410e5937a3ba"}, + {file = "python_dotenv-1.0.0-py3-none-any.whl", hash = "sha256:f5971a9226b701070a4bf2c38c89e5a3f0d64de8debda981d1db98583009122a"}, +] + +[package.extras] +cli = ["click (>=5.0)"] + [[package]] name = "python-slugify" version = "8.0.0" @@ -1217,6 +1373,24 @@ urllib3 = ">=1.25.10" [package.extras] tests = ["coverage (>=6.0.0)", "flake8", "mypy", "pytest (>=7.0.0)", "pytest-asyncio", "pytest-cov", "pytest-httpserver", "types-requests"] +[[package]] +name = "selenium" +version = "4.10.0" +description = "" +category = "main" +optional = false +python-versions = ">=3.7" +files = [ + {file = "selenium-4.10.0-py3-none-any.whl", hash = "sha256:40241b9d872f58959e9b34e258488bf11844cd86142fd68182bd41db9991fc5c"}, + {file = "selenium-4.10.0.tar.gz", hash = "sha256:871bf800c4934f745b909c8dfc7d15c65cf45bd2e943abd54451c810ada395e3"}, +] + +[package.dependencies] +certifi = ">=2021.10.8" +trio = ">=0.17,<1.0" +trio-websocket = ">=0.9,<1.0" +urllib3 = {version = ">=1.26,<3", extras = ["socks"]} + [[package]] name = "setuptools" version = "67.4.0" @@ -1258,6 +1432,30 @@ files = [ {file = "smmap-5.0.0.tar.gz", hash = "sha256:c840e62059cd3be204b0c9c9f74be2c09d5648eddd4580d9314c3ecde0b30936"}, ] +[[package]] +name = "sniffio" +version = "1.3.0" +description = "Sniff out which async library your code is running under" +category = "main" +optional = false +python-versions = ">=3.7" +files = [ + {file = "sniffio-1.3.0-py3-none-any.whl", hash = "sha256:eecefdce1e5bbfb7ad2eeaabf7c1eeb404d7757c379bd1f7e5cce9d8bf425384"}, + {file = "sniffio-1.3.0.tar.gz", hash = "sha256:e60305c5e5d314f5389259b7f22aaa33d8f7dee49763119234af3755c55b9101"}, +] + +[[package]] +name = "sortedcontainers" +version = "2.4.0" +description = "Sorted Containers -- Sorted List, Sorted Dict, Sorted Set" +category = "main" +optional = false +python-versions = "*" +files = [ + {file = "sortedcontainers-2.4.0-py2.py3-none-any.whl", hash = "sha256:a163dcaede0f1c021485e957a39245190e74249897e2ae4b2aa38595db237ee0"}, + {file = "sortedcontainers-2.4.0.tar.gz", hash = "sha256:25caa5a06cc30b6b83d11423433f65d1f9d76c4c6a0c90e3379eaa43b9bfdb88"}, +] + [[package]] name = "stevedore" version = "5.0.0" @@ -1324,6 +1522,66 @@ files = [ {file = "tomli-2.0.1.tar.gz", hash = "sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f"}, ] +[[package]] +name = "tqdm" +version = "4.65.0" +description = "Fast, Extensible Progress Meter" +category = "main" +optional = false +python-versions = ">=3.7" +files = [ + {file = "tqdm-4.65.0-py3-none-any.whl", hash = "sha256:c4f53a17fe37e132815abceec022631be8ffe1b9381c2e6e30aa70edc99e9671"}, + {file = "tqdm-4.65.0.tar.gz", hash = "sha256:1871fb68a86b8fb3b59ca4cdd3dcccbc7e6d613eeed31f4c332531977b89beb5"}, +] + +[package.dependencies] +colorama = {version = "*", markers = "platform_system == \"Windows\""} + +[package.extras] +dev = ["py-make (>=0.1.0)", "twine", "wheel"] +notebook = ["ipywidgets (>=6)"] +slack = ["slack-sdk"] +telegram = ["requests"] + +[[package]] +name = "trio" +version = "0.22.0" +description = "A friendly Python library for async concurrency and I/O" +category = "main" +optional = false +python-versions = ">=3.7" +files = [ + {file = "trio-0.22.0-py3-none-any.whl", hash = "sha256:f1dd0780a89bfc880c7c7994519cb53f62aacb2c25ff487001c0052bd721cdf0"}, + {file = "trio-0.22.0.tar.gz", hash = "sha256:ce68f1c5400a47b137c5a4de72c7c901bd4e7a24fbdebfe9b41de8c6c04eaacf"}, +] + +[package.dependencies] +async-generator = ">=1.9" +attrs = ">=19.2.0" +cffi = {version = ">=1.14", markers = "os_name == \"nt\" and implementation_name != \"pypy\""} +exceptiongroup = {version = ">=1.0.0rc9", markers = "python_version < \"3.11\""} +idna = "*" +outcome = "*" +sniffio = "*" +sortedcontainers = "*" + +[[package]] +name = "trio-websocket" +version = "0.10.2" +description = "WebSocket library for Trio" +category = "main" +optional = false +python-versions = ">=3.7" +files = [ + {file = "trio-websocket-0.10.2.tar.gz", hash = "sha256:af13e9393f9051111300287947ec595d601758ce3d165328e7d36325135a8d62"}, + {file = "trio_websocket-0.10.2-py3-none-any.whl", hash = "sha256:0908435e4eecc49d830ae1c4d6c47b978a75f00594a2be2104d58b61a04cdb53"}, +] + +[package.dependencies] +exceptiongroup = "*" +trio = ">=0.11" +wsproto = ">=0.14" + [[package]] name = "typer" version = "0.7.0" @@ -1412,6 +1670,22 @@ files = [ mypy-extensions = ">=0.3.0" typing-extensions = ">=3.7.4" +[[package]] +name = "undetected-chromedriver" +version = "3.5.0" +description = "('Selenium.webdriver.Chrome replacement with compatiblity for Brave, and other Chromium based browsers.', 'Not triggered by CloudFlare/Imperva/hCaptcha and such.', 'NOTE: results may vary due to many factors. No guarantees are given, except for ongoing efforts in understanding detection algorithms.')" +category = "main" +optional = false +python-versions = "*" +files = [ + {file = "undetected-chromedriver-3.5.0.tar.gz", hash = "sha256:355bfc5bc78800795a9a49ef2a1199402d461483f6c0efd7bd7a85809464df6c"}, +] + +[package.dependencies] +requests = "*" +selenium = ">=4.9.0" +websockets = "*" + [[package]] name = "urllib3" version = "1.26.14" @@ -1424,6 +1698,9 @@ files = [ {file = "urllib3-1.26.14.tar.gz", hash = "sha256:076907bf8fd355cde77728471316625a4d2f7e713c125f51953bb5b3eecf4f72"}, ] +[package.dependencies] +PySocks = {version = ">=1.5.6,<1.5.7 || >1.5.7,<2.0", optional = true, markers = "extra == \"socks\""} + [package.extras] brotli = ["brotli (>=1.0.9)", "brotlicffi (>=0.8.0)", "brotlipy (>=0.6.0)"] secure = ["certifi", "cryptography (>=1.3.4)", "idna (>=2.0.0)", "ipaddress", "pyOpenSSL (>=0.14)", "urllib3-secure-extra"] @@ -1450,7 +1727,120 @@ platformdirs = ">=2.4,<4" docs = ["furo (>=2022.12.7)", "proselint (>=0.13)", "sphinx (>=6.1.3)", "sphinx-argparse (>=0.4)", "sphinxcontrib-towncrier (>=0.2.1a0)", "towncrier (>=22.12)"] test = ["covdefaults (>=2.2.2)", "coverage (>=7.1)", "coverage-enable-subprocess (>=1)", "flaky (>=3.7)", "packaging (>=23)", "pytest (>=7.2.1)", "pytest-env (>=0.8.1)", "pytest-freezegun (>=0.4.2)", "pytest-mock (>=3.10)", "pytest-randomly (>=3.12)", "pytest-timeout (>=2.1)"] +[[package]] +name = "webdriver-manager" +version = "3.8.6" +description = "Library provides the way to automatically manage drivers for different browsers" +category = "main" +optional = false +python-versions = ">=3.7" +files = [ + {file = "webdriver_manager-3.8.6-py2.py3-none-any.whl", hash = "sha256:7d3aa8d67bd6c92a5d25f4abd75eea2c6dd24ea6617bff986f502280903a0e2b"}, + {file = "webdriver_manager-3.8.6.tar.gz", hash = "sha256:ee788d389b8f45222a8a62f6f39b579360a1f87be46dad6da89918354af3ce73"}, +] + +[package.dependencies] +packaging = "*" +python-dotenv = "*" +requests = "*" +tqdm = "*" + +[[package]] +name = "websockets" +version = "11.0.1" +description = "An implementation of the WebSocket Protocol (RFC 6455 & 7692)" +category = "main" +optional = false +python-versions = ">=3.7" +files = [ + {file = "websockets-11.0.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:3d30cc1a90bcbf9e22e1f667c1c5a7428e2d37362288b4ebfd5118eb0b11afa9"}, + {file = "websockets-11.0.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:dc77283a7c7b2b24e00fe8c3c4f7cf36bba4f65125777e906aae4d58d06d0460"}, + {file = "websockets-11.0.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:0929c2ebdf00cedda77bf77685693e38c269011236e7c62182fee5848c29a4fa"}, + {file = "websockets-11.0.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:db234da3aff01e8483cf0015b75486c04d50dbf90004bd3e5b46d384e1bd6c9e"}, + {file = "websockets-11.0.1-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c7fdfbed727ce6b4b5e6622d15a6efb2098b2d9e22ba4dc54b2e3ce80f982045"}, + {file = "websockets-11.0.1-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d5f3d0d177b3db3d1d02cce7ba6c0063586499ac28afe0c992be74ffc40d9257"}, + {file = "websockets-11.0.1-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:25ea5dbd3b00c56b034639dc6fe4d1dd095b8205bab1782d9a47cb020695fdf4"}, + {file = "websockets-11.0.1-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:dbeada3b8f1f6d9497840f761906c4236f912a42da4515520168bc7c525b52b0"}, + {file = "websockets-11.0.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:892959b627eedcdf98ac7022f9f71f050a59624b380b67862da10c32ea3c221a"}, + {file = "websockets-11.0.1-cp310-cp310-win32.whl", hash = "sha256:fc0a96a6828bfa6f1ccec62b54630bcdcc205d483f5a8806c0a8abb26101c54b"}, + {file = "websockets-11.0.1-cp310-cp310-win_amd64.whl", hash = "sha256:3a88375b648a2c479532943cc19a018df1e5fcea85d5f31963c0b22794d1bdc1"}, + {file = "websockets-11.0.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:3cf18bbd44b36749b7b66f047a30a40b799b8c0bd9a1b9173cba86a234b4306b"}, + {file = "websockets-11.0.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:deb0dd98ea4e76b833f0bfd7a6042b51115360d5dfcc7c1daa72dfc417b3327a"}, + {file = "websockets-11.0.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:45a85dc6b3ff76239379feb4355aadebc18d6e587c8deb866d11060755f4d3ea"}, + {file = "websockets-11.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5d68bd2a3e9fff6f7043c0a711cb1ebba9f202c196a3943d0c885650cd0b6464"}, + {file = "websockets-11.0.1-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:cfd0b9b18d64c51e5cd322e16b5bf4fe490db65c9f7b18fd5382c824062ead7e"}, + {file = "websockets-11.0.1-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ef0e6253c36e42f2637cfa3ff9b3903df60d05ec040c718999f6a0644ce1c497"}, + {file = "websockets-11.0.1-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:12180bc1d72c6a9247472c1dee9dfd7fc2e23786f25feee7204406972d8dab39"}, + {file = "websockets-11.0.1-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:a797da96d4127e517a5cb0965cd03fd6ec21e02667c1258fa0579501537fbe5c"}, + {file = "websockets-11.0.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:07cc20655fb16aeef1a8f03236ba8671c61d332580b996b6396a5b7967ba4b3d"}, + {file = "websockets-11.0.1-cp311-cp311-win32.whl", hash = "sha256:a01c674e0efe0f14aec7e722ed0e0e272fa2f10e8ea8260837e1f4f5dc4b3e53"}, + {file = "websockets-11.0.1-cp311-cp311-win_amd64.whl", hash = "sha256:2796f097841619acf053245f266a4f66cb27c040f0d9097e5f21301aab95ff43"}, + {file = "websockets-11.0.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:54d084756c50dfc8086dce97b945f210ca43950154e1e04a44a30c6e6a2bcbb1"}, + {file = "websockets-11.0.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4fe2aed5963ca267c40a2d29b1ee4e8ab008ac8d5daa284fdda9275201b8a334"}, + {file = "websockets-11.0.1-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:84e92dbac318a84fef722f38ca57acef19cbb89527aba5d420b96aa2656970ee"}, + {file = "websockets-11.0.1-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ec4e87eb9916b481216b1fede7d8913be799915f5216a0c801867cbed8eeb903"}, + {file = "websockets-11.0.1-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:d4e0990b6a04b07095c969969da659eecf9069cf8e7b8f49c8f5ee1bb50e3352"}, + {file = "websockets-11.0.1-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:c90343fd0774749d23c1891dd8b3e9210f9afd30986673ce0f9d5857f5cb1562"}, + {file = "websockets-11.0.1-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:ac042e8ba9d7f2618e84af27927fdce0f3e03528eb74f343977486c093868389"}, + {file = "websockets-11.0.1-cp37-cp37m-win32.whl", hash = "sha256:385c5391becb9b58e0a4f33345e12762fd857ccf9fbf6fee428669929ba45e4c"}, + {file = "websockets-11.0.1-cp37-cp37m-win_amd64.whl", hash = "sha256:aef1602db81096ce3d3847865128c8879635bdad7963fb2b7df290edb9e9150a"}, + {file = "websockets-11.0.1-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:52ba83ea132390e426f9a7b48848248a2dc0e7120ca8c65d5a8fc1efaa4eb51b"}, + {file = "websockets-11.0.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:007ed0d62f7e06eeb6e3a848b0d83b9fbd9e14674a59a61326845f27d20d7452"}, + {file = "websockets-11.0.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:f888b9565ca1d1c25ab827d184f57f4772ffbfa6baf5710b873b01936cc335ee"}, + {file = "websockets-11.0.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:db78535b791840a584c48cf3f4215eae38a7e2f43271ecd27ce4ba8a798beaaa"}, + {file = "websockets-11.0.1-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:fa1c23ed3a02732fba906ec337df65d4cc23f9f453635e1a803c285b59c7d987"}, + {file = "websockets-11.0.1-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e039f106d48d3c241f1943bccfb383bd38ec39900d6dcaad0c73cc5fe129f346"}, + {file = "websockets-11.0.1-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:b0ed24a3aa4213029e100257e5e73c5f912e70ca35630081de94b7f9e2cf4a9b"}, + {file = "websockets-11.0.1-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:d5a3022f9291bf2d35ebf65929297d625e68effd3a5647b8eb8b89d51b09394c"}, + {file = "websockets-11.0.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:1cb23597819f68ac6a6d133a002a1b3ef12a22850236b083242c93f81f206d5a"}, + {file = "websockets-11.0.1-cp38-cp38-win32.whl", hash = "sha256:349dd1fa56a30d530555988be98013688de67809f384671883f8bf8b8c9de984"}, + {file = "websockets-11.0.1-cp38-cp38-win_amd64.whl", hash = "sha256:87ae582cf2319e45bc457a57232daded27a3c771263cab42fb8864214bbd74ea"}, + {file = "websockets-11.0.1-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:a88815a0c6253ad1312ef186620832fb347706c177730efec34e3efe75e0e248"}, + {file = "websockets-11.0.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:d5a6fa353b5ef36970c3bd1cd7cecbc08bb8f2f1a3d008b0691208cf34ebf5b0"}, + {file = "websockets-11.0.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:5ffe6fc5e5fe9f2634cdc59b805e4ba1fcccf3a5622f5f36c3c7c287f606e283"}, + {file = "websockets-11.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b138f4bf8a64c344e12c76283dac279d11adab89ac62ae4a32ac8490d3c94832"}, + {file = "websockets-11.0.1-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:aedd94422745da60672a901f53de1f50b16e85408b18672b9b210db4a776b5a6"}, + {file = "websockets-11.0.1-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a4667d4e41fa37fa3d836b2603b8b40d6887fa4838496d48791036394f7ace39"}, + {file = "websockets-11.0.1-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:43e0de552be624e5c0323ff4fcc9f0b4a9a6dc6e0116b8aa2cbb6e0d3d2baf09"}, + {file = "websockets-11.0.1-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:ceeef57b9aec8f27e523de4da73c518ece7721aefe7064f18aa28baabfe61b94"}, + {file = "websockets-11.0.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:9d91279d57f6546eaf43671d1de50621e0578f13c2f17c96c458a72d170698d7"}, + {file = "websockets-11.0.1-cp39-cp39-win32.whl", hash = "sha256:29282631da3bfeb5db497e4d3d94d56ee36222fbebd0b51014e68a2e70736fb1"}, + {file = "websockets-11.0.1-cp39-cp39-win_amd64.whl", hash = "sha256:e2654e94c705ce9b768441d8e3a387a84951ca1056efdc4a26a4a6ee723c01b6"}, + {file = "websockets-11.0.1-pp37-pypy37_pp73-macosx_10_9_x86_64.whl", hash = "sha256:60a19d4ff5f451254f8623f6aa4169065f73a50ec7b59ab6b9dcddff4aa00267"}, + {file = "websockets-11.0.1-pp37-pypy37_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2a58e83f82098d062ae5d4cbe7073b8783999c284d6f079f2fefe87cd8957ac8"}, + {file = "websockets-11.0.1-pp37-pypy37_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b91657b65355954e47f0df874917fa200426b3a7f4e68073326a8cfc2f6deef8"}, + {file = "websockets-11.0.1-pp37-pypy37_pp73-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:53b8e1ee01eb5b8be5c8a69ae26b0820dbc198d092ad50b3451adc3cdd55d455"}, + {file = "websockets-11.0.1-pp37-pypy37_pp73-win_amd64.whl", hash = "sha256:5d8d5d17371ed9eb9f0e3a8d326bdf8172700164c2e705bc7f1905a719a189be"}, + {file = "websockets-11.0.1-pp38-pypy38_pp73-macosx_10_9_x86_64.whl", hash = "sha256:e53419201c6c1439148feb99de6b307651a88b8defd41348cc23bbe2a290de1d"}, + {file = "websockets-11.0.1-pp38-pypy38_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:718d19c494637f28e651031b3df6a791b9e86e0097c65ed5e8ec49b400b1210e"}, + {file = "websockets-11.0.1-pp38-pypy38_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f7b2544eb3e7bc39ce59812371214cd97762080dab90c3afc857890039384753"}, + {file = "websockets-11.0.1-pp38-pypy38_pp73-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ec4a887d2236e3878c07033ad5566f6b4d5d954b85f92a219519a1745d0c93e9"}, + {file = "websockets-11.0.1-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:ae59a9f0a77ecb0cbdedea7d206a547ff136e8bfbc7d2d98772fb02d398797bb"}, + {file = "websockets-11.0.1-pp39-pypy39_pp73-macosx_10_9_x86_64.whl", hash = "sha256:ef35cef161f76031f833146f895e7e302196e01c704c00d269c04d8e18f3ac37"}, + {file = "websockets-11.0.1-pp39-pypy39_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:79b6548e57ab18f071b9bfe3ffe02af7184dd899bc674e2817d8fe7e9e7489ec"}, + {file = "websockets-11.0.1-pp39-pypy39_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a8d9793f3fb0da16232503df14411dabafed5a81fc9077dc430cfc6f60e71179"}, + {file = "websockets-11.0.1-pp39-pypy39_pp73-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:42aa05e890fcf1faed8e535c088a1f0f27675827cbacf62d3024eb1e6d4c9e0c"}, + {file = "websockets-11.0.1-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:5d4f4b341100d313b08149d7031eb6d12738ac758b0c90d2f9be8675f401b019"}, + {file = "websockets-11.0.1-py3-none-any.whl", hash = "sha256:85b4127f7da332feb932eee833c70e5e1670469e8c9de7ef3874aa2a91a6fbb2"}, + {file = "websockets-11.0.1.tar.gz", hash = "sha256:369410925b240b30ef1c1deadbd6331e9cd865ad0b8966bf31e276cc8e0da159"}, +] + +[[package]] +name = "wsproto" +version = "1.2.0" +description = "WebSockets state-machine based protocol implementation" +category = "main" +optional = false +python-versions = ">=3.7.0" +files = [ + {file = "wsproto-1.2.0-py3-none-any.whl", hash = "sha256:b9acddd652b585d75b20477888c56642fdade28bdfd3579aa24a4d2c037dd736"}, + {file = "wsproto-1.2.0.tar.gz", hash = "sha256:ad565f26ecb92588a3e43bc3d96164de84cd9902482b130d0ddbaa9664a85065"}, +] + +[package.dependencies] +h11 = ">=0.9.0,<1" + [metadata] lock-version = "2.0" python-versions = "^3.8" -content-hash = "ff60e52da21cf017bd24ba13550cf556582f6add0db46e33d16cb35f727017c8" +content-hash = "51556b8992d9ad3b62fff8227f8b6a2b928eed5a22b355bb647ca7fd1a4bfe71" diff --git a/pyproject.toml b/pyproject.toml index 30cda5c..6459375 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -21,6 +21,9 @@ classifiers = [ python = "^3.8" dataclasses-json = "*" requests = "*" +selenium = "^4.10" +webdriver-manager = "^3.8.6" +undetected-chromedriver = "^3.5.0" [tool.poetry.dev-dependencies] bandit = {extras = ["toml"], version = "*"} From cf49c7dbe4d83d55a71e29fa019dad3473f954cf Mon Sep 17 00:00:00 2001 From: Stephen Kent Date: Sun, 9 Jul 2023 10:29:45 -0700 Subject: [PATCH 2/6] Update Dockerfile for chromedriver --- Dockerfile | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index 5f72dd2..b76687d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM python:3-alpine +FROM python:3 ARG POETRY_DYNAMIC_VERSIONING_BYPASS="0.0.0" ENV CRON_SCHEDULE "5 2 * * *" ENV SMTPHOST= @@ -8,11 +8,18 @@ ENV SAFEWAY_ACCOUNT_MAIL_FROM= ENV SAFEWAY_ACCOUNT_MAIL_TO= ENV SAFEWAY_ACCOUNTS_FILE= -RUN apk add --no-cache tini +ENV DISPLAY=:1 +RUN DEBIAN_FRONTEND=noninteractive && \ + wget -q -O - https://dl-ssl.google.com/linux/linux_signing_key.pub > /usr/share/keyrings/chrome.pub && \ + echo 'deb [arch=amd64 signed-by=/usr/share/keyrings/chrome.pub] http://dl.google.com/linux/chrome/deb/ stable main' > /etc/apt/sources.list.d/google-chrome.list && \ + apt update -y && \ + apt install -y google-chrome-stable +RUN apt install -y tini busybox-static xvfb + COPY docker/entrypoint / COPY . /python-build RUN python3 -m pip install /python-build && rm -rf /python-build -ENTRYPOINT ["/sbin/tini", "--"] +ENTRYPOINT ["/usr/bin/tini", "--"] CMD ["/entrypoint"] From c6d7d27fe66fd46bac3ff5a1c31eb21c292e3a9a Mon Sep 17 00:00:00 2001 From: Stephen Kent Date: Sun, 9 Jul 2023 10:30:11 -0700 Subject: [PATCH 3/6] Update entrypoint for chromedriver --- docker/entrypoint | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/docker/entrypoint b/docker/entrypoint index c5d21ec..4a72cb5 100755 --- a/docker/entrypoint +++ b/docker/entrypoint @@ -21,11 +21,13 @@ else env | grep -vie 'password' | grep -e '^SAFEWAY_' -e '^SMTPHOST=' fi -mkdir /etc/cron.d +mkdir -vp /etc/cron.d /var/spool/cron/crontabs ( echo "${CRON_SCHEDULE?} safeway-coupons ${args} >/proc/1/fd/1 2>/proc/1/fd/2" ) > /var/spool/cron/crontabs/root -crontab -l +busybox crontab -l -exec crond -f -d 8 +Xvfb "${DISPLAY}" -screen "${DISPLAY}" 1280x1024x16 & + +exec busybox crond -f From fdce19bae63491922f2517527bee985afa571f00 Mon Sep 17 00:00:00 2001 From: Stephen Kent Date: Sun, 9 Jul 2023 10:54:16 -0700 Subject: [PATCH 4/6] Perform Safeway login using undetected-chromedriver Use official busybox binary for crond, crontab, sendmail --- Dockerfile | 11 ++- README.md | 23 +++++- docker/entrypoint | 2 - safeway_coupons/session.py | 151 ++++++++++++++++++++++++------------- safeway_coupons/utils.py | 118 ----------------------------- tests/conftest.py | 67 +++++++++------- 6 files changed, 170 insertions(+), 202 deletions(-) diff --git a/Dockerfile b/Dockerfile index b76687d..d07e078 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,3 +1,5 @@ +FROM busybox + FROM python:3 ARG POETRY_DYNAMIC_VERSIONING_BYPASS="0.0.0" ENV CRON_SCHEDULE "5 2 * * *" @@ -8,13 +10,18 @@ ENV SAFEWAY_ACCOUNT_MAIL_FROM= ENV SAFEWAY_ACCOUNT_MAIL_TO= ENV SAFEWAY_ACCOUNTS_FILE= -ENV DISPLAY=:1 RUN DEBIAN_FRONTEND=noninteractive && \ wget -q -O - https://dl-ssl.google.com/linux/linux_signing_key.pub > /usr/share/keyrings/chrome.pub && \ echo 'deb [arch=amd64 signed-by=/usr/share/keyrings/chrome.pub] http://dl.google.com/linux/chrome/deb/ stable main' > /etc/apt/sources.list.d/google-chrome.list && \ apt update -y && \ apt install -y google-chrome-stable -RUN apt install -y tini busybox-static xvfb +RUN apt install -y tini + +# Install busybox utilities using static binary from official image +COPY --from=busybox /bin/busybox /bin/busybox +RUN for target in /usr/sbin/sendmail /usr/sbin/crond /usr/bin/crontab; do \ + ln -svf /bin/busybox ${target}; \ + done COPY docker/entrypoint / diff --git a/README.md b/README.md index d89f409..00f88f0 100644 --- a/README.md +++ b/README.md @@ -10,6 +10,17 @@ and attempt to select all of the "Safeway for U" electronic coupons on the site so they don't have to each be clicked manually. +## Design notes + +Safeway's sign in page is protected by a web application firewall (WAF). +safeway-coupons performs authentication using a headless instance of Google +Chrome. Authentication may fail based on your IP's reputation, either by +presenting a CAPTCHA or denying sign in attempts altogether. safeway-coupons +currently does not have support for prompting the user to solve CAPTCHAs. + +Once a signed in session is established, coupon clipping is performed using HTTP +requests via [requests][requests]. + ## Installation and usage with Docker A Docker container is provided which runs safeway-coupons with cron. The cron @@ -65,18 +76,23 @@ docker-compose logs -f ## Installation from PyPI +### Prerequisites + +* Google Chrome (for authentication performed via Selenium). +* Optional: `sendmail` (for email support) + +### Installation + [safeway-coupons is available on PyPI][pypi]: ```console pip install safeway-coupons ``` -`sendmail` is needed for email support. +### Usage For best results, run this program once a day or so with a cron daemon. -### Usage - For full usage options, run ```console @@ -163,3 +179,4 @@ Created from [smkent/cookie-python][cookie-python] using [poetry]: https://python-poetry.org/docs/#installation [pypi]: https://pypi.org/project/safeway-coupons/ [repo]: https://github.com/smkent/safeway-coupons +[requests]: https://requests.readthedocs.io/en/latest/ diff --git a/docker/entrypoint b/docker/entrypoint index 4a72cb5..72af319 100755 --- a/docker/entrypoint +++ b/docker/entrypoint @@ -28,6 +28,4 @@ mkdir -vp /etc/cron.d /var/spool/cron/crontabs busybox crontab -l -Xvfb "${DISPLAY}" -screen "${DISPLAY}" 1280x1024x16 & - exec busybox crond -f diff --git a/safeway_coupons/session.py b/safeway_coupons/session.py index 30ccdeb..d4d5de0 100644 --- a/safeway_coupons/session.py +++ b/safeway_coupons/session.py @@ -1,22 +1,16 @@ import json +import time import urllib -from typing import Optional +from typing import Any, Optional import requests +import selenium.webdriver.support.expected_conditions as ec +import undetected_chromedriver as uc # type: ignore +from selenium.webdriver.remote.webdriver import By +from selenium.webdriver.support.wait import WebDriverWait from .accounts import Account from .errors import AuthenticationFailure -from .utils import make_nonce, make_token - -LOGIN_URL = "https://albertsons.okta.com/api/v1/authn" -AUTHORIZE_URL = ( - "https://albertsons.okta.com/oauth2/ausp6soxrIyPrm8rS2p6/v1/authorize" -) - -OAUTH_CLIENT_ID = "0oap6ku01XJqIRdl42p6" -OAUTH_REDIRECT_URI = ( - "https://www.safeway.com/bin/safeway/unified/sso/authorize" -) class BaseSession: @@ -47,43 +41,98 @@ def __init__(self, account: Account) -> None: raise AuthenticationFailure(e, account) from e def _login(self, account: Account) -> None: - # Log in - response = self.requests.post( - LOGIN_URL, - json={"username": account.username, "password": account.password}, - ) - response.raise_for_status() - login_data = response.json() - if login_data.get("status") != "SUCCESS": - raise Exception("Login was not successful") - session_token = login_data["sessionToken"] - # Retrieve session information - state_token = make_token() - nonce = make_nonce() - params = { - "client_id": OAUTH_CLIENT_ID, - "redirect_uri": OAUTH_REDIRECT_URI, - "response_type": "code", - "response_mode": "query", - "state": state_token, - "nonce": nonce, - "prompt": "none", - "sessionToken": session_token, - "scope": "openid profile email offline_access used_credentials", - } - url = f"{AUTHORIZE_URL}?{urllib.parse.urlencode(params)}" - response = self.requests.get(url) - response.raise_for_status() - session = json.loads( - urllib.parse.unquote(self.requests.cookies["SWY_SHARED_SESSION"]) - ) - self.access_token = session["accessToken"] - session_info = json.loads( - urllib.parse.unquote( - self.requests.cookies["SWY_SHARED_SESSION_INFO"] + screenshot_dir = "/data" + options = uc.ChromeOptions() + for option in [ + "--incognito", + "--no-sandbox", + "--disable-extensions", + "--disable-application-cache", + "--disable-gpu", + "--disable-setuid-sandbox", + "--disable-dev-shm-usage", + "--headless=new", + ]: + options.add_argument(option) + with uc.Chrome(options=options) as driver: + driver.implicitly_wait(10) + # Navigate to the website URL + url = "https://www.safeway.com" + print("GO", url) + driver.get(url) + print("CLICK") + button = driver.find_element( + By.XPATH, "//button [contains(text(), 'Necessary Only')]" ) - ) - try: - self.store_id = session_info["info"]["J4U"]["storeId"] - except Exception as e: - raise Exception("Unable to retrieve store ID") from e + if button: + print("click no cookie-button") + button.click() + print("SS 0") + driver.save_screenshot(f"{screenshot_dir}/screenshot_0.png") + driver.find_element( + By.XPATH, "//span [contains(text(), 'Sign In')]" + ).click() + time.sleep(2) + driver.find_element( + By.XPATH, "//a [contains(text(), 'Sign In')]" + ).click() + time.sleep(2) + + driver.find_element(By.ID, "label-email").send_keys( + account.username + ) + driver.find_element(By.ID, "label-password").send_keys( + account.password + ) + print("CLICK 2") + time.sleep(0.5) + driver.find_element( + By.XPATH, "//span [contains(text(), 'Keep Me Signed In')]" + ).click() + print("SS 1") + driver.save_screenshot(f"{screenshot_dir}/screenshot_1.png") + # print("RETURN") + # return + time.sleep(0.5) + print("CLICK 3") + driver.find_element("id", "btnSignIn").click() + time.sleep(0.5) + wdw = WebDriverWait(driver, 10) + wdw.until( + ec.text_to_be_present_in_element( + (By.XPATH, '//span [contains(@class, "user-greeting")]'), + "Account", + ) + ) + el = driver.find_element( + By.XPATH, '//span [contains(@class, "user-greeting")]' + ) + print("TEXT", el.text) + print("SS 2") + driver.save_screenshot(f"{screenshot_dir}/screenshot_2.png") + print("PRINT COOKIE") + session_cookie = self._parse_cookie_value( + driver.get_cookie("SWY_SHARED_SESSION")["value"] + ) + session_info_cookie = self._parse_cookie_value( + driver.get_cookie("SWY_SHARED_SESSION_INFO")["value"] + ) + from pprint import pprint + + print("SESSION COOKIE") + pprint(session_cookie) + print("SESSION INFO COOKIE") + pprint(session_info_cookie) + print("SESSION COOKIE ACCESS TOKEN") + pprint(session_cookie["accessToken"]) + self.access_token = session_cookie["accessToken"] + print("SESSION COOKIE STORE ID") + try: + pprint(session_info_cookie["info"]["J4U"]["storeId"]) + self.store_id = session_info_cookie["info"]["J4U"]["storeId"] + except Exception as e: + raise Exception("Unable to retrieve store ID") from e + print("DONE LOGGING IN") + + def _parse_cookie_value(self, value: str) -> Any: + return json.loads(urllib.parse.unquote(value)) diff --git a/safeway_coupons/utils.py b/safeway_coupons/utils.py index e1cdf20..d5e52cb 100644 --- a/safeway_coupons/utils.py +++ b/safeway_coupons/utils.py @@ -1,128 +1,10 @@ import random -import string import time from typing import Generator, Iterable, TypeVar T = TypeVar("T") -WORD_LIST = [ - "almost", - "avid", - "awning", - "bakery", - "barmaid", - "bribe", - "briskness", - "canopener", - "collage", - "component", - "computing", - "corned", - "cried", - "crumb", - "curliness", - "curvature", - "cycle", - "darkening", - "deflector", - "desolate", - "detergent", - "dipped", - "disfigure", - "distinct", - "duress", - "ebony", - "elated", - "elusive", - "emcee", - "emphasize", - "enjoyable", - "exact", - "fall", - "flatness", - "footpath", - "getaway", - "groom", - "guts", - "hangnail", - "headlamp", - "headlock", - "jailbird", - "kinetic", - "landfill", - "linguini", - "maroon", - "mouse", - "nectar", - "neuron", - "numerous", - "nylon", - "ounce", - "outburst", - "overbuilt", - "pogo", - "profusely", - "province", - "puzzling", - "quickly", - "rascal", - "record", - "refutable", - "robotics", - "rule", - "sadness", - "sandlot", - "schematic", - "scruffy", - "secrecy", - "shaping", - "skipper", - "smolder", - "smooth", - "snowiness", - "spirits", - "spoon", - "spray", - "steering", - "stung", - "stylized", - "subarctic", - "sulfate", - "throwback", - "tipper", - "tweezers", - "uncoated", - "unlovely", - "unmanaged", - "unmatched", - "unproven", - "unrevised", - "uplifted", - "viability", - "vocalist", - "vocalize", - "washbasin", - "washing", - "womb", - "xerox", - "yin", -] - - -def make_token() -> str: - return "-".join(random.choices(WORD_LIST, k=4)) - - -def make_nonce() -> str: - return "".join( - random.choices( - string.ascii_lowercase + string.ascii_uppercase + string.digits, - k=62, - ) - ) - - def yield_delay( iterable: Iterable[T], sleep_level: int, debug_level: int ) -> Generator[T, None, None]: diff --git a/tests/conftest.py b/tests/conftest.py index c8cf92a..9da966f 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,12 +1,15 @@ import json import time -from typing import Iterable, List, Mapping, Tuple, cast +import urllib +from typing import Dict, Iterator, List, Mapping, Optional, Tuple, cast from unittest import mock import pytest import pytest_mock import requests import responses +import undetected_chromedriver as uc # type: ignore +from selenium.webdriver.support.wait import WebDriverWait from safeway_coupons.models import Offer, OfferList @@ -14,39 +17,51 @@ @pytest.fixture -def http_responses() -> Iterable[responses.RequestsMock]: +def http_responses() -> Iterator[responses.RequestsMock]: with responses.RequestsMock() as resp_mock: yield resp_mock +@pytest.fixture(autouse=True) +def mock_undetected_chromedriver( + mock_web_driver_wait: mock.MagicMock, + mock_sleep: mock.MagicMock, +) -> Iterator[mock.MagicMock]: + with mock.patch.object(uc, "Chrome") as mock_uc: + mock_driver = mock_uc.return_value.__enter__.return_value + mock_uc.return_value.__exit__.side_effect = ( + lambda *a, **kw: mock_sleep.reset_mock() + ) + yield mock_driver + + +@pytest.fixture(autouse=True) +def mock_web_driver_wait() -> Iterator[mock.MagicMock]: + with mock.patch.object(WebDriverWait, "until") as mock_wdw: + yield mock_wdw + + @pytest.fixture -def login_success(http_responses: responses.RequestsMock) -> None: - http_responses.add( - method=responses.POST, - url="https://albertsons.okta.com/api/v1/authn", - body=json.dumps({"status": "SUCCESS", "sessionToken": "test"}), - ) - session_data = {"accessToken": "test_token"} - session_info = {"info": {"J4U": {"storeId": 42}}} - http_responses.add( - method=responses.GET, - url=( - "https://albertsons.okta.com" - "/oauth2/ausp6soxrIyPrm8rS2p6/v1/authorize" - ), - headers={ - "Set-Cookie": f"SWY_SHARED_SESSION={json.dumps(session_data)} ", - "set-cookie": ( - f"SWY_SHARED_SESSION_INFO={json.dumps(session_info)}" - ), - }, - ) +def login_success(mock_undetected_chromedriver: mock.MagicMock) -> None: + cookies = { + "SWY_SHARED_SESSION": {"accessToken": "test_token"}, + "SWY_SHARED_SESSION_INFO": {"info": {"J4U": {"storeId": 42}}}, + } + + def _get_cookie(name: str) -> Dict[str, Optional[str]]: + value = cookies.get(name) + return { + "value": urllib.parse.quote(json.dumps(value)) if value else None + } + + mock_undetected_chromedriver.get_cookie.side_effect = _get_cookie + return @pytest.fixture def clips( http_responses: responses.RequestsMock, -) -> Iterable[ClipsTestConfig]: +) -> Iterator[ClipsTestConfig]: clips_test_config = ClipsTestConfig() def _clip_response( @@ -85,7 +100,7 @@ def _clip_response( @pytest.fixture def available_offers( http_responses: responses.RequestsMock, -) -> Iterable[List[Offer]]: +) -> Iterator[List[Offer]]: offers_list: List[Offer] = [] http_responses.add_callback( method=responses.GET, @@ -104,6 +119,6 @@ def available_offers( @pytest.fixture -def mock_sleep(mocker: pytest_mock.MockerFixture) -> Iterable[mock.MagicMock]: +def mock_sleep(mocker: pytest_mock.MockerFixture) -> Iterator[mock.MagicMock]: mocker.patch.object(time, "sleep") yield cast(mock.MagicMock, time.sleep) From c7cb2640c58b6cc1d29df5e25826e3c619ca3412 Mon Sep 17 00:00:00 2001 From: Stephen Kent Date: Sun, 9 Jul 2023 12:00:06 -0700 Subject: [PATCH 5/6] Add `-D`/`--debug-dir` option for error screenshot output --- Dockerfile | 1 + docker/entrypoint | 4 + safeway_coupons/app.py | 13 ++++ safeway_coupons/client.py | 5 +- safeway_coupons/safeway.py | 7 +- safeway_coupons/session.py | 154 ++++++++++++++++++------------------- tests/test_app.py | 5 ++ 7 files changed, 106 insertions(+), 83 deletions(-) diff --git a/Dockerfile b/Dockerfile index d07e078..a792198 100644 --- a/Dockerfile +++ b/Dockerfile @@ -9,6 +9,7 @@ ENV SAFEWAY_ACCOUNT_PASSWORD= ENV SAFEWAY_ACCOUNT_MAIL_FROM= ENV SAFEWAY_ACCOUNT_MAIL_TO= ENV SAFEWAY_ACCOUNTS_FILE= +ENV DEBUG_DIR="/debug" RUN DEBIAN_FRONTEND=noninteractive && \ wget -q -O - https://dl-ssl.google.com/linux/linux_signing_key.pub > /usr/share/keyrings/chrome.pub && \ diff --git a/docker/entrypoint b/docker/entrypoint index 72af319..2903f95 100755 --- a/docker/entrypoint +++ b/docker/entrypoint @@ -21,6 +21,10 @@ else env | grep -vie 'password' | grep -e '^SAFEWAY_' -e '^SMTPHOST=' fi +if [ -d "${DEBUG_DIR}" ]; then + args="${args} --debug-dir ${DEBUG_DIR}" +fi + mkdir -vp /etc/cron.d /var/spool/cron/crontabs ( echo "${CRON_SCHEDULE?} safeway-coupons ${args} >/proc/1/fd/1 2>/proc/1/fd/2" diff --git a/safeway_coupons/app.py b/safeway_coupons/app.py index c9ab440..2402d06 100644 --- a/safeway_coupons/app.py +++ b/safeway_coupons/app.py @@ -1,6 +1,7 @@ import argparse import sys from http.client import HTTPConnection +from pathlib import Path from .config import Config from .safeway import SafewayCoupons @@ -19,6 +20,17 @@ def _parse_args() -> argparse.Namespace: "accounts information" ), ) + arg_parser.add_argument( + "-D", + "--debug-dir", + dest="debug_dir", + metavar="directory", + default=".", + help=( + "Destination directory for debug output files, " + "such as browser screenshots (default: %(default)s)" + ), + ) arg_parser.add_argument( "-d", "--debug", @@ -79,6 +91,7 @@ def main() -> None: sc = SafewayCoupons( send_email=args.send_email, debug_level=args.debug_level, + debug_dir=Path(args.debug_dir) if args.debug_dir else None, sleep_level=args.sleep_level, dry_run=args.dry_run, max_clip_count=args.max_clip_count, diff --git a/safeway_coupons/client.py b/safeway_coupons/client.py index 19a1f5e..a03f15b 100644 --- a/safeway_coupons/client.py +++ b/safeway_coupons/client.py @@ -1,5 +1,6 @@ import json import random +from pathlib import Path from typing import List, Optional import requests @@ -12,8 +13,8 @@ class SafewayClient(BaseSession): - def __init__(self, account: Account) -> None: - self.session = LoginSession(account) + def __init__(self, account: Account, debug_dir: Optional[Path]) -> None: + self.session = LoginSession(account, debug_dir) self.requests.headers.update( { "Authorization": f"Bearer {self.session.access_token}", diff --git a/safeway_coupons/safeway.py b/safeway_coupons/safeway.py index 8c000a1..398cc4f 100644 --- a/safeway_coupons/safeway.py +++ b/safeway_coupons/safeway.py @@ -1,4 +1,5 @@ -from typing import List +from pathlib import Path +from typing import List, Optional from .accounts import Account from .client import SafewayClient @@ -15,6 +16,7 @@ def __init__( self, send_email: bool = True, debug_level: int = 0, + debug_dir: Optional[Path] = None, sleep_level: int = 0, dry_run: bool = False, max_clip_count: int = 0, @@ -22,6 +24,7 @@ def __init__( ) -> None: self.send_email = send_email self.debug_level = debug_level + self.debug_dir = debug_dir self.sleep_level = sleep_level self.dry_run = dry_run self.max_clip_count = max_clip_count @@ -29,7 +32,7 @@ def __init__( def clip_for_account(self, account: Account) -> None: print(f"Clipping coupons for Safeway account {account.username}") - swy = SafewayClient(account) + swy = SafewayClient(account, self.debug_dir) clipped_offers: List[Offer] = [] clip_errors: List[ClipError] = [] try: diff --git a/safeway_coupons/session.py b/safeway_coupons/session.py index d4d5de0..a897cec 100644 --- a/safeway_coupons/session.py +++ b/safeway_coupons/session.py @@ -1,11 +1,13 @@ import json import time import urllib +from pathlib import Path from typing import Any, Optional import requests import selenium.webdriver.support.expected_conditions as ec import undetected_chromedriver as uc # type: ignore +from selenium.common.exceptions import TimeoutException from selenium.webdriver.remote.webdriver import By from selenium.webdriver.support.wait import WebDriverWait @@ -32,16 +34,16 @@ def requests(self) -> requests.Session: class LoginSession(BaseSession): - def __init__(self, account: Account) -> None: + def __init__(self, account: Account, debug_dir: Optional[Path]) -> None: self.access_token: Optional[str] = None self.store_id: Optional[str] = None + self.debug_dir: Optional[Path] = debug_dir try: self._login(account) except Exception as e: raise AuthenticationFailure(e, account) from e def _login(self, account: Account) -> None: - screenshot_dir = "/data" options = uc.ChromeOptions() for option in [ "--incognito", @@ -55,84 +57,78 @@ def _login(self, account: Account) -> None: ]: options.add_argument(option) with uc.Chrome(options=options) as driver: - driver.implicitly_wait(10) - # Navigate to the website URL - url = "https://www.safeway.com" - print("GO", url) - driver.get(url) - print("CLICK") - button = driver.find_element( - By.XPATH, "//button [contains(text(), 'Necessary Only')]" - ) - if button: - print("click no cookie-button") - button.click() - print("SS 0") - driver.save_screenshot(f"{screenshot_dir}/screenshot_0.png") - driver.find_element( - By.XPATH, "//span [contains(text(), 'Sign In')]" - ).click() - time.sleep(2) - driver.find_element( - By.XPATH, "//a [contains(text(), 'Sign In')]" - ).click() - time.sleep(2) - - driver.find_element(By.ID, "label-email").send_keys( - account.username - ) - driver.find_element(By.ID, "label-password").send_keys( - account.password - ) - print("CLICK 2") - time.sleep(0.5) - driver.find_element( - By.XPATH, "//span [contains(text(), 'Keep Me Signed In')]" - ).click() - print("SS 1") - driver.save_screenshot(f"{screenshot_dir}/screenshot_1.png") - # print("RETURN") - # return - time.sleep(0.5) - print("CLICK 3") - driver.find_element("id", "btnSignIn").click() - time.sleep(0.5) - wdw = WebDriverWait(driver, 10) - wdw.until( - ec.text_to_be_present_in_element( - (By.XPATH, '//span [contains(@class, "user-greeting")]'), - "Account", - ) - ) - el = driver.find_element( - By.XPATH, '//span [contains(@class, "user-greeting")]' - ) - print("TEXT", el.text) - print("SS 2") - driver.save_screenshot(f"{screenshot_dir}/screenshot_2.png") - print("PRINT COOKIE") - session_cookie = self._parse_cookie_value( - driver.get_cookie("SWY_SHARED_SESSION")["value"] - ) - session_info_cookie = self._parse_cookie_value( - driver.get_cookie("SWY_SHARED_SESSION_INFO")["value"] - ) - from pprint import pprint - - print("SESSION COOKIE") - pprint(session_cookie) - print("SESSION INFO COOKIE") - pprint(session_info_cookie) - print("SESSION COOKIE ACCESS TOKEN") - pprint(session_cookie["accessToken"]) - self.access_token = session_cookie["accessToken"] - print("SESSION COOKIE STORE ID") try: - pprint(session_info_cookie["info"]["J4U"]["storeId"]) - self.store_id = session_info_cookie["info"]["J4U"]["storeId"] - except Exception as e: - raise Exception("Unable to retrieve store ID") from e - print("DONE LOGGING IN") + driver.implicitly_wait(10) + wait = WebDriverWait(driver, 10) + # Navigate to the website URL + url = "https://www.safeway.com" + print("Connect to safeway.com") + driver.get(url) + button = driver.find_element( + By.XPATH, "//button [contains(text(), 'Necessary Only')]" + ) + if button: + print("Decline cookie prompt") + button.click() + print("Open Sign In sidebar") + wait.until( + ec.visibility_of_element_located( + (By.XPATH, "//span [contains(text(), 'Sign In')]") + ) + ).click() + print("Open Sign In form") + wait.until( + ec.visibility_of_element_located( + (By.XPATH, "//a [contains(text(), 'Sign In')]") + ) + ).click() + time.sleep(2) + print("Populate Sign In form") + driver.find_element(By.ID, "label-email").send_keys( + account.username + ) + driver.find_element(By.ID, "label-password").send_keys( + account.password + ) + time.sleep(0.5) + print("Deselect Keep Me Signed In") + driver.find_element( + By.XPATH, "//span [contains(text(), 'Keep Me Signed In')]" + ).click() + time.sleep(0.5) + print("Click Sign In button") + driver.find_element("id", "btnSignIn").click() + time.sleep(0.5) + print("Wait for signed in landing page to load") + wait.until( + ec.text_to_be_present_in_element( + ( + By.XPATH, + '//span [contains(@class, "user-greeting")]', + ), + "Account", + ) + ) + print("Retrieve session information") + session_cookie = self._parse_cookie_value( + driver.get_cookie("SWY_SHARED_SESSION")["value"] + ) + session_info_cookie = self._parse_cookie_value( + driver.get_cookie("SWY_SHARED_SESSION_INFO")["value"] + ) + self.access_token = session_cookie["accessToken"] + try: + self.store_id = session_info_cookie["info"]["J4U"][ + "storeId" + ] + except Exception as e: + raise Exception("Unable to retrieve store ID") from e + except TimeoutException as e: + if self.debug_dir: + driver.save_screenshot( + self.debug_dir / "screenshot_error.png" + ) + raise Exception("Browser authentication timed out") from e def _parse_cookie_value(self, value: str) -> Any: return json.loads(urllib.parse.unquote(value)) diff --git a/tests/test_app.py b/tests/test_app.py index d663555..df80ace 100644 --- a/tests/test_app.py +++ b/tests/test_app.py @@ -1,5 +1,6 @@ import os import sys +from pathlib import Path from typing import Any, Dict, List, Type, cast from unittest import mock @@ -57,6 +58,7 @@ def test_app_error( dict( send_email=True, debug_level=0, + debug_dir=Path("."), sleep_level=0, dry_run=False, max_clip_count=0, @@ -67,6 +69,7 @@ def test_app_error( dict( send_email=True, debug_level=2, + debug_dir=Path("."), sleep_level=2, dry_run=False, max_clip_count=0, @@ -77,6 +80,7 @@ def test_app_error( dict( send_email=False, debug_level=0, + debug_dir=Path("."), sleep_level=0, dry_run=False, max_clip_count=0, @@ -87,6 +91,7 @@ def test_app_error( dict( send_email=True, debug_level=0, + debug_dir=Path("."), sleep_level=0, dry_run=True, max_clip_count=42, From c0902690453b960e2369d074b1deef078e1b2f06 Mon Sep 17 00:00:00 2001 From: Stephen Kent Date: Sun, 9 Jul 2023 19:49:09 -0700 Subject: [PATCH 6/6] Attach screenshot to email on authentication failure --- safeway_coupons/email.py | 34 +++++++++++++++++++++++++++------- safeway_coupons/errors.py | 2 ++ safeway_coupons/safeway.py | 6 +++--- safeway_coupons/session.py | 27 ++++++++++++++++++++++----- 4 files changed, 54 insertions(+), 15 deletions(-) diff --git a/safeway_coupons/email.py b/safeway_coupons/email.py index c80d806..653303e 100644 --- a/safeway_coupons/email.py +++ b/safeway_coupons/email.py @@ -1,7 +1,9 @@ import collections +import mimetypes import os import subprocess -from email.mime.text import MIMEText +from email.message import EmailMessage +from pathlib import Path from typing import List, Optional from .accounts import Account @@ -15,6 +17,7 @@ def _send_email( mail_message: List[str], debug_level: int, send_email: bool, + attachments: Optional[List[Path]] = None, ) -> None: mail_message_str = os.linesep.join(mail_message) if debug_level >= 1: @@ -27,16 +30,26 @@ def _send_email( print("<<<<<<") if not send_email: return - email_data = MIMEText(mail_message_str) - email_data["To"] = account.mail_to - email_data["From"] = account.mail_from + msg = EmailMessage() + msg["To"] = account.mail_to + msg["From"] = account.mail_from if subject: - email_data["Subject"] = subject + msg["Subject"] = subject + msg.set_content(mail_message_str) + for attachment in attachments or []: + mt = mimetypes.guess_type(attachment.name)[0] + main, sub = mt.split("/", 1) if mt else ("application", "octet-stream") + msg.add_attachment( + attachment.read_bytes(), + filename=attachment.name, + maintype=main, + subtype=sub, + ) p = subprocess.Popen( ["/usr/sbin/sendmail", "-f", account.mail_to, "-t"], stdin=subprocess.PIPE, ) - p.communicate(bytes(email_data.as_string(), "UTF-8")) + p.communicate(bytes(msg.as_string(), "UTF-8")) def email_clip_results( @@ -77,4 +90,11 @@ def email_error( mail_message += ["Clipped coupons:", ""] for offer in error.clipped_offers: mail_message += str(offer) - _send_email(account, mail_subject, mail_message, debug_level, send_email) + _send_email( + account, + mail_subject, + mail_message, + debug_level, + send_email, + attachments=getattr(error, "attachments", None), + ) diff --git a/safeway_coupons/errors.py b/safeway_coupons/errors.py index bc6a1d2..1915e5a 100644 --- a/safeway_coupons/errors.py +++ b/safeway_coupons/errors.py @@ -1,4 +1,5 @@ from dataclasses import dataclass +from pathlib import Path from typing import List, Optional import requests @@ -15,6 +16,7 @@ class Error(Exception): @dataclass class AuthenticationFailure(Error): account: Account + attachments: Optional[List[Path]] = None def __str__(self) -> str: return f"Authentication Failure ({self.exception})" diff --git a/safeway_coupons/safeway.py b/safeway_coupons/safeway.py index 398cc4f..27960aa 100644 --- a/safeway_coupons/safeway.py +++ b/safeway_coupons/safeway.py @@ -32,10 +32,10 @@ def __init__( def clip_for_account(self, account: Account) -> None: print(f"Clipping coupons for Safeway account {account.username}") - swy = SafewayClient(account, self.debug_dir) - clipped_offers: List[Offer] = [] - clip_errors: List[ClipError] = [] try: + swy = SafewayClient(account, self.debug_dir) + clipped_offers: List[Offer] = [] + clip_errors: List[ClipError] = [] offers = swy.get_offers() unclipped_offers = [ o for o in offers if o.status == OfferStatus.Unclipped diff --git a/safeway_coupons/session.py b/safeway_coupons/session.py index a897cec..01b655c 100644 --- a/safeway_coupons/session.py +++ b/safeway_coupons/session.py @@ -2,7 +2,7 @@ import time import urllib from pathlib import Path -from typing import Any, Optional +from typing import Any, List, Optional import requests import selenium.webdriver.support.expected_conditions as ec @@ -15,6 +15,16 @@ from .errors import AuthenticationFailure +class ExceptionWithAttachments(Exception): + def __init__( + self, + *args: Any, + attachments: Optional[List[Path]] = None, + **kwargs: Any + ): + self.attachments = attachments + + class BaseSession: USER_AGENT = ( "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:103.0) " @@ -40,6 +50,10 @@ def __init__(self, account: Account, debug_dir: Optional[Path]) -> None: self.debug_dir: Optional[Path] = debug_dir try: self._login(account) + except ExceptionWithAttachments as e: + raise AuthenticationFailure( + e, account, attachments=e.attachments + ) from e except Exception as e: raise AuthenticationFailure(e, account) from e @@ -124,11 +138,14 @@ def _login(self, account: Account) -> None: except Exception as e: raise Exception("Unable to retrieve store ID") from e except TimeoutException as e: + attachments: List[Path] = [] if self.debug_dir: - driver.save_screenshot( - self.debug_dir / "screenshot_error.png" - ) - raise Exception("Browser authentication timed out") from e + path = self.debug_dir / "screenshot.png" + driver.save_screenshot(path) + attachments.append(path) + raise ExceptionWithAttachments( + "Browser authentication timed out", attachments=attachments + ) from e def _parse_cookie_value(self, value: str) -> Any: return json.loads(urllib.parse.unquote(value))