From f257591e4f5611a8409488275b710d4b683b0181 Mon Sep 17 00:00:00 2001 From: ssakkout Date: Fri, 27 Dec 2024 16:04:02 +0200 Subject: [PATCH 1/4] fixing the cloudflare block in storytel --- audiobookdl/sources/storytel.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/audiobookdl/sources/storytel.py b/audiobookdl/sources/storytel.py index 3633a42..2622060 100644 --- a/audiobookdl/sources/storytel.py +++ b/audiobookdl/sources/storytel.py @@ -170,7 +170,7 @@ def _login(self, url: str, username: str, password: str) -> None: def _do_login(self) -> None: resp = self._session.post( - f"https://www.storytel.com/api/login.action?m=1&token=guestsv&userid=-1&version=24.22&terminal=android&locale=sv&deviceId=995f2562-0e44-4410-b1b9-8d08261f33c4&kidsMode=false", + f"https://www.storytel.com/api/login.action?m=1&token=guestsv&userid=-1&version=24.22&terminal=android&locale=sv&deviceId=42a38c7d-d51a-49f5-a79e-8922e797eccc&kidsMode=false", data={ "uid": self._username, "pwd": self._password, From acb363346a1727a5a8af875b930adf54593a7900 Mon Sep 17 00:00:00 2001 From: ssakkout Date: Wed, 8 Jan 2025 16:24:52 +0200 Subject: [PATCH 2/4] Generate new device id for each request --- audiobookdl/sources/storytel.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/audiobookdl/sources/storytel.py b/audiobookdl/sources/storytel.py index 2622060..15f0a92 100644 --- a/audiobookdl/sources/storytel.py +++ b/audiobookdl/sources/storytel.py @@ -30,6 +30,8 @@ import json import re import os +import uuid + # fmt: off metadata_corrections: Dict[str, Dict[str, Any]] = { @@ -167,20 +169,26 @@ def _login(self, url: str, username: str, password: str) -> None: } ) self._do_login() - + def _do_login(self) -> None: + # Generate a new UUID for each request + generated_device_id = str(uuid.uuid4()) + resp = self._session.post( - f"https://www.storytel.com/api/login.action?m=1&token=guestsv&userid=-1&version=24.22&terminal=android&locale=sv&deviceId=42a38c7d-d51a-49f5-a79e-8922e797eccc&kidsMode=false", + f"https://www.storytel.com/api/login.action?m=1&token=guestsv&userid=-1&version=24.22" + f"&terminal=android&locale=sv&deviceId={generated_device_id}&kidsMode=false", data={ "uid": self._username, "pwd": self._password, }, headers={"content-type": "application/x-www-form-urlencoded"}, ) + if resp.status_code != 200: if resp.status_code == 403: self.check_cloudflare_blocked(resp) raise UserNotAuthorized + user_data = resp.json() jwt = user_data["accountInfo"]["jwt"] self._language = user_data["accountInfo"]["lang"] From 539ea148fad69b2565ce565722cda81b9286a911 Mon Sep 17 00:00:00 2001 From: Frederik Kriewitz Date: Thu, 9 Jan 2025 08:47:24 +0100 Subject: [PATCH 3/4] whitespaces --- audiobookdl/sources/storytel.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/audiobookdl/sources/storytel.py b/audiobookdl/sources/storytel.py index 989a29b..28690f9 100644 --- a/audiobookdl/sources/storytel.py +++ b/audiobookdl/sources/storytel.py @@ -32,7 +32,6 @@ import os import uuid - # fmt: off metadata_corrections: Dict[str, Dict[str, Any]] = { "books": { @@ -169,7 +168,7 @@ def _login(self, url: str, username: str, password: str) -> None: } ) self._do_login() - + def _do_login(self) -> None: # Generate a new UUID for each request generated_device_id = str(uuid.uuid4()) From 22b8414cd56e1157fbb0f9140e2f79e930026ec4 Mon Sep 17 00:00:00 2001 From: ssakkout Date: Mon, 20 Jan 2025 20:25:18 +0200 Subject: [PATCH 4/4] Use curl_cffi for requests Co-authored-by: dev-O-T <73471370+dev-O-T@users.noreply.github.com> --- audiobookdl/__init__.py | 2 +- audiobookdl/sources/source/__init__.py | 25 +++---------------------- audiobookdl/sources/storytel.py | 4 ++-- audiobookdl/utils/audiobook.py | 2 +- pyproject.toml | 1 + 5 files changed, 8 insertions(+), 26 deletions(-) diff --git a/audiobookdl/__init__.py b/audiobookdl/__init__.py index d4705de..b6b636a 100644 --- a/audiobookdl/__init__.py +++ b/audiobookdl/__init__.py @@ -2,4 +2,4 @@ from .sources.source import Source from . import utils -__version__ = "0.7.3" +__version__ = "0.7.7" diff --git a/audiobookdl/sources/source/__init__.py b/audiobookdl/sources/source/__init__.py index c9243bc..3913607 100644 --- a/audiobookdl/sources/source/__init__.py +++ b/audiobookdl/sources/source/__init__.py @@ -5,7 +5,7 @@ from audiobookdl.utils import CustomSSLContextHTTPAdapter # External imports -import requests +import curl_cffi import lxml.html from lxml.cssselect import CSSSelector import re @@ -176,25 +176,6 @@ def find_all_in_page(self, url: str, regex: str, **kwargs) -> list: get_json = networking.get_json get_stream_files = networking.get_stream_files - def create_ssl_context(self, options: Any) -> SSLContext: - try: - ssl_context: SSLContext = urllib3.util.create_urllib3_context() # type: ignore[attr-defined] - - # Workaround for regression in requests version 2.32.3 - # https://github.com/psf/requests/issues/6730 - ssl_context.load_default_certs() - - # Prevent the padding extension from appearing in the TLS ClientHello - # It's used by Cloudflare for bot detection - # See issue #106 - ssl_context.options &= ~(1 << 4) # SSL_OP_TLSEXT_PADDING - return ssl_context - except AttributeError: # AttributeError: module 'urllib3.util' has no attribute 'create_urllib3_context' - raise GenericAudiobookDLException(f"Please update urllib3 to version >= 2 using the command 'pip install -U urllib3'") - - def create_session(self, options: Any) -> requests.Session: - session = requests.Session() - ssl_context: SSLContext = self.create_ssl_context(options) - # session.adapters.pop("https://", None) - session.mount("https://", CustomSSLContextHTTPAdapter(ssl_context)) + def create_session(self, options: Any) -> curl_cffi.requests.Session: + session = curl_cffi.requests.Session() return session diff --git a/audiobookdl/sources/storytel.py b/audiobookdl/sources/storytel.py index 28690f9..f051f65 100644 --- a/audiobookdl/sources/storytel.py +++ b/audiobookdl/sources/storytel.py @@ -1,4 +1,4 @@ -from requests.models import Response +from curl_cffi.requests.models import Response from .source import Source from audiobookdl import ( AudiobookFile, @@ -164,7 +164,7 @@ def _login(self, url: str, username: str, password: str) -> None: self._password = self.encrypt_password(password) self._session.headers.update( { - "User-Agent": "Storytel/24.22 (Android 14; Google Pixel 8 Pro) Release/2288629", + "User-Agent": "Storytel/24.52 (Android 14; Google Pixel 8 Pro) Release/2288809", } ) self._do_login() diff --git a/audiobookdl/utils/audiobook.py b/audiobookdl/utils/audiobook.py index 042fd13..521317f 100644 --- a/audiobookdl/utils/audiobook.py +++ b/audiobookdl/utils/audiobook.py @@ -1,5 +1,5 @@ from datetime import date -import requests +from curl_cffi import requests from typing import Dict, Generic, List, Optional, Union, Sequence, Tuple, TypeVar, Any, MutableMapping import json from attrs import define, Factory diff --git a/pyproject.toml b/pyproject.toml index 69a8cdc..567f23c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -30,6 +30,7 @@ dependencies = [ "rich", "tomli", "urllib3>=2", + "curl_cffi", ] dynamic = ["version"]