From 68f00c7d331ffbc2c66b90789d18056e2e39342d Mon Sep 17 00:00:00 2001 From: Arne Weber Date: Sun, 18 Aug 2024 15:42:37 +0200 Subject: [PATCH 1/8] add "Run App (DEBUG+GQL+WS+Log)" debug option --- .vscode/launch.json | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/.vscode/launch.json b/.vscode/launch.json index 67a63140..9edcc18f 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -64,5 +64,14 @@ "console": "integratedTerminal", "justMyCode": false }, + { + "name": "Run App (DEBUG+GQL+WS+Log)", + "type": "python", + "request": "launch", + "program": "main.py", + "args": ["-vvv", "--debug-gql", "--debug-ws", "--log"], + "console": "integratedTerminal", + "justMyCode": false + }, ] } From d2be93594f9e249dd77de6563c46af7a34dcaf85 Mon Sep 17 00:00:00 2001 From: Arne Weber Date: Sun, 18 Aug 2024 15:43:14 +0200 Subject: [PATCH 2/8] gitgnore add wildcard to log.txt --- .gitignore | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 3abda3d9..edd24fdf 100644 --- a/.gitignore +++ b/.gitignore @@ -12,7 +12,7 @@ __pycache__ /env /cache /*.jar -log.txt +/*log.txt /lock.file settings.json #/lang/English.json From 10922078c5ecd0ea9da7b0e7b4e8aa09eb62f122 Mon Sep 17 00:00:00 2001 From: Arne Weber Date: Sun, 18 Aug 2024 15:46:48 +0200 Subject: [PATCH 3/8] Add game name to 0 required minutes error message --- inventory.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/inventory.py b/inventory.py index 3bd090ba..f0654e84 100644 --- a/inventory.py +++ b/inventory.py @@ -220,7 +220,7 @@ def progress(self) -> float: if self.required_minutes: # Quick fix to prevent division by zero crash return self.current_minutes / self.required_minutes else: - self._manager.print(f'!!!required_minutes for "{self.name}" is 0 This could be due to a subscription requirement, tracked in Issue #101!!!') + self._manager.print(f'!!!required_minutes for "{self.name}" from "{self.campaign.game.name}" is 0 This could be due to a subscription requirement, tracked in Issue #101!!!') self.preconditions_met = False return 0 From 159b6186eb8c6ee4bd2db79691e22d297aaa264e Mon Sep 17 00:00:00 2001 From: Arne Weber Date: Sun, 18 Aug 2024 15:56:58 +0200 Subject: [PATCH 4/8] (1)This mostly works, but has a lot of debug stuff --- channel.py | 79 +++++++++++++++++++++++++++++++++++------------------- twitch.py | 16 +++++++---- 2 files changed, 63 insertions(+), 32 deletions(-) diff --git a/channel.py b/channel.py index 7aa594bc..37393553 100644 --- a/channel.py +++ b/channel.py @@ -377,7 +377,7 @@ def _payload(self) -> JsonType: ] return {"data": (b64encode(json_minify(payload).encode("utf8"))).decode("utf8")} - async def send_watch(self) -> bool: + async def send_watch(self) -> tuple[bool, bool]: """ Start of fix for 2024/5 API Change """ @@ -390,41 +390,66 @@ async def send_watch(self) -> bool: signature: JsonType | None = response["data"]['streamPlaybackAccessToken']["signature"] value: JsonType | None = response["data"]['streamPlaybackAccessToken']["value"] if not signature or not value: - return False + return False, False RequestBroadcastQualitiesURL = f"https://usher.ttvnw.net/api/channel/hls/{self._login}.m3u8?sig={signature}&token={value}" try: - async with self._twitch.request( # Gets list of streams + async with self._twitch.request( # Gets list of m3u8 playlists "GET", RequestBroadcastQualitiesURL ) as response1: - BroadcastQualities = await response1.text() + BroadcastQualitiesM3U = await response1.text() except RequestException: - return False + logger.error(f"Failed to recieve list of m3u8 playlists.") + return False, False + logger.log(CALL,f"BroadcastQualities: {BroadcastQualitiesM3U}") - BroadcastLowestQualityURL = BroadcastQualities.split("\n")[-1] # Just takes the last line, this could probably be handled better in the future - if not validators.url(BroadcastLowestQualityURL): - return False - - try: - async with self._twitch.request( # Gets actual streams - "GET", BroadcastLowestQualityURL - ) as response2: - StreamURLList = await response2.text() - except RequestException: - return False - - StreamLowestQualityURL = StreamURLList.split("\n")[-2] # For whatever reason this includes a blank line at the end, this should probably be handled better in the future - if not validators.url(StreamLowestQualityURL): - return False + BroadcastQualitiesM3U = BroadcastQualitiesM3U.split("\n") + BroadcastQualitiesList = [] + for i in range(int(len(BroadcastQualitiesM3U)/3)): # gets all m3u8 playlists + BroadcastQualitiesList.append(BroadcastQualitiesM3U[4+3*i]) + + if not all(validators.url(url) for url in BroadcastQualitiesList): + logger.error(f"Couldn't parse list of m3u8 playlists.") + return False, False + logger.log(CALL,f"BroadcastQualitiesList: {BroadcastQualitiesList}") + + retries = -1 + for BroadcastQuality in BroadcastQualitiesList: + retries = retries + 1 + try: + async with self._twitch.request( # Gets actual streams + "GET", BroadcastQuality, return_error = True + ) as response2: + if response2.status == 200: + StreamURLList = await response2.text() + else: + logger.error(f"Request for streams from m3u8 returned: {response2}") + continue + except RequestException: + logger.error(f"Failed to recieve list of streams.") + return False, False + + StreamURL = StreamURLList.split("\n")[-2] # For whatever reason this includes a blank line at the end, this should probably be handled better in the future + if not validators.url(StreamURL): + logger.error(f"Failed to parse streamURL.") + return False, False - try: - async with self._twitch.request( # Downloads the stream - "HEAD", StreamLowestQualityURL - ) as response3: # I lied, well idk, but this code doesn't listen for the actual video data - return response3.status == 200 - except RequestException: - return False + try: + async with self._twitch.request( # The HEAD request is enough to advance drops + "HEAD", StreamURL, return_error = True + ) as response3: + if response3.status == 200: + logger.log(CALL,f"Successfully watched after {retries} retries.") + return True, False + else: + logger.error(f"Request for stream HEAD returned: {response2}") + except RequestException: + logger.error(f"Failed to recieve list of streams.") + return False, False + await asyncio.sleep(1) # Wait a second to not spam twitch API + logger.error(f"Failed to watch last streamURL for all of {len(BroadcastQualitiesList)} Broadcast qualities.") + return False, True """ End of fix for 2024/5 API Change. Old code below. diff --git a/twitch.py b/twitch.py index 90eefca6..ced9ad81 100644 --- a/twitch.py +++ b/twitch.py @@ -1045,12 +1045,18 @@ async def _watch_sleep(self, delay: float) -> None: async def _watch_loop(self) -> NoReturn: interval: float = WATCH_INTERVAL.total_seconds() while True: + logger.log(CALL,f"beginning of watch loop") channel: Channel = await self.watching_channel.get() - succeeded: bool = await channel.send_watch() + logger.log(CALL,f"about to send watch") + succeeded, repeat_now = await channel.send_watch() + logger.log(CALL,f"returned watch, succeeded: {succeeded}, repeat_new: {repeat_now}") if not succeeded: # this usually means the campaign expired in the middle of mining + # or the m3u8 playlists all returned a 500 Internal server error # NOTE: the maintenance task should switch the channel right after this happens - await self._watch_sleep(interval) + if not repeat_now: + await self._watch_sleep(interval) + logger.log(CALL,f"send_watch returned false") continue last_watch = time() self._drop_update = asyncio.Future() @@ -1480,7 +1486,7 @@ async def get_auth(self) -> _AuthState: @asynccontextmanager async def request( - self, method: str, url: URL | str, *, invalidate_after: datetime | None = None, **kwargs + self, method: str, url: URL | str, *, invalidate_after: datetime | None = None, return_error: bool = False, **kwargs ) -> abc.AsyncIterator[aiohttp.ClientResponse]: session = await self.get_session() method = method.upper() @@ -1505,12 +1511,12 @@ async def request( ) assert response is not None logger.debug(f"Response: {response.status}: {response}") - if response.status < 500: + if response.status < 500 or return_error: # pre-read the response to avoid getting errors outside of the context manager raw_response = await response.read() # noqa yield response return - self.print(_("error", "site_down").format(seconds=round(delay))) + self.print(_("error", "site_down").format(seconds=round(delay)) + f"\nResponse: {response}" + f"\nStatus: {response.status}") except aiohttp.ClientConnectorCertificateError: # type: ignore[unused-ignore] # for a case where SSL verification fails raise From 744e434bb7a830c3a9494f2be09a52bf325ecd5e Mon Sep 17 00:00:00 2001 From: Arne Weber Date: Sun, 18 Aug 2024 17:57:31 +0200 Subject: [PATCH 5/8] minor logging changes --- channel.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/channel.py b/channel.py index 37393553..618f4dac 100644 --- a/channel.py +++ b/channel.py @@ -424,7 +424,7 @@ async def send_watch(self) -> tuple[bool, bool]: if response2.status == 200: StreamURLList = await response2.text() else: - logger.error(f"Request for streams from m3u8 returned: {response2}") + logger.log(CALL,f"Request for streams from m3u8 returned: {response2}") continue except RequestException: logger.error(f"Failed to recieve list of streams.") @@ -448,7 +448,7 @@ async def send_watch(self) -> tuple[bool, bool]: logger.error(f"Failed to recieve list of streams.") return False, False await asyncio.sleep(1) # Wait a second to not spam twitch API - logger.error(f"Failed to watch last streamURL for all of {len(BroadcastQualitiesList)} Broadcast qualities.") + logger.error(f"Failed to watch all of {len(BroadcastQualitiesList)} Broadcast qualities. Can be ignored if occuring ~15x/hour.") return False, True """ End of fix for 2024/5 API Change. From 74de20468c206287f5691a027b3f48940712786a Mon Sep 17 00:00:00 2001 From: Arne Weber Date: Tue, 27 Aug 2024 20:10:00 +0200 Subject: [PATCH 6/8] Changed 0 Required minutes message for clarity --- inventory.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/inventory.py b/inventory.py index f0654e84..6e7ed557 100644 --- a/inventory.py +++ b/inventory.py @@ -220,7 +220,7 @@ def progress(self) -> float: if self.required_minutes: # Quick fix to prevent division by zero crash return self.current_minutes / self.required_minutes else: - self._manager.print(f'!!!required_minutes for "{self.name}" from "{self.campaign.game.name}" is 0 This could be due to a subscription requirement, tracked in Issue #101!!!') + self._manager.print(f'Required_minutes for "{self.name}" from "{self.campaign.game.name}" is 0. This could be due to a subscription requirement, tracked in Issue #101. Take a look at the drop, but this can likely be ignored.') self.preconditions_met = False return 0 From 9c2ec99ac1a812b0f46f152ccb16c1391db4ee61 Mon Sep 17 00:00:00 2001 From: Arne Weber Date: Tue, 27 Aug 2024 20:15:58 +0200 Subject: [PATCH 7/8] Cleaned up error and debug logging --- channel.py | 4 +--- twitch.py | 3 --- 2 files changed, 1 insertion(+), 6 deletions(-) diff --git a/channel.py b/channel.py index 618f4dac..54c0bff8 100644 --- a/channel.py +++ b/channel.py @@ -402,7 +402,6 @@ async def send_watch(self) -> tuple[bool, bool]: except RequestException: logger.error(f"Failed to recieve list of m3u8 playlists.") return False, False - logger.log(CALL,f"BroadcastQualities: {BroadcastQualitiesM3U}") BroadcastQualitiesM3U = BroadcastQualitiesM3U.split("\n") BroadcastQualitiesList = [] @@ -412,7 +411,6 @@ async def send_watch(self) -> tuple[bool, bool]: if not all(validators.url(url) for url in BroadcastQualitiesList): logger.error(f"Couldn't parse list of m3u8 playlists.") return False, False - logger.log(CALL,f"BroadcastQualitiesList: {BroadcastQualitiesList}") retries = -1 for BroadcastQuality in BroadcastQualitiesList: @@ -448,7 +446,7 @@ async def send_watch(self) -> tuple[bool, bool]: logger.error(f"Failed to recieve list of streams.") return False, False await asyncio.sleep(1) # Wait a second to not spam twitch API - logger.error(f"Failed to watch all of {len(BroadcastQualitiesList)} Broadcast qualities. Can be ignored if occuring ~15x/hour.") + logger.error(f"Failed to watch all of {len(BroadcastQualitiesList)} Broadcast qualities. Can be ignored if occuring up to ~15x/hour.") return False, True """ End of fix for 2024/5 API Change. diff --git a/twitch.py b/twitch.py index ced9ad81..a7d3778e 100644 --- a/twitch.py +++ b/twitch.py @@ -1045,9 +1045,7 @@ async def _watch_sleep(self, delay: float) -> None: async def _watch_loop(self) -> NoReturn: interval: float = WATCH_INTERVAL.total_seconds() while True: - logger.log(CALL,f"beginning of watch loop") channel: Channel = await self.watching_channel.get() - logger.log(CALL,f"about to send watch") succeeded, repeat_now = await channel.send_watch() logger.log(CALL,f"returned watch, succeeded: {succeeded}, repeat_new: {repeat_now}") if not succeeded: @@ -1056,7 +1054,6 @@ async def _watch_loop(self) -> NoReturn: # NOTE: the maintenance task should switch the channel right after this happens if not repeat_now: await self._watch_sleep(interval) - logger.log(CALL,f"send_watch returned false") continue last_watch = time() self._drop_update = asyncio.Future() From 3a8e67acc9d1bf826c132f88dd55998ca3b1fe3a Mon Sep 17 00:00:00 2001 From: Arne Weber Date: Fri, 30 Aug 2024 22:44:33 +0200 Subject: [PATCH 8/8] patch notes for v15.9.0 --- patch_notes.txt | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/patch_notes.txt b/patch_notes.txt index 01602845..538425f0 100644 --- a/patch_notes.txt +++ b/patch_notes.txt @@ -1,3 +1,14 @@ +## v15.9.0 + +18.8.2024 +- Added game name to `0 required minutes` error message + +27.8.2024 +- Clarified that 0 required minutes error is not critical +- Fixed "Twitch is down, retrying" for my case. It seems like some might still experience the issue. Tracked in #172 + + + ## v15.8.2 13.8.2024