Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Enhance timeout check to avoid false positives during processing [of video download to IIAB] #170

Merged
merged 8 commits into from
Jun 4, 2024
Merged

Enhance timeout check to avoid false positives during processing [of video download to IIAB] #170

Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
b72784d
Enhance timeout check to avoid false positives during post processing
deldesir Jun 4, 2024
2fdf1a0
Remove redundant logging
deldesir Jun 4, 2024
aab5bf7
Strengthen timeout check by verifying file existence
deldesir Jun 4, 2024
c47d1e3
Default fragment stuck timeout to 30s again
deldesir Jun 4, 2024
c337940
Remove fail upon timeout reach
deldesir Jun 4, 2024
c934f20
Use absolute imports
deldesir Jun 4, 2024
5674e13
Fix imports
deldesir Jun 4, 2024
de4fdce
Optimize run time
deldesir Jun 4, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
24 changes: 10 additions & 14 deletions cps/tasks/download.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -51,16 +51,17 @@ def run(self, worker_thread):

complete_progress_cycle = 0

last_progress_time = datetime.now()
fragment_stuck_timeout = 30 # seconds
fragment_stuck_time = 0

while p.poll() is None:
self.end_time = datetime.now()
# Check if there's data available to read
rlist, _, _ = select.select([p.stdout], [], [], 0.1)
if rlist:
line = p.stdout.readline()
if line:
if re.search(pattern_success, line):
if re.search(pattern_success, line) or complete_progress_cycle == 4:
# 2024-01-10: 99% (a bit arbitrary) is explained here...
# https://github.com/iiab/calibre-web/pull/88#issuecomment-1885916421
self.progress = 0.99
Expand All @@ -69,21 +70,17 @@ def run(self, worker_thread):
percentage = int(re.search(r'\d+', line).group())
if percentage < 100:
self.message = f"Downloading {self.media_url_link}..."
self.end_time = datetime.now()
self.progress = min(0.99, (complete_progress_cycle + (percentage / 100)) / 4)
if percentage == 100:
complete_progress_cycle += 1
if complete_progress_cycle == 4:
break
last_progress_time = datetime.now()
else:
fragment_stuck_time += 0.1
if fragment_stuck_time >= fragment_stuck_timeout:
log.error("Download appears to be stuck.")
self.record_error_in_database("Download appears to be stuck.")
raise ValueError("Download appears to be stuck.")
elapsed_time = (datetime.now() - last_progress_time).total_seconds()
if elapsed_time >= fragment_stuck_timeout:
self.message = f"Downloading {self.media_url_link}... (This is taking longer than expected)"

sleep(0.1)

p.wait()

# Database operations
Expand Down Expand Up @@ -118,7 +115,7 @@ def run(self, worker_thread):
else:
log.error("Failed to send the requested file to %s", self.original_url)
self.message = f"{self.media_url_link} failed to download: {response.status_code} {response.reason}"

conn.close()

except Exception as e:
Expand All @@ -127,12 +124,11 @@ def run(self, worker_thread):
self.record_error_in_database(str(e))

finally:
self.end_time = datetime.now()
if p.returncode == 0 or self.progress == 1.0:
self.end_time = datetime.now()
self.stat = STAT_FINISH_SUCCESS
log.info("Download task for %s completed successfully", self.media_url)
else:
self.end_time = datetime.now()
self.stat = STAT_FAIL

else:
Expand Down