diff --git a/config.yaml b/config.yaml index 80341ac..7b98bef 100644 --- a/config.yaml +++ b/config.yaml @@ -101,9 +101,12 @@ # Time in seconds to wait before considering a request stuck (0 = infinite) chunk_timeout: 10 + # Number of times to retry getting one chunk + chunk_retry: 3 + # Time in seconds to wait before considering a download stuck (0 = infinite) - # Beware: setting this nonzero may cause a dialog backup to be canceled - # because of a large media file + # This setting is not very useful at the moment, it just skips to the next + # dialog if downloading a media file takes too long media_timeout: 0 diff --git a/telegram-history-dump.rb b/telegram-history-dump.rb index 9e5511e..cfb2fc2 100755 --- a/telegram-history-dump.rb +++ b/telegram-history-dump.rb @@ -54,17 +54,39 @@ def dump_dialog(dialog) offset = 0 keep_dumping = true while keep_dumping do + cur_offset = offset $log.info('Dumping "%s" (range %d-%d)' % [ dialog['print_name'], - offset + 1, - offset + $config['chunk_size'] + cur_offset + 1, + cur_offset + $config['chunk_size'] ]) msg_chunk = nil - Timeout::timeout($config['chunk_timeout']) do - msg_chunk = exec_tg_command('history', dialog['print_name'], - $config['chunk_size'], offset) + retry_count = 0 + while retry_count <= $config['chunk_retry'] do + begin + Timeout::timeout($config['chunk_timeout']) do + msg_chunk = exec_tg_command('history', dialog['print_name'], + $config['chunk_size'], cur_offset) + end + break + rescue Timeout::Error + if retry_count == $config['chunk_retry'] + $log.error('Failed to fetch chunk of %d messages from offset %d '\ + 'after retrying %d times. Dump of "%s" is incomplete.' % [ + $config['chunk_size'], cur_offset, + retry_count, dialog['print_name'] + ]) + msg_chunk = [] + offset += $config['chunk_size'] + break + end + $log.error('Timeout, retrying... (%d/%d)' % [ + retry_count += 1, $config['chunk_retry'] + ]) + end end raise 'Expected array' unless msg_chunk.is_a?(Array) + msg_chunk.reverse_each do |msg| dump_msg = true unless msg['id'] @@ -103,7 +125,8 @@ def dump_dialog(dialog) break end end - keep_dumping = false if msg_chunk.length < $config['chunk_size'] + + keep_dumping = false if offset < cur_offset + $config['chunk_size'] sleep($config['chunk_delay']) if keep_dumping end state = $dumper.end_dialog(dialog) || {} @@ -300,7 +323,7 @@ def save_progress dump_dialog(dialog) save_progress rescue Timeout::Error - $log.error('Command timeout, skipping to next dialog') + $log.error('Unhandled timeout, skipping to next dialog') disconnect_socket end end