Merge branch '4r0n05-master'

config.yaml

@@ -101,9 +101,12 @@
   # Time in seconds to wait before considering a request stuck (0 = infinite)
   chunk_timeout: 10
 
+  # Number of times to retry getting one chunk
+  chunk_retry: 3
+
   # Time in seconds to wait before considering a download stuck (0 = infinite)
-  # Beware: setting this nonzero may cause a dialog backup to be canceled
-  # because of a large media file
+  # This setting is not very useful at the moment, it just skips to the next
+  # dialog if downloading a media file takes too long
   media_timeout: 0
 
 

telegram-history-dump.rb

@@ -54,17 +54,39 @@ def dump_dialog(dialog)
   offset = 0
   keep_dumping = true
   while keep_dumping do
+    cur_offset = offset
     $log.info('Dumping "%s" (range %d-%d)' % [
                 dialog['print_name'],
-                offset + 1,
-                offset + $config['chunk_size']
+                cur_offset + 1,
+                cur_offset + $config['chunk_size']
               ])
     msg_chunk = nil
-    Timeout::timeout($config['chunk_timeout']) do
-      msg_chunk = exec_tg_command('history', dialog['print_name'],
-                                  $config['chunk_size'], offset)
+    retry_count = 0
+    while retry_count <= $config['chunk_retry'] do
+      begin
+        Timeout::timeout($config['chunk_timeout']) do
+          msg_chunk = exec_tg_command('history', dialog['print_name'],
+                                      $config['chunk_size'], cur_offset)
+        end
+        break
+      rescue Timeout::Error
+        if retry_count == $config['chunk_retry']
+          $log.error('Failed to fetch chunk of %d messages from offset %d '\
+                     'after retrying %d times. Dump of "%s" is incomplete.' % [
+                       $config['chunk_size'], cur_offset,
+                       retry_count, dialog['print_name']
+                     ])
+          msg_chunk = []
+          offset += $config['chunk_size']
+          break
+        end
+        $log.error('Timeout, retrying... (%d/%d)' % [
+          retry_count += 1, $config['chunk_retry']
+        ])
+      end
     end
     raise 'Expected array' unless msg_chunk.is_a?(Array)
+
     msg_chunk.reverse_each do |msg|
       dump_msg = true
       unless msg['id']
@@ -103,7 +125,8 @@ def dump_dialog(dialog)
         break
       end
     end
-    keep_dumping = false if msg_chunk.length < $config['chunk_size']
+
+    keep_dumping = false if offset < cur_offset + $config['chunk_size']
     sleep($config['chunk_delay']) if keep_dumping
   end
   state = $dumper.end_dialog(dialog) || {}
@@ -300,7 +323,7 @@ def save_progress
     dump_dialog(dialog)
     save_progress
   rescue Timeout::Error
-    $log.error('Command timeout, skipping to next dialog')
+    $log.error('Unhandled timeout, skipping to next dialog')
     disconnect_socket
   end
 end