Pull support, default SSH cipher update and a few documentation fixes

README.md

@@ -12,14 +12,16 @@ blocksync.py is also a workaround for a limitation when using [rsync](https://rs
 * SSH client on source server
 * SSH server on destination server, with root permissions (directly using root login or using sudo) if syncing to a device file
 * Python on both source and destination server
-* blocksync.py in home directory of destination server (executable)
 
 ## Usage
 Please make sure that the source file isn't changed during sync, blocksync.py will **not** notice any changes made at file positions which were already copied. You may want to boot a live linux ([grml](https://grml.org/), [knoppix](http://www.knoppix.org), [systemrescuecd](http://www.system-rescue-cd.org) etc.) if you want to sync the system drives from a running machine.
 
 ### Synchronize to a file on remote server
 `root@source# python blocksync.py /dev/source/file [email protected] /path/to/destination/file`
 
+### Synchronize from a file on remote server
+`root@source# python blocksync.py --pull /dev/destination/file [email protected] /path/to/source/file`
+
 ### Synchronize to a local file
 `root@source# python blocksync.py /dev/source/file localhost /path/to/destination/file`
 

blocksync.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3
 """
 Synchronise block devices over the network
 
@@ -16,6 +16,7 @@
 * Make sure your local user can ssh to the remote host (use -i for a SSH key)
 * Invoke:
     python blocksync.py /dev/source [user@]remotehost [/dev/dest]
+    python blocksync.py --pull /dev/dest [user@]remotehost [/dev/source]
 
 * Specify localhost for local usage:
     python blocksync.py /dev/source localhost /dev/dest
@@ -30,6 +31,7 @@
 import subprocess
 import time
 from datetime import timedelta
+import shlex
 
 SAME = b"0"
 DIFF = b"1"
@@ -55,7 +57,7 @@
     USE_NOREUSE = USE_DONTNEED = False
 
 def do_create(f, size):
-    f = open(f, 'a', 0)
+    f = open(f, 'wb', 0)
     f.truncate(size)
     f.close()
 
@@ -106,8 +108,19 @@ def server(dev, deleteonexit, options):
     if size > 0:
         do_create(dev, size)
 
-    print(dev, blocksize)
-    f, size = do_open(dev, 'rb+')
+    print(dev)
+    print(blocksize)
+
+    try:
+        if options.pull:
+            f, size = do_open(dev, 'rb')
+        else:
+            f, size = do_open(dev, 'rb+')
+    except:
+        # Unable to access the device
+        print(-1)
+        return
+
     print(size)
     sys.stdout.flush()
 
@@ -130,12 +143,16 @@ def server(dev, deleteonexit, options):
         stdout.flush()
         res = stdin.read(COMPLEN)
         if res == DIFF:
-            newblock = stdin.read(blocksize)
-            newblocklen = len(newblock)
-            f.seek(-newblocklen, 1)
-            f.write(newblock)
-            if USE_DONTNEED:
-                fadvise(f, f.tell() - newblocklen, newblocklen, POSIX_FADV_DONTNEED)
+            if options.pull:
+                stdout.write(block)
+                stdout.flush()
+            else:
+                newblock = stdin.read(blocksize)
+                newblocklen = len(newblock)
+                f.seek(-newblocklen, 1)
+                f.write(newblock)
+                if USE_DONTNEED:
+                    fadvise(f, f.tell() - newblocklen, newblocklen, POSIX_FADV_DONTNEED)
         if i == maxblock:
             break
 
@@ -184,11 +201,18 @@ def sync(workerid, srcdev, dsthost, dstdev, options):
         fadv = "None"
     print("[worker %d] Local fadvise: %s" % (workerid, fadv), file = options.outfile)
 
-    try:
-        f, size = do_open(srcdev, 'rb')
-    except Exception as e:
-        print("[worker %d] Error accessing source device! %s" % (workerid, e), file = options.outfile)
-        sys.exit(1)
+    if options.pull:
+        try:
+            f, size = do_open(srcdev, 'rb+')
+        except Exception as e:
+            print("[worker %d] Error accessing destination device! %s" % (workerid, e), file = options.outfile)
+            sys.exit(1)
+    else:
+        try:
+            f, size = do_open(srcdev, 'rb')
+        except Exception as e:
+            print("[worker %d] Error accessing source device! %s" % (workerid, e), file = options.outfile)
+            sys.exit(1)
 
     chunksize = int(size / options.workers)
     startpos = workerid * chunksize
@@ -237,12 +261,15 @@ def sync(workerid, srcdev, dsthost, dstdev, options):
         servercmd = 'tmpserver'
         remotescript = copy_self(workerid, cmd)
 
-    cmd += [options.interpreter, remotescript, servercmd, dstdev, '-b', str(blocksize)]
+    cmd += [options.interpreter, remotescript, servercmd, shlex.quote(dstdev), '-b', str(blocksize)]
 
     cmd += ['-d', str(options.fadvise), '-1', options.hash]
     if options.addhash:
         cmd += ['-2', options.addhash]
 
+    if options.pull:
+        cmd += ['--pull']
+
     print("[worker %d] Running: %s" % (workerid, " ".join(cmd[2 if options.passenv and (dsthost != 'localhost') else 0:])), file = options.outfile)
 
     p = subprocess.Popen(cmd, bufsize=0, stdin=subprocess.PIPE, stdout=subprocess.PIPE, close_fds=True)
@@ -257,16 +284,16 @@ def sync(workerid, srcdev, dsthost, dstdev, options):
     fadv = p_out.readline().decode('UTF-8').strip()
     print("[worker %d] Remote fadvise: %s" % (workerid, fadv), file = options.outfile)
 
-    p_in.write(bytes(("%d\n" % (size if options.createdest else 0)).encode("UTF-8")))
+    p_in.write(bytes(("%d\n" % (size if options.createdest and not dryrun else 0)).encode("UTF-8")))
     p_in.flush()
 
-    line = p_out.readline().decode('UTF-8')
+    a = p_out.readline().decode('UTF-8').strip()
+    b = p_out.readline().decode('UTF-8').strip()
     p.poll()
     if p.returncode is not None:
       print("[worker %d] Failed creating destination file on the remote host!" % workerid, file = options.outfile)
       sys.exit(1)
 
-    a, b = line.split()
     if a != dstdev:
         print("[worker %d] Dest device (%s) doesn't match with the remote host (%s)!" % (workerid, dstdev, a), file = options.outfile)
         sys.exit(1)
@@ -280,7 +307,10 @@ def sync(workerid, srcdev, dsthost, dstdev, options):
         print("[worker %d] Error accessing device on remote host!" % workerid, file = options.outfile)
         sys.exit(1)
     remote_size = int(line)
-    if size > remote_size:
+    if remote_size < 0:
+        print("[worker %d] Remote device doesn't exists!" % workerid, file = options.outfile)
+        sys.exit(1)
+    elif size > remote_size:
         print("[worker %d] Source device size (%d) doesn't fit into remote device size (%d)!" % (workerid, size, remote_size), file = options.outfile)
         sys.exit(1)
     elif size < remote_size:
@@ -317,8 +347,16 @@ def sync(workerid, srcdev, dsthost, dstdev, options):
             else:
                 p_in.write(DIFF)
                 p_in.flush()
-                p_in.write(l_block)
-                p_in.flush()
+                if options.pull:
+                    newblock = p_out.read(blocksize)
+                    newblocklen = len(newblock)
+                    f.seek(-newblocklen, 1)
+                    f.write(newblock)
+                    if USE_DONTNEED:
+                        fadvise(f, f.tell() - newblocklen, newblocklen, POSIX_FADV_DONTNEED)
+                else:
+                    p_in.write(l_block)
+                    p_in.flush()
 
         if pause_ms:
             time.sleep(pause_ms)
@@ -349,25 +387,26 @@ def sync(workerid, srcdev, dsthost, dstdev, options):
     from optparse import OptionParser, SUPPRESS_HELP
     parser = OptionParser(usage = "%prog [options] /dev/source [user@]remotehost [/dev/dest]")
     parser.add_option("-w", "--workers", dest = "workers", type = "int", help = "number of workers to fork (defaults to 1)", default = 1)
-    parser.add_option("-l", "--splay", dest = "splay", type = "int", help = "sleep between creating workers (ms, defaults to 0)", default = 250)
+    parser.add_option("-l", "--splay", dest = "splay", type = "int", help = "sleep between creating workers (ms, defaults to 250)", default = 250)
     parser.add_option("-b", "--blocksize", dest = "blocksize", type = "int", help = "block size (bytes, defaults to 1MB)", default = 1024 * 1024)
     parser.add_option("-1", "--hash", dest = "hash", help = "hash used for block comparison (defaults to \"sha512\")", default = "sha512")
     parser.add_option("-2", "--additionalhash", dest = "addhash", help = "second hash used for extra comparison (default is none)")
     parser.add_option("-d", "--fadvise", dest = "fadvise", type = "int", help = "lower cache pressure by using posix_fadivse (requires Python 3 or python-fadvise; 0 = off, 1 = local on, 2 = remote on, 3 = both on; defaults to 3)", default = 3)
     parser.add_option("-p", "--pause", dest = "pause", type="int", help = "pause between processing blocks, reduces system load (ms, defaults to 0)", default = 0)
-    parser.add_option("-c", "--cipher", dest = "cipher", help = "cipher specification for SSH (defaults to blowfish)", default = "blowfish")
-    parser.add_option("-C", "--compress", dest = "compress", action = "store_true", help = "enable compression over SSH (defaults to on)", default = True)
+    parser.add_option("-c", "--cipher", dest = "cipher", help = "cipher specification for SSH (defaults to aes256-ctr)", default = "aes256-ctr")
+    parser.add_option("-N", "--nocompress", dest = "compress", action = "store_false", help = "diable compression over SSH (defaults to on)", default = True)
     parser.add_option("-i", "--id", dest = "keyfile", help = "SSH public key file")
     parser.add_option("-P", "--pass", dest = "passenv", help = "environment variable containing SSH password (requires sshpass)")
     parser.add_option("-s", "--sudo", dest = "sudo", action = "store_true", help = "use sudo on the remote end (defaults to off)", default = False)
     parser.add_option("-x", "--extraparams", dest = "sshparams", help = "additional parameters to pass to SSH")
     parser.add_option("-n", "--dryrun", dest = "dryrun", action = "store_true", help = "do a dry run (don't write anything, just report differences)", default = False)
     parser.add_option("-T", "--createdest", dest = "createdest", action = "store_true", help = "create destination file using truncate(2). Should be safe for subsequent syncs as truncate only modifies the file when the size differs", default = False)
     parser.add_option("-S", "--script", dest = "script", help = "location of script on remote host (otherwise current script is sent over)")
-    parser.add_option("-I", "--interpreter", dest = "interpreter", help = "[full path to] interpreter used to invoke remote server (defaults to python2)", default = "python2")
+    parser.add_option("-I", "--interpreter", dest = "interpreter", help = "[full path to] interpreter used to invoke remote server (defaults to python3)", default = "python3")
     parser.add_option("-t", "--interval", dest = "interval", type = "int", help = "interval between stats output (seconds, defaults to 1)", default = 1)
     parser.add_option("-o", "--output", dest = "outfile", help = "send output to file instead of console")
-    parser.add_option("-f", "--force", dest = "force", action= "store_true", help = "force sync and DO NOT ask for confirmation if the destination file already exists")
+    parser.add_option("-f", "--force", dest = "force", action = "store_true", help = "force sync and DO NOT ask for confirmation if the destination file already exists")
+    parser.add_option("--pull", dest = "pull", action = "store_true", help = "synchronize changes from the remote host to this host")
     (options, args) = parser.parse_args()
 
     if len(args) < 2: