From 4114a0041d2ae5a11d89a385545bba92958ba592 Mon Sep 17 00:00:00 2001 From: Jeff Fisher Date: Wed, 2 Dec 2020 10:06:01 -0600 Subject: [PATCH 1/5] blocksync.py copies itself to the destination server --- README.md | 1 - 1 file changed, 1 deletion(-) diff --git a/README.md b/README.md index 56f1ec4..9751b09 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,6 @@ blocksync.py is also a workaround for a limitation when using [rsync](https://rs * SSH client on source server * SSH server on destination server, with root permissions (directly using root login or using sudo) if syncing to a device file * Python on both source and destination server -* blocksync.py in home directory of destination server (executable) ## Usage Please make sure that the source file isn't changed during sync, blocksync.py will **not** notice any changes made at file positions which were already copied. You may want to boot a live linux ([grml](https://grml.org/), [knoppix](http://www.knoppix.org), [systemrescuecd](http://www.system-rescue-cd.org) etc.) if you want to sync the system drives from a running machine. From b5661f4d7149df60747008aea25f6a6a1aa3a9b3 Mon Sep 17 00:00:00 2001 From: Jeff Fisher Date: Fri, 4 Dec 2020 11:10:39 -0600 Subject: [PATCH 2/5] add support for pulling changes from the remote side --- README.md | 3 +++ blocksync.py | 57 ++++++++++++++++++++++++++++++++++++++-------------- 2 files changed, 45 insertions(+), 15 deletions(-) mode change 100644 => 100755 blocksync.py diff --git a/README.md b/README.md index 9751b09..9821d30 100644 --- a/README.md +++ b/README.md @@ -19,6 +19,9 @@ Please make sure that the source file isn't changed during sync, blocksync.py wi ### Synchronize to a file on remote server `root@source# python blocksync.py /dev/source/file user@destination.example.com /path/to/destination/file` +### Synchronize from a file on remote server +`root@source# python blocksync.py --pull /dev/destination/file user@destination.example.com /path/to/source/file` + ### Synchronize to a local file `root@source# python blocksync.py /dev/source/file localhost /path/to/destination/file` diff --git a/blocksync.py b/blocksync.py old mode 100644 new mode 100755 index 1e32dce..4e19ecd --- a/blocksync.py +++ b/blocksync.py @@ -16,6 +16,7 @@ * Make sure your local user can ssh to the remote host (use -i for a SSH key) * Invoke: python blocksync.py /dev/source [user@]remotehost [/dev/dest] + python blocksync.py --pull /dev/dest [user@]remotehost [/dev/source] * Specify localhost for local usage: python blocksync.py /dev/source localhost /dev/dest @@ -107,7 +108,10 @@ def server(dev, deleteonexit, options): do_create(dev, size) print(dev, blocksize) - f, size = do_open(dev, 'rb+') + if options.pull: + f, size = do_open(dev, 'rb') + else: + f, size = do_open(dev, 'rb+') print(size) sys.stdout.flush() @@ -130,12 +134,16 @@ def server(dev, deleteonexit, options): stdout.flush() res = stdin.read(COMPLEN) if res == DIFF: - newblock = stdin.read(blocksize) - newblocklen = len(newblock) - f.seek(-newblocklen, 1) - f.write(newblock) - if USE_DONTNEED: - fadvise(f, f.tell() - newblocklen, newblocklen, POSIX_FADV_DONTNEED) + if options.pull: + stdout.write(block) + stdout.flush() + else: + newblock = stdin.read(blocksize) + newblocklen = len(newblock) + f.seek(-newblocklen, 1) + f.write(newblock) + if USE_DONTNEED: + fadvise(f, f.tell() - newblocklen, newblocklen, POSIX_FADV_DONTNEED) if i == maxblock: break @@ -184,11 +192,18 @@ def sync(workerid, srcdev, dsthost, dstdev, options): fadv = "None" print("[worker %d] Local fadvise: %s" % (workerid, fadv), file = options.outfile) - try: - f, size = do_open(srcdev, 'rb') - except Exception as e: - print("[worker %d] Error accessing source device! %s" % (workerid, e), file = options.outfile) - sys.exit(1) + if options.pull: + try: + f, size = do_open(srcdev, 'rb+') + except Exception as e: + print("[worker %d] Error accessing destination device! %s" % (workerid, e), file = options.outfile) + sys.exit(1) + else: + try: + f, size = do_open(srcdev, 'rb') + except Exception as e: + print("[worker %d] Error accessing source device! %s" % (workerid, e), file = options.outfile) + sys.exit(1) chunksize = int(size / options.workers) startpos = workerid * chunksize @@ -243,6 +258,9 @@ def sync(workerid, srcdev, dsthost, dstdev, options): if options.addhash: cmd += ['-2', options.addhash] + if options.pull: + cmd += ['--pull'] + print("[worker %d] Running: %s" % (workerid, " ".join(cmd[2 if options.passenv and (dsthost != 'localhost') else 0:])), file = options.outfile) p = subprocess.Popen(cmd, bufsize=0, stdin=subprocess.PIPE, stdout=subprocess.PIPE, close_fds=True) @@ -317,8 +335,16 @@ def sync(workerid, srcdev, dsthost, dstdev, options): else: p_in.write(DIFF) p_in.flush() - p_in.write(l_block) - p_in.flush() + if options.pull: + newblock = p_out.read(blocksize) + newblocklen = len(newblock) + f.seek(-newblocklen, 1) + f.write(newblock) + if USE_DONTNEED: + fadvise(f, f.tell() - newblocklen, newblocklen, POSIX_FADV_DONTNEED) + else: + p_in.write(l_block) + p_in.flush() if pause_ms: time.sleep(pause_ms) @@ -367,7 +393,8 @@ def sync(workerid, srcdev, dsthost, dstdev, options): parser.add_option("-I", "--interpreter", dest = "interpreter", help = "[full path to] interpreter used to invoke remote server (defaults to python2)", default = "python2") parser.add_option("-t", "--interval", dest = "interval", type = "int", help = "interval between stats output (seconds, defaults to 1)", default = 1) parser.add_option("-o", "--output", dest = "outfile", help = "send output to file instead of console") - parser.add_option("-f", "--force", dest = "force", action= "store_true", help = "force sync and DO NOT ask for confirmation if the destination file already exists") + parser.add_option("-f", "--force", dest = "force", action = "store_true", help = "force sync and DO NOT ask for confirmation if the destination file already exists") + parser.add_option("--pull", dest = "pull", action = "store_true", help = "synchronize changes from the remote host to this host") (options, args) = parser.parse_args() if len(args) < 2: From e1ab34b723a9d539d3a2d89b2ad02b54097308d5 Mon Sep 17 00:00:00 2001 From: Jeff Fisher Date: Fri, 4 Dec 2020 11:39:06 -0600 Subject: [PATCH 3/5] default was changed to 250 in pull request #8 but the documentation wasn't updated --- blocksync.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/blocksync.py b/blocksync.py index 4e19ecd..0a27d68 100755 --- a/blocksync.py +++ b/blocksync.py @@ -375,7 +375,7 @@ def sync(workerid, srcdev, dsthost, dstdev, options): from optparse import OptionParser, SUPPRESS_HELP parser = OptionParser(usage = "%prog [options] /dev/source [user@]remotehost [/dev/dest]") parser.add_option("-w", "--workers", dest = "workers", type = "int", help = "number of workers to fork (defaults to 1)", default = 1) - parser.add_option("-l", "--splay", dest = "splay", type = "int", help = "sleep between creating workers (ms, defaults to 0)", default = 250) + parser.add_option("-l", "--splay", dest = "splay", type = "int", help = "sleep between creating workers (ms, defaults to 250)", default = 250) parser.add_option("-b", "--blocksize", dest = "blocksize", type = "int", help = "block size (bytes, defaults to 1MB)", default = 1024 * 1024) parser.add_option("-1", "--hash", dest = "hash", help = "hash used for block comparison (defaults to \"sha512\")", default = "sha512") parser.add_option("-2", "--additionalhash", dest = "addhash", help = "second hash used for extra comparison (default is none)") From 8ea65cebb075ccfa63273b36627c24ea52c41a08 Mon Sep 17 00:00:00 2001 From: Jeff Fisher Date: Fri, 4 Dec 2020 11:44:43 -0600 Subject: [PATCH 4/5] change the default SSH cipher to aes256-ctr as blowfish is not well supported anymore. --- blocksync.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/blocksync.py b/blocksync.py index 0a27d68..e85487d 100755 --- a/blocksync.py +++ b/blocksync.py @@ -381,7 +381,7 @@ def sync(workerid, srcdev, dsthost, dstdev, options): parser.add_option("-2", "--additionalhash", dest = "addhash", help = "second hash used for extra comparison (default is none)") parser.add_option("-d", "--fadvise", dest = "fadvise", type = "int", help = "lower cache pressure by using posix_fadivse (requires Python 3 or python-fadvise; 0 = off, 1 = local on, 2 = remote on, 3 = both on; defaults to 3)", default = 3) parser.add_option("-p", "--pause", dest = "pause", type="int", help = "pause between processing blocks, reduces system load (ms, defaults to 0)", default = 0) - parser.add_option("-c", "--cipher", dest = "cipher", help = "cipher specification for SSH (defaults to blowfish)", default = "blowfish") + parser.add_option("-c", "--cipher", dest = "cipher", help = "cipher specification for SSH (defaults to aes256-ctr)", default = "aes256-ctr") parser.add_option("-C", "--compress", dest = "compress", action = "store_true", help = "enable compression over SSH (defaults to on)", default = True) parser.add_option("-i", "--id", dest = "keyfile", help = "SSH public key file") parser.add_option("-P", "--pass", dest = "passenv", help = "environment variable containing SSH password (requires sshpass)") From 75764aead288a59d8ee7d05aa90bae81b0299199 Mon Sep 17 00:00:00 2001 From: Jeff Fisher Date: Sun, 23 Jul 2023 11:24:39 -0600 Subject: [PATCH 5/5] Since theraser's blocksync seems to be dead, I've taken a pending PR (https://github.com/theraser/blocksync/pull/22) and merged in the pieces that don't conflict with my own pull support. I didn't use their wrap function and instead am using shlex.quote. --- blocksync.py | 40 ++++++++++++++++++++++++++-------------- 1 file changed, 26 insertions(+), 14 deletions(-) diff --git a/blocksync.py b/blocksync.py index e85487d..e7e7d36 100755 --- a/blocksync.py +++ b/blocksync.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python2 +#!/usr/bin/env python3 """ Synchronise block devices over the network @@ -31,6 +31,7 @@ import subprocess import time from datetime import timedelta +import shlex SAME = b"0" DIFF = b"1" @@ -56,7 +57,7 @@ USE_NOREUSE = USE_DONTNEED = False def do_create(f, size): - f = open(f, 'a', 0) + f = open(f, 'wb', 0) f.truncate(size) f.close() @@ -107,11 +108,19 @@ def server(dev, deleteonexit, options): if size > 0: do_create(dev, size) - print(dev, blocksize) - if options.pull: - f, size = do_open(dev, 'rb') - else: - f, size = do_open(dev, 'rb+') + print(dev) + print(blocksize) + + try: + if options.pull: + f, size = do_open(dev, 'rb') + else: + f, size = do_open(dev, 'rb+') + except: + # Unable to access the device + print(-1) + return + print(size) sys.stdout.flush() @@ -252,7 +261,7 @@ def sync(workerid, srcdev, dsthost, dstdev, options): servercmd = 'tmpserver' remotescript = copy_self(workerid, cmd) - cmd += [options.interpreter, remotescript, servercmd, dstdev, '-b', str(blocksize)] + cmd += [options.interpreter, remotescript, servercmd, shlex.quote(dstdev), '-b', str(blocksize)] cmd += ['-d', str(options.fadvise), '-1', options.hash] if options.addhash: @@ -275,16 +284,16 @@ def sync(workerid, srcdev, dsthost, dstdev, options): fadv = p_out.readline().decode('UTF-8').strip() print("[worker %d] Remote fadvise: %s" % (workerid, fadv), file = options.outfile) - p_in.write(bytes(("%d\n" % (size if options.createdest else 0)).encode("UTF-8"))) + p_in.write(bytes(("%d\n" % (size if options.createdest and not dryrun else 0)).encode("UTF-8"))) p_in.flush() - line = p_out.readline().decode('UTF-8') + a = p_out.readline().decode('UTF-8').strip() + b = p_out.readline().decode('UTF-8').strip() p.poll() if p.returncode is not None: print("[worker %d] Failed creating destination file on the remote host!" % workerid, file = options.outfile) sys.exit(1) - a, b = line.split() if a != dstdev: print("[worker %d] Dest device (%s) doesn't match with the remote host (%s)!" % (workerid, dstdev, a), file = options.outfile) sys.exit(1) @@ -298,7 +307,10 @@ def sync(workerid, srcdev, dsthost, dstdev, options): print("[worker %d] Error accessing device on remote host!" % workerid, file = options.outfile) sys.exit(1) remote_size = int(line) - if size > remote_size: + if remote_size < 0: + print("[worker %d] Remote device doesn't exists!" % workerid, file = options.outfile) + sys.exit(1) + elif size > remote_size: print("[worker %d] Source device size (%d) doesn't fit into remote device size (%d)!" % (workerid, size, remote_size), file = options.outfile) sys.exit(1) elif size < remote_size: @@ -382,7 +394,7 @@ def sync(workerid, srcdev, dsthost, dstdev, options): parser.add_option("-d", "--fadvise", dest = "fadvise", type = "int", help = "lower cache pressure by using posix_fadivse (requires Python 3 or python-fadvise; 0 = off, 1 = local on, 2 = remote on, 3 = both on; defaults to 3)", default = 3) parser.add_option("-p", "--pause", dest = "pause", type="int", help = "pause between processing blocks, reduces system load (ms, defaults to 0)", default = 0) parser.add_option("-c", "--cipher", dest = "cipher", help = "cipher specification for SSH (defaults to aes256-ctr)", default = "aes256-ctr") - parser.add_option("-C", "--compress", dest = "compress", action = "store_true", help = "enable compression over SSH (defaults to on)", default = True) + parser.add_option("-N", "--nocompress", dest = "compress", action = "store_false", help = "diable compression over SSH (defaults to on)", default = True) parser.add_option("-i", "--id", dest = "keyfile", help = "SSH public key file") parser.add_option("-P", "--pass", dest = "passenv", help = "environment variable containing SSH password (requires sshpass)") parser.add_option("-s", "--sudo", dest = "sudo", action = "store_true", help = "use sudo on the remote end (defaults to off)", default = False) @@ -390,7 +402,7 @@ def sync(workerid, srcdev, dsthost, dstdev, options): parser.add_option("-n", "--dryrun", dest = "dryrun", action = "store_true", help = "do a dry run (don't write anything, just report differences)", default = False) parser.add_option("-T", "--createdest", dest = "createdest", action = "store_true", help = "create destination file using truncate(2). Should be safe for subsequent syncs as truncate only modifies the file when the size differs", default = False) parser.add_option("-S", "--script", dest = "script", help = "location of script on remote host (otherwise current script is sent over)") - parser.add_option("-I", "--interpreter", dest = "interpreter", help = "[full path to] interpreter used to invoke remote server (defaults to python2)", default = "python2") + parser.add_option("-I", "--interpreter", dest = "interpreter", help = "[full path to] interpreter used to invoke remote server (defaults to python3)", default = "python3") parser.add_option("-t", "--interval", dest = "interval", type = "int", help = "interval between stats output (seconds, defaults to 1)", default = 1) parser.add_option("-o", "--output", dest = "outfile", help = "send output to file instead of console") parser.add_option("-f", "--force", dest = "force", action = "store_true", help = "force sync and DO NOT ask for confirmation if the destination file already exists")