From eb267c323b3617baed2af4c35fbb5849938e463b Mon Sep 17 00:00:00 2001 From: Aleksandr Kuzminsky Date: Tue, 7 Apr 2020 20:43:44 -0700 Subject: [PATCH 1/3] Fix integration tests on python 3 --- .gitignore | 1 + Makefile | 2 + support/bootstrap/master/debian/master1.sh | 10 +- tests/integration/__init__.py | 20 +- tests/integration/backup/s3/conftest.py | 38 +- tests/integration/backup/s3/test_backup.py | 810 +++++++++--------- tests/integration/backup/s3/test_restore.py | 110 ++- tests/integration/backup/ssh/conftest.py | 13 +- tests/integration/backup/ssh/test_backup.py | 127 +-- tests/integration/backup/ssh/test_restore.py | 151 ++-- tests/integration/clone/test_clone.py | 89 +- tests/integration/conftest.py | 311 ++++--- tests/integration/verify/conftest.py | 13 +- tests/integration/verify/test_verify.py | 83 +- tests/unit/source/test_remote_mysql_source.py | 3 - twindb_backup/__init__.py | 2 +- twindb_backup/backup.py | 2 +- twindb_backup/destination/local.py | 4 +- twindb_backup/source/remote_mysql_source.py | 2 +- twindb_backup/ssh/client.py | 11 +- twindb_backup/status/base_status.py | 7 +- twindb_backup/verify.py | 6 +- vagrant/Vagrantfile | 11 +- .../puppet/modules/profile/manifests/base.pp | 10 +- 24 files changed, 916 insertions(+), 920 deletions(-) diff --git a/.gitignore b/.gitignore index 9c3270f87..c22090f93 100644 --- a/.gitignore +++ b/.gitignore @@ -61,3 +61,4 @@ target/ .DS_Store /vagrant/README.html /.pytest_cache/ +/.venv/ diff --git a/Makefile b/Makefile index f73d9cb3b..2a08be8f2 100644 --- a/Makefile +++ b/Makefile @@ -181,6 +181,8 @@ docker-start: --env AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY} \ --env PLATFORM=${PLATFORM} \ --env OS_VERSION=${OS_VERSION} \ + --env LC_ALL=C.UTF-8 \ + --env LANG=C.UTF-8 \ "twindb/omnibus-${PLATFORM}:${OS_VERSION}" \ bash -l diff --git a/support/bootstrap/master/debian/master1.sh b/support/bootstrap/master/debian/master1.sh index bff9da536..22ac12edc 100644 --- a/support/bootstrap/master/debian/master1.sh +++ b/support/bootstrap/master/debian/master1.sh @@ -8,18 +8,18 @@ do apt-get update && break echo "Waiting ${wait_time} seconds before retry" sleep ${wait_time} - let wait_time=${wait_time}*2 + wait_time=$((wait_time * 2)) done -TB_VERSION=$(PYTHONPATH=/twindb-backup python -c "from twindb_backup import __version__; print __version__") +TB_VERSION=$(PYTHONPATH=/twindb-backup python -c "from twindb_backup import __version__; print(__version__)") package="/twindb-backup/omnibus/pkg/twindb-backup_${TB_VERSION}-1_amd64.deb" -dpkg -I ${package} | grep Depends: | sed -e 's/Depends://' -e 's/,//g' | xargs apt-get -y install -dpkg -i ${package} +dpkg -I "${package}" | grep Depends: | sed -e 's/Depends://' -e 's/,//g' | xargs apt-get -y install +dpkg -i "${package}" set +u if ! test -z "${DEV}"; then - /bin/cp -R /twindb-backup/twindb_backup /opt/twindb-backup/embedded/lib/python2.7/site-packages + /bin/cp -R /twindb-backup/twindb_backup /opt/twindb-backup/embedded/lib/python3.7/site-packages fi diff --git a/tests/integration/__init__.py b/tests/integration/__init__.py index f82f162d8..2d556ba4c 100644 --- a/tests/integration/__init__.py +++ b/tests/integration/__init__.py @@ -1,22 +1,22 @@ from os import environ -__author__ = 'aleks' +__author__ = "aleks" def ensure_aws_creds(): - print('Integration tests need Amazon API credentials:') - print(' AWS_ACCESS_KEY_ID') - print(' AWS_SECRET_ACCESS_KEY') + print("Integration tests need Amazon API credentials:") + print(" AWS_ACCESS_KEY_ID") + print(" AWS_SECRET_ACCESS_KEY") try: try: - environ['AWS_ACCESS_KEY_ID'] + environ["AWS_ACCESS_KEY_ID"] except KeyError: - print('Environment variable AWS_ACCESS_KEY_ID is not defined.') - environ['AWS_ACCESS_KEY_ID'] = raw_input('Please enter it: ') + print("Environment variable AWS_ACCESS_KEY_ID is not defined.") + environ["AWS_ACCESS_KEY_ID"] = input("Please enter it: ") try: - environ['AWS_SECRET_ACCESS_KEY'] + environ["AWS_SECRET_ACCESS_KEY"] except KeyError: - print('Environment variable AWS_SECRET_ACCESS_KEY is not defined.') - environ['AWS_SECRET_ACCESS_KEY'] = raw_input('Please enter it: ') + print("Environment variable AWS_SECRET_ACCESS_KEY is not defined.") + environ["AWS_SECRET_ACCESS_KEY"] = input("Please enter it: ") except KeyboardInterrupt: exit(1) diff --git a/tests/integration/backup/s3/conftest.py b/tests/integration/backup/s3/conftest.py index 2e13117fd..055b2c512 100644 --- a/tests/integration/backup/s3/conftest.py +++ b/tests/integration/backup/s3/conftest.py @@ -8,32 +8,33 @@ from twindb_backup import LOG, setup_logging from twindb_backup.destination.s3 import S3 + setup_logging(LOG, debug=True) @pytest.fixture() def bucket_name(): - travis_job_number = os.environ.get('TRAVIS_JOB_NUMBER') - LOG.debug('TRAVIS_JOB_NUMBER=%s' % travis_job_number) + travis_job_number = os.environ.get("TRAVIS_JOB_NUMBER") + LOG.debug("TRAVIS_JOB_NUMBER=%s" % travis_job_number) number = random.randint(0, 1000000) - LOG.debug('Default job number %d' % number) + LOG.debug("Default job number %d" % number) if travis_job_number: - bucket = 'twindb-backup-test-travis-%s' % travis_job_number + bucket = "twindb-backup-test-travis-%s" % travis_job_number else: - bucket = 'twindb-backup-test-travis-%d' % number + bucket = "twindb-backup-test-travis-%d" % number - return '%s-%s' % (bucket, time.time()) + return "%s-%s" % (bucket, time.time()) @pytest.fixture() def s3_client(bucket_name): - LOG.debug('Bucket: %s' % bucket_name) + LOG.debug("Bucket: %s" % bucket_name) client = S3( bucket=bucket_name, - aws_access_key_id=os.environ['AWS_ACCESS_KEY_ID'], - aws_secret_access_key=os.environ['AWS_SECRET_ACCESS_KEY'] + aws_access_key_id=os.environ["AWS_ACCESS_KEY_ID"], + aws_secret_access_key=os.environ["AWS_SECRET_ACCESS_KEY"], ) try: assert client.create_bucket() @@ -47,12 +48,11 @@ def s3_client(bucket_name): @pytest.fixture def foo_bar_dir(tmpdir): - test_dir = tmpdir.join('foo/bar') + test_dir = tmpdir.join("foo/bar") - assert call('rm -rf %s' % str(test_dir), shell=True) == 0 - assert call('mkdir -p %s' % str(test_dir), shell=True) == 0 - assert call('echo $RANDOM > %s' % - str(test_dir.join('file')), shell=True) == 0 + assert call("rm -rf %s" % str(test_dir), shell=True) == 0 + assert call("mkdir -p %s" % str(test_dir), shell=True) == 0 + assert call("echo $RANDOM > %s" % str(test_dir.join("file")), shell=True) == 0 return str(test_dir) @@ -205,21 +205,27 @@ def gpg_private_key(): @pytest.fixture def config_content_mysql_aenc(config_content_mysql_only): - return config_content_mysql_only + """ + return ( + config_content_mysql_only + + """ [gpg] keyring = {gpg_keyring} secret_keyring = {gpg_secret_keyring} recipient = foo@bar """ + ) @pytest.fixture def config_content_files_aenc(config_content_files_only): - return config_content_files_only + """ + return ( + config_content_files_only + + """ [gpg] keyring = {gpg_keyring} secret_keyring = {gpg_secret_keyring} recipient = foo@bar """ + ) diff --git a/tests/integration/backup/s3/test_backup.py b/tests/integration/backup/s3/test_backup.py index 12fed33fc..293aaa4a1 100644 --- a/tests/integration/backup/s3/test_backup.py +++ b/tests/integration/backup/s3/test_backup.py @@ -1,260 +1,269 @@ -import StringIO import json import os +from io import StringIO import magic -from tests.integration.conftest import docker_execute, get_twindb_config_dir, \ - pause_test +from tests.integration.conftest import docker_execute, get_twindb_config_dir, pause_test, assert_and_pause +from twindb_backup import LOG from twindb_backup.destination.s3 import S3 -def test__take_file_backup(master1, - docker_client, - s3_client, - config_content_files_only): - twindb_config_dir = get_twindb_config_dir(docker_client, master1['Id']) +def test__take_file_backup( + master1, docker_client, s3_client, config_content_files_only +): + twindb_config_dir = get_twindb_config_dir(docker_client, master1["Id"]) twindb_config_host = "%s/twindb-backup-1.cfg" % twindb_config_dir - twindb_config_guest = '/etc/twindb/twindb-backup-1.cfg' + twindb_config_guest = "/etc/twindb/twindb-backup-1.cfg" backup_dir = "/etc/twindb" - with open(twindb_config_host, 'w') as fp: + with open(twindb_config_host, "w") as fp: content = config_content_files_only.format( TEST_DIR=backup_dir, - AWS_ACCESS_KEY_ID=os.environ['AWS_ACCESS_KEY_ID'], - AWS_SECRET_ACCESS_KEY=os.environ['AWS_SECRET_ACCESS_KEY'], - BUCKET=s3_client.bucket + AWS_ACCESS_KEY_ID=os.environ["AWS_ACCESS_KEY_ID"], + AWS_SECRET_ACCESS_KEY=os.environ["AWS_SECRET_ACCESS_KEY"], + BUCKET=s3_client.bucket, ) fp.write(content) - cmd = ['twindb-backup', '--debug', - '--config', twindb_config_guest, - 'backup', 'hourly'] - ret, cout = docker_execute(docker_client, master1['Id'], cmd) - print(cout) - assert ret == 0 + cmd = [ + "twindb-backup", + "--debug", + "--config", + twindb_config_guest, + "backup", + "hourly", + ] + ret, cout = docker_execute(docker_client, master1["Id"], cmd) + assert_and_pause((ret == 0,), cout) # Check that backup copy is in "twindb-backup ls" output - hostname = 'master1_1' - s3_backup_path = 's3://%s/%s/hourly/files/%s' % ( + hostname = "master1_1" + s3_backup_path = "s3://%s/%s/hourly/files/%s" % ( s3_client.bucket, hostname, - backup_dir.replace('/', '_') + backup_dir.replace("/", "_"), ) - cmd = ['twindb-backup', '--debug', - '--config', twindb_config_guest, - 'ls'] + cmd = ["twindb-backup", "--debug", "--config", twindb_config_guest, "ls"] - ret, cout = docker_execute(docker_client, master1['Id'], cmd) + ret, cout = docker_execute(docker_client, master1["Id"], cmd) - print(cout) - assert ret == 0 - - assert s3_backup_path in cout + assert_and_pause((ret == 0,), cout) + assert_and_pause((s3_backup_path in cout,), "%s is not in %s" % (s3_backup_path, cout)) backup_to_restore = None - for line in StringIO.StringIO(cout): + for line in StringIO(cout): if line.startswith(s3_backup_path): backup_to_restore = line.strip() break - cmd = ['twindb-backup', '--debug', - '--config', twindb_config_guest, - 'restore', 'file', '--dst', '/tmp/restore', backup_to_restore] + cmd = [ + "twindb-backup", + "--debug", + "--config", + twindb_config_guest, + "restore", + "file", + "--dst", + "/tmp/restore", + backup_to_restore, + ] + assert_and_pause((backup_to_restore is not None,), s3_backup_path) - # print('test paused') - # print(' '.join(cmd)) - # from time import sleep - # sleep(36000) - ret, cout = docker_execute(docker_client, master1['Id'], cmd) - print(cout) - assert ret == 0 + ret, cout = docker_execute(docker_client, master1["Id"], cmd) + assert_and_pause((ret == 0,), cout) # Check that restored file exists - path_to_file_restored = '/tmp/restore/etc/twindb/twindb-backup-1.cfg' - cmd = ['ls', path_to_file_restored] - ret, cout = docker_execute(docker_client, master1['Id'], cmd) - print(cout) - assert ret == 0 + path_to_file_restored = "/tmp/restore/etc/twindb/twindb-backup-1.cfg" + cmd = ["ls", path_to_file_restored] + ret, cout = docker_execute(docker_client, master1["Id"], cmd) + assert_and_pause((ret == 0,), cout) # And content is same - cmd = ['diff', - '/tmp/restore/etc/twindb/twindb-backup-1.cfg', - twindb_config_guest] - ret, cout = docker_execute(docker_client, master1['Id'], cmd) + cmd = ["diff", "/tmp/restore/etc/twindb/twindb-backup-1.cfg", twindb_config_guest] + ret, cout = docker_execute(docker_client, master1["Id"], cmd) # empty output - assert not cout + assert_and_pause((not cout,), cout) # zero exit code if no differences - assert ret == 0 + assert_and_pause((ret == 0,), "%s exited with %d" % (" ".join(cmd), ret)) -def test__take_mysql_backup(master1, - docker_client, - s3_client, - config_content_mysql_only, - client_my_cnf): - twindb_config_dir = get_twindb_config_dir(docker_client, master1['Id']) +def test__take_mysql_backup( + master1, docker_client, s3_client, config_content_mysql_only, client_my_cnf +): + twindb_config_dir = get_twindb_config_dir(docker_client, master1["Id"]) twindb_config_host = "%s/twindb-backup-1.cfg" % twindb_config_dir - twindb_config_guest = '/etc/twindb/twindb-backup-1.cfg' + twindb_config_guest = "/etc/twindb/twindb-backup-1.cfg" my_cnf_path = "%s/my.cnf" % twindb_config_dir with open(my_cnf_path, "w") as my_cnf: my_cnf.write(client_my_cnf) - with open(twindb_config_host, 'w') as fp: + with open(twindb_config_host, "w") as fp: content = config_content_mysql_only.format( - AWS_ACCESS_KEY_ID=os.environ['AWS_ACCESS_KEY_ID'], - AWS_SECRET_ACCESS_KEY=os.environ['AWS_SECRET_ACCESS_KEY'], + AWS_ACCESS_KEY_ID=os.environ["AWS_ACCESS_KEY_ID"], + AWS_SECRET_ACCESS_KEY=os.environ["AWS_SECRET_ACCESS_KEY"], BUCKET=s3_client.bucket, daily_copies=1, hourly_copies=2, - MY_CNF='/etc/twindb/my.cnf' + MY_CNF="/etc/twindb/my.cnf", ) fp.write(content) cmd = [ - 'twindb-backup', - '--debug', - '--config', twindb_config_guest, - 'backup', 'hourly' + "twindb-backup", + "--debug", + "--config", + twindb_config_guest, + "backup", + "hourly", ] - pause_test(' '.join(cmd)) - ret, cout = docker_execute(docker_client, master1['Id'], cmd) - print(cout) - assert ret == 0 + ret, cout = docker_execute(docker_client, master1["Id"], cmd) + assert_and_pause((ret == 0,), cout) - cmd = ['twindb-backup', - '--config', twindb_config_guest, - 'status'] - ret, cout = docker_execute(docker_client, master1['Id'], cmd) - print(cout) - key = json.loads(cout)['hourly'].keys()[0] + cmd = ["twindb-backup", "--config", twindb_config_guest, "status"] + ret, cout = docker_execute(docker_client, master1["Id"], cmd) + LOG.debug("STDOUT: %s", cout) + key = list(json.loads(cout)["hourly"].keys())[0] - assert key.endswith('.xbstream.gz') + assert_and_pause((key.endswith(".xbstream.gz"),), key) -def test__take_mysql_backup_retention(master1, - docker_client, - s3_client, - config_content_mysql_only, - client_my_cnf): +def test__take_mysql_backup_retention( + master1, docker_client, s3_client, config_content_mysql_only, client_my_cnf +): - twindb_config_dir = get_twindb_config_dir(docker_client, master1['Id']) + twindb_config_dir = get_twindb_config_dir(docker_client, master1["Id"]) twindb_config_host = "%s/twindb-backup-1.cfg" % twindb_config_dir - twindb_config_guest = '/etc/twindb/twindb-backup-1.cfg' + twindb_config_guest = "/etc/twindb/twindb-backup-1.cfg" my_cnf_path = "%s/my.cnf" % twindb_config_dir with open(my_cnf_path, "w") as my_cnf: my_cnf.write(client_my_cnf) - with open(twindb_config_host, 'w') as fp: + with open(twindb_config_host, "w") as fp: content = config_content_mysql_only.format( - AWS_ACCESS_KEY_ID=os.environ['AWS_ACCESS_KEY_ID'], - AWS_SECRET_ACCESS_KEY=os.environ['AWS_SECRET_ACCESS_KEY'], + AWS_ACCESS_KEY_ID=os.environ["AWS_ACCESS_KEY_ID"], + AWS_SECRET_ACCESS_KEY=os.environ["AWS_SECRET_ACCESS_KEY"], BUCKET=s3_client.bucket, daily_copies=1, hourly_copies=2, - MY_CNF='/etc/twindb/my.cnf' + MY_CNF="/etc/twindb/my.cnf", ) fp.write(content) - cmd = ['twindb-backup', '--debug', '--config', twindb_config_guest, - 'backup', 'daily'] + cmd = [ + "twindb-backup", + "--debug", + "--config", + twindb_config_guest, + "backup", + "daily", + ] - pause_test(' '.join(cmd)) for i in range(0, 3): - ret, cout = docker_execute(docker_client, master1['Id'], cmd) + ret, cout = docker_execute(docker_client, master1["Id"], cmd) print(cout) assert ret == 0 - cmd = ['twindb-backup', '--debug', '--config', twindb_config_guest, - 'backup', 'hourly'] + cmd = [ + "twindb-backup", + "--debug", + "--config", + twindb_config_guest, + "backup", + "hourly", + ] for i in range(0, 3): - ret, cout = docker_execute(docker_client, master1['Id'], cmd) + ret, cout = docker_execute(docker_client, master1["Id"], cmd) print(cout) assert ret == 0 - cmd = ['twindb-backup', '--config', twindb_config_guest, 'status'] - ret, cout = docker_execute(docker_client, master1['Id'], cmd) - assert ret == 0 - print(cout) + cmd = ["twindb-backup", "--config", twindb_config_guest, "status"] + ret, cout = docker_execute(docker_client, master1["Id"], cmd) + assert_and_pause((ret == 0,), cout) + status = json.loads(cout) - assert len(status['daily'].keys()) == 1 - assert len(status['hourly'].keys()) == 2 + assert_and_pause((len(status["daily"].keys()) == 1,), status) + assert_and_pause((len(status["hourly"].keys()) == 2,), status) -def test__s3_find_files_returns_sorted(master1, - docker_client, - s3_client, - config_content_mysql_only, - client_my_cnf): +def test__s3_find_files_returns_sorted( + master1, docker_client, s3_client, config_content_mysql_only, client_my_cnf +): # cleanup the bucket first s3_client.delete_all_objects() - twindb_config_dir = get_twindb_config_dir(docker_client, master1['Id']) + twindb_config_dir = get_twindb_config_dir(docker_client, master1["Id"]) twindb_config_host = "%s/twindb-backup-1.cfg" % twindb_config_dir - twindb_config_guest = '/etc/twindb/twindb-backup-1.cfg' + twindb_config_guest = "/etc/twindb/twindb-backup-1.cfg" my_cnf_path = "%s/my.cnf" % twindb_config_dir with open(my_cnf_path, "w") as my_cnf: my_cnf.write(client_my_cnf) - with open(twindb_config_host, 'w') as fp: + with open(twindb_config_host, "w") as fp: content = config_content_mysql_only.format( - AWS_ACCESS_KEY_ID=os.environ['AWS_ACCESS_KEY_ID'], - AWS_SECRET_ACCESS_KEY=os.environ['AWS_SECRET_ACCESS_KEY'], + AWS_ACCESS_KEY_ID=os.environ["AWS_ACCESS_KEY_ID"], + AWS_SECRET_ACCESS_KEY=os.environ["AWS_SECRET_ACCESS_KEY"], BUCKET=s3_client.bucket, daily_copies=5, hourly_copies=2, - MY_CNF='/etc/twindb/my.cnf' + MY_CNF="/etc/twindb/my.cnf", ) fp.write(content) - cmd = ['twindb-backup', '--debug', '--config', twindb_config_guest, - 'backup', 'daily'] + cmd = [ + "twindb-backup", + "--debug", + "--config", + twindb_config_guest, + "backup", + "daily", + ] n_runs = 3 - for x in xrange(n_runs): - ret, cout = docker_execute(docker_client, master1['Id'], cmd) + for x in range(n_runs): + ret, cout = docker_execute(docker_client, master1["Id"], cmd) print(cout) assert ret == 0 - hostname = 'master1_1' + hostname = "master1_1" dst = S3( bucket=s3_client.bucket, - aws_access_key_id=os.environ['AWS_ACCESS_KEY_ID'], - aws_secret_access_key=os.environ['AWS_SECRET_ACCESS_KEY'] + aws_access_key_id=os.environ["AWS_ACCESS_KEY_ID"], + aws_secret_access_key=os.environ["AWS_SECRET_ACCESS_KEY"], ) - for x in xrange(10): - result = dst.list_files(dst.remote_path, pattern='/daily/') + for x in range(10): + result = dst.list_files(dst.remote_path, pattern="/daily/") assert len(result) == n_runs assert result == sorted(result) prefix = "{remote_path}/{hostname}/{run_type}/mysql/mysql-".format( - remote_path=dst.remote_path, - hostname=hostname, - run_type='daily' + remote_path=dst.remote_path, hostname=hostname, run_type="daily" ) files = dst.list_files(prefix) assert len(files) == n_runs assert files == sorted(files) -def test_take_file_backup_with_aenc(master1, - docker_client, - s3_client, - config_content_files_aenc, - gpg_public_key, - gpg_private_key): - twindb_config_dir = get_twindb_config_dir(docker_client, master1['Id']) +def test_take_file_backup_with_aenc( + master1, + docker_client, + s3_client, + config_content_files_aenc, + gpg_public_key, + gpg_private_key, +): + twindb_config_dir = get_twindb_config_dir(docker_client, master1["Id"]) twindb_config_host = "%s/twindb-backup-1.cfg" % twindb_config_dir - twindb_config_guest = '/etc/twindb/twindb-backup-1.cfg' + twindb_config_guest = "/etc/twindb/twindb-backup-1.cfg" backup_dir = "/etc/twindb" gpg_public_key_path_host = "%s/public_key" % twindb_config_dir @@ -269,114 +278,128 @@ def test_take_file_backup_with_aenc(master1, with open(gpg_private_key_path_host, "w") as fd: fd.write(gpg_private_key) - cmd = ['rm', '-f', gpg_keyring, gpg_secret_keyring] - ret, cout = docker_execute(docker_client, master1['Id'], cmd) + cmd = ["rm", "-f", gpg_keyring, gpg_secret_keyring] + ret, cout = docker_execute(docker_client, master1["Id"], cmd) print(cout) assert ret == 0 - cmd = ['gpg', - '--no-default-keyring', - '--keyring', gpg_keyring, - '--secret-keyring', gpg_secret_keyring, - '--yes', - '--no-tty', - '--batch', - '--import', - gpg_private_key_path_guest] - ret, cout = docker_execute(docker_client, master1['Id'], cmd) + cmd = [ + "gpg", + "--no-default-keyring", + "--keyring", + gpg_keyring, + "--secret-keyring", + gpg_secret_keyring, + "--yes", + "--no-tty", + "--batch", + "--import", + gpg_private_key_path_guest, + ] + ret, cout = docker_execute(docker_client, master1["Id"], cmd) print(cout) assert ret == 0 - with open(twindb_config_host, 'w') as fp: + with open(twindb_config_host, "w") as fp: content = config_content_files_aenc.format( TEST_DIR=backup_dir, - AWS_ACCESS_KEY_ID=os.environ['AWS_ACCESS_KEY_ID'], - AWS_SECRET_ACCESS_KEY=os.environ['AWS_SECRET_ACCESS_KEY'], + AWS_ACCESS_KEY_ID=os.environ["AWS_ACCESS_KEY_ID"], + AWS_SECRET_ACCESS_KEY=os.environ["AWS_SECRET_ACCESS_KEY"], BUCKET=s3_client.bucket, gpg_keyring=gpg_keyring, - gpg_secret_keyring=gpg_secret_keyring + gpg_secret_keyring=gpg_secret_keyring, ) fp.write(content) # write some content to the directory - with open(os.path.join(twindb_config_dir, 'file'), 'w') as f: + with open(os.path.join(twindb_config_dir, "file"), "w") as f: f.write("Hello world.") - hostname = 'master1_1' - s3_backup_path = 's3://%s/%s/hourly/files/%s' % \ - (s3_client.bucket, hostname, backup_dir.replace('/', '_')) + hostname = "master1_1" + s3_backup_path = "s3://%s/%s/hourly/files/%s" % ( + s3_client.bucket, + hostname, + backup_dir.replace("/", "_"), + ) - cmd = ['twindb-backup', '--debug', - '--config', twindb_config_guest, - 'backup', 'hourly'] - ret, cout = docker_execute(docker_client, master1['Id'], cmd) + cmd = [ + "twindb-backup", + "--debug", + "--config", + twindb_config_guest, + "backup", + "hourly", + ] + ret, cout = docker_execute(docker_client, master1["Id"], cmd) print(cout) assert ret == 0 - cmd = ['twindb-backup', '--debug', - '--config', twindb_config_guest, - 'ls'] - ret, cout = docker_execute(docker_client, master1['Id'], cmd) + cmd = ["twindb-backup", "--debug", "--config", twindb_config_guest, "ls"] + ret, cout = docker_execute(docker_client, master1["Id"], cmd) print(cout) assert ret == 0 assert s3_backup_path in cout backup_to_restore = None - for line in StringIO.StringIO(cout): + for line in StringIO(cout): if line.startswith(s3_backup_path): backup_to_restore = line.strip() break - assert backup_to_restore.endswith('.tar.gz.gpg') - key = backup_to_restore.lstrip('s3://').lstrip(s3_client.bucket).lstrip('/') - local_copy = '%s/backup_to_restore.tar.gz.gpg' % twindb_config_dir - s3_client.s3_client.download_file( - s3_client.bucket, - key, - local_copy - ) - assert magic.from_file(local_copy) == 'data' + assert backup_to_restore.endswith(".tar.gz.gpg") + key = backup_to_restore.lstrip("s3://").lstrip(s3_client.bucket).lstrip("/") + local_copy = "%s/backup_to_restore.tar.gz.gpg" % twindb_config_dir + s3_client.s3_client.download_file(s3_client.bucket, key, local_copy) + assert magic.from_file(local_copy) == "data" - dest_dir = '/tmp/simple_backup_aenc' - cmd = ['mkdir', '-p', '/tmp/simple_backup_aenc'] - ret, cout = docker_execute(docker_client, master1['Id'], cmd) + dest_dir = "/tmp/simple_backup_aenc" + cmd = ["mkdir", "-p", "/tmp/simple_backup_aenc"] + ret, cout = docker_execute(docker_client, master1["Id"], cmd) print(cout) assert ret == 0 - cmd = ['twindb-backup', '--debug', - '--config', twindb_config_guest, - 'restore', 'file', '--dst', dest_dir, backup_to_restore] + cmd = [ + "twindb-backup", + "--debug", + "--config", + twindb_config_guest, + "restore", + "file", + "--dst", + dest_dir, + backup_to_restore, + ] - ret, cout = docker_execute(docker_client, master1['Id'], cmd) + ret, cout = docker_execute(docker_client, master1["Id"], cmd) print(cout) assert ret == 0 - path_to_file_restored = '%s%s/file' % (dest_dir, backup_dir) - cmd = ['test', '-f', path_to_file_restored] - ret, cout = docker_execute(docker_client, master1['Id'], cmd) + path_to_file_restored = "%s%s/file" % (dest_dir, backup_dir) + cmd = ["test", "-f", path_to_file_restored] + ret, cout = docker_execute(docker_client, master1["Id"], cmd) print(cout) assert ret == 0 # And content is same path_to_file_orig = "%s/file" % backup_dir - cmd = ['diff', '-Nur', - path_to_file_orig, - path_to_file_restored] - ret, cout = docker_execute(docker_client, master1['Id'], cmd) + cmd = ["diff", "-Nur", path_to_file_orig, path_to_file_restored] + ret, cout = docker_execute(docker_client, master1["Id"], cmd) print(cout) assert ret == 0 assert not cout -def test__take_mysql_backup_aenc_suffix_gpg(master1, - docker_client, - s3_client, - config_content_mysql_aenc, - gpg_public_key, - gpg_private_key, - client_my_cnf): - twindb_config_dir = get_twindb_config_dir(docker_client, master1['Id']) +def test__take_mysql_backup_aenc_suffix_gpg( + master1, + docker_client, + s3_client, + config_content_mysql_aenc, + gpg_public_key, + gpg_private_key, + client_my_cnf, +): + twindb_config_dir = get_twindb_config_dir(docker_client, master1["Id"]) twindb_config_host = "%s/twindb-backup-1.cfg" % twindb_config_dir - twindb_config_guest = '/etc/twindb/twindb-backup-1.cfg' + twindb_config_guest = "/etc/twindb/twindb-backup-1.cfg" my_cnf_path = "%s/my.cnf" % twindb_config_dir @@ -395,75 +418,79 @@ def test__take_mysql_backup_aenc_suffix_gpg(master1, with open(gpg_private_key_path_host, "w") as fd: fd.write(gpg_private_key) - cmd = ['rm', '-f', gpg_keyring, gpg_secret_keyring] - ret, cout = docker_execute(docker_client, master1['Id'], cmd) + cmd = ["rm", "-f", gpg_keyring, gpg_secret_keyring] + ret, cout = docker_execute(docker_client, master1["Id"], cmd) print(cout) assert ret == 0 - cmd = ['gpg', - '--no-default-keyring', - '--keyring', gpg_keyring, - '--secret-keyring', gpg_secret_keyring, - '--yes', - '--no-tty', - '--batch', - '--import', - gpg_private_key_path_guest] - ret, cout = docker_execute(docker_client, master1['Id'], cmd) + cmd = [ + "gpg", + "--no-default-keyring", + "--keyring", + gpg_keyring, + "--secret-keyring", + gpg_secret_keyring, + "--yes", + "--no-tty", + "--batch", + "--import", + gpg_private_key_path_guest, + ] + ret, cout = docker_execute(docker_client, master1["Id"], cmd) print(cout) assert ret == 0 - with open(twindb_config_host, 'w') as fp: + with open(twindb_config_host, "w") as fp: content = config_content_mysql_aenc.format( - AWS_ACCESS_KEY_ID=os.environ['AWS_ACCESS_KEY_ID'], - AWS_SECRET_ACCESS_KEY=os.environ['AWS_SECRET_ACCESS_KEY'], + AWS_ACCESS_KEY_ID=os.environ["AWS_ACCESS_KEY_ID"], + AWS_SECRET_ACCESS_KEY=os.environ["AWS_SECRET_ACCESS_KEY"], BUCKET=s3_client.bucket, gpg_keyring=gpg_keyring, gpg_secret_keyring=gpg_secret_keyring, daily_copies=1, hourly_copies=2, - MY_CNF='/etc/twindb/my.cnf' + MY_CNF="/etc/twindb/my.cnf", ) fp.write(content) - cmd = ['twindb-backup', '--debug', - '--config', twindb_config_guest, - 'backup', 'daily'] - ret, cout = docker_execute(docker_client, master1['Id'], cmd) + cmd = [ + "twindb-backup", + "--debug", + "--config", + twindb_config_guest, + "backup", + "daily", + ] + ret, cout = docker_execute(docker_client, master1["Id"], cmd) print(cout) assert ret == 0 - cmd = ['twindb-backup', - '--config', twindb_config_guest, - 'status'] - ret, cout = docker_execute(docker_client, master1['Id'], cmd) + cmd = ["twindb-backup", "--config", twindb_config_guest, "status"] + ret, cout = docker_execute(docker_client, master1["Id"], cmd) print(cout) assert ret == 0 - key = json.loads(cout)['daily'].keys()[0] - assert key.endswith('xbstream.gz.gpg') - local_copy = '%s/mysql_backup.tar.gz.gpg' % twindb_config_dir + key = list(json.loads(cout)["daily"].keys())[0] + assert key.endswith("xbstream.gz.gpg") + local_copy = "%s/mysql_backup.tar.gz.gpg" % twindb_config_dir - s3_client.s3_client.download_file( - s3_client.bucket, - key, - local_copy - ) - assert magic.from_file(local_copy) == 'data' + s3_client.s3_client.download_file(s3_client.bucket, key, local_copy) + assert magic.from_file(local_copy) == "data" def test_take_mysql_backup_aenc_restores_full( - master1, - docker_client, - s3_client, - config_content_mysql_aenc, - gpg_public_key, - gpg_private_key, - tmpdir, - client_my_cnf): - - twindb_config_dir = get_twindb_config_dir(docker_client, master1['Id']) + master1, + docker_client, + s3_client, + config_content_mysql_aenc, + gpg_public_key, + gpg_private_key, + tmpdir, + client_my_cnf, +): + + twindb_config_dir = get_twindb_config_dir(docker_client, master1["Id"]) twindb_config_host = "%s/twindb-backup-1.cfg" % twindb_config_dir - twindb_config_guest = '/etc/twindb/twindb-backup-1.cfg' + twindb_config_guest = "/etc/twindb/twindb-backup-1.cfg" my_cnf_path = "%s/my.cnf" % twindb_config_dir @@ -482,160 +509,131 @@ def test_take_mysql_backup_aenc_restores_full( with open(gpg_private_key_path_host, "w") as fd: fd.write(gpg_private_key) - cmd = ['rm', '-f', gpg_keyring, gpg_secret_keyring] - ret, cout = docker_execute(docker_client, master1['Id'], cmd) + cmd = ["rm", "-f", gpg_keyring, gpg_secret_keyring] + ret, cout = docker_execute(docker_client, master1["Id"], cmd) print(cout) assert ret == 0 cmd = [ - 'gpg', - '--no-default-keyring', - '--keyring', gpg_keyring, - '--secret-keyring', gpg_secret_keyring, - '--yes', - '--no-tty', - '--batch', - '--import', - gpg_private_key_path_guest + "gpg", + "--no-default-keyring", + "--keyring", + gpg_keyring, + "--secret-keyring", + gpg_secret_keyring, + "--yes", + "--no-tty", + "--batch", + "--import", + gpg_private_key_path_guest, ] - ret, cout = docker_execute(docker_client, master1['Id'], cmd) + ret, cout = docker_execute(docker_client, master1["Id"], cmd) print(cout) assert ret == 0 - with open(twindb_config_host, 'w') as fp: + with open(twindb_config_host, "w") as fp: content = config_content_mysql_aenc.format( - AWS_ACCESS_KEY_ID=os.environ['AWS_ACCESS_KEY_ID'], - AWS_SECRET_ACCESS_KEY=os.environ['AWS_SECRET_ACCESS_KEY'], + AWS_ACCESS_KEY_ID=os.environ["AWS_ACCESS_KEY_ID"], + AWS_SECRET_ACCESS_KEY=os.environ["AWS_SECRET_ACCESS_KEY"], BUCKET=s3_client.bucket, gpg_keyring=gpg_keyring, gpg_secret_keyring=gpg_secret_keyring, daily_copies=1, hourly_copies=2, - MY_CNF='/etc/twindb/my.cnf' + MY_CNF="/etc/twindb/my.cnf", ) fp.write(content) cmd = [ - 'twindb-backup', - '--debug', - '--config', + "twindb-backup", + "--debug", + "--config", twindb_config_guest, - 'backup', - 'daily' + "backup", + "daily", ] - ret, cout = docker_execute(docker_client, master1['Id'], cmd) + ret, cout = docker_execute(docker_client, master1["Id"], cmd) print(cout) assert ret == 0 - cmd = [ - 'twindb-backup', - '--config', - twindb_config_guest, - 'status' - ] - ret, cout = docker_execute(docker_client, master1['Id'], cmd) + cmd = ["twindb-backup", "--config", twindb_config_guest, "status"] + ret, cout = docker_execute(docker_client, master1["Id"], cmd) print(cout) assert ret == 0 - key = json.loads(cout)['daily'].keys()[0] + key = list(json.loads(cout)["daily"].keys())[0] - backup_copy = 's3://' + s3_client.bucket + '/' + key - dst_dir = str(tmpdir.mkdir('dst')) + backup_copy = "s3://" + s3_client.bucket + "/" + key + dst_dir = str(tmpdir.mkdir("dst")) ret, cout = docker_execute( - docker_client, - master1['Id'], - ["mkdir", "-p", str(dst_dir)] + docker_client, master1["Id"], ["mkdir", "-p", str(dst_dir)] ) print(cout) assert ret == 0 cmd = [ - 'twindb-backup', - '--debug', - '--config', + "twindb-backup", + "--debug", + "--config", str(twindb_config_guest), - 'restore', - 'mysql', + "restore", + "mysql", backup_copy, - '--dst', - dst_dir + "--dst", + dst_dir, ] - pause_test(' '.join(cmd)) - - ret, cout = docker_execute( - docker_client, - master1['Id'], - cmd - ) + ret, cout = docker_execute(docker_client, master1["Id"], cmd) print(cout) assert ret == 0 - print('Files in restored datadir:') - ret, cout = docker_execute( - docker_client, - master1['Id'], - ["find", dst_dir] - ) + print("Files in restored datadir:") + ret, cout = docker_execute(docker_client, master1["Id"], ["find", dst_dir]) print(cout) assert ret == 0 files_to_test = [] mysql_files = [ - 'ibdata1', - 'ib_logfile0', - 'ib_logfile1', - 'mysql/user.MYD', - 'backup-my.cnf', - 'xtrabackup_logfile' + "ibdata1", + "ib_logfile0", + "ib_logfile1", + "mysql/user.MYD", + "backup-my.cnf", + "xtrabackup_logfile", ] for datadir_file in mysql_files: - files_to_test += [ - "test -f %s/%s" % (dst_dir, datadir_file) - ] - cmd = [ - "bash", - "-c", - " && ".join(files_to_test) - ] + files_to_test += ["test -f %s/%s" % (dst_dir, datadir_file)] + cmd = ["bash", "-c", " && ".join(files_to_test)] print(cmd) - ret, cout = docker_execute( - docker_client, - master1['Id'], - cmd - ) + ret, cout = docker_execute(docker_client, master1["Id"], cmd) print(cout) assert ret == 0 cmd = [ - "bash", "-c", + "bash", + "-c", "test -f {datadir}/_config/etc/my.cnf " - "|| test -f {datadir}/_config/etc/mysql/my.cnf".format( - datadir=dst_dir - ) + "|| test -f {datadir}/_config/etc/mysql/my.cnf".format(datadir=dst_dir), ] - ret, cout = docker_execute( - docker_client, - master1['Id'], - cmd - ) + ret, cout = docker_execute(docker_client, master1["Id"], cmd) print(cout) assert ret == 0 def test_take_mysql_backup_aenc_restores_inc( - master1, - docker_client, - s3_client, - config_content_mysql_aenc, - gpg_public_key, - gpg_private_key, - tmpdir, - client_my_cnf): - twindb_config_dir = get_twindb_config_dir(docker_client, master1['Id']) + master1, + docker_client, + s3_client, + config_content_mysql_aenc, + gpg_public_key, + gpg_private_key, + tmpdir, + client_my_cnf, +): + twindb_config_dir = get_twindb_config_dir(docker_client, master1["Id"]) twindb_config_host = "%s/twindb-backup-1.cfg" % twindb_config_dir - twindb_config_guest = '/etc/twindb/twindb-backup-1.cfg' + twindb_config_guest = "/etc/twindb/twindb-backup-1.cfg" my_cnf_path = "%s/my.cnf" % twindb_config_dir @@ -654,134 +652,124 @@ def test_take_mysql_backup_aenc_restores_inc( with open(gpg_private_key_path_host, "w") as fd: fd.write(gpg_private_key) - cmd = ['rm', '-f', gpg_keyring, gpg_secret_keyring] - ret, cout = docker_execute(docker_client, master1['Id'], cmd) + cmd = ["rm", "-f", gpg_keyring, gpg_secret_keyring] + ret, cout = docker_execute(docker_client, master1["Id"], cmd) print(cout) assert ret == 0 - cmd = ['gpg', - '--no-default-keyring', - '--keyring', gpg_keyring, - '--secret-keyring', gpg_secret_keyring, - '--yes', - '--no-tty', - '--batch', - '--import', - gpg_private_key_path_guest] - ret, cout = docker_execute(docker_client, master1['Id'], cmd) + cmd = [ + "gpg", + "--no-default-keyring", + "--keyring", + gpg_keyring, + "--secret-keyring", + gpg_secret_keyring, + "--yes", + "--no-tty", + "--batch", + "--import", + gpg_private_key_path_guest, + ] + ret, cout = docker_execute(docker_client, master1["Id"], cmd) print(cout) assert ret == 0 - with open(twindb_config_host, 'w') as fp: + with open(twindb_config_host, "w") as fp: content = config_content_mysql_aenc.format( - AWS_ACCESS_KEY_ID=os.environ['AWS_ACCESS_KEY_ID'], - AWS_SECRET_ACCESS_KEY=os.environ['AWS_SECRET_ACCESS_KEY'], + AWS_ACCESS_KEY_ID=os.environ["AWS_ACCESS_KEY_ID"], + AWS_SECRET_ACCESS_KEY=os.environ["AWS_SECRET_ACCESS_KEY"], BUCKET=s3_client.bucket, gpg_keyring=gpg_keyring, gpg_secret_keyring=gpg_secret_keyring, daily_copies=1, hourly_copies=2, - MY_CNF='/etc/twindb/my.cnf' + MY_CNF="/etc/twindb/my.cnf", ) fp.write(content) - cmd = ['twindb-backup', '--debug', - '--config', twindb_config_guest, - 'backup', 'daily'] - ret, cout = docker_execute(docker_client, master1['Id'], cmd) + cmd = [ + "twindb-backup", + "--debug", + "--config", + twindb_config_guest, + "backup", + "daily", + ] + ret, cout = docker_execute(docker_client, master1["Id"], cmd) print(cout) assert ret == 0 - cmd = ['twindb-backup', '--debug', - '--config', twindb_config_guest, - 'backup', 'hourly'] - ret, cout = docker_execute(docker_client, master1['Id'], cmd) + cmd = [ + "twindb-backup", + "--debug", + "--config", + twindb_config_guest, + "backup", + "hourly", + ] + ret, cout = docker_execute(docker_client, master1["Id"], cmd) print(cout) assert ret == 0 - cmd = ['twindb-backup', - '--config', twindb_config_guest, - 'status'] - ret, cout = docker_execute(docker_client, master1['Id'], cmd) + cmd = ["twindb-backup", "--config", twindb_config_guest, "status"] + ret, cout = docker_execute(docker_client, master1["Id"], cmd) print(cout) assert ret == 0 - key = json.loads(cout)['hourly'].keys()[0] + key = list(json.loads(cout)["hourly"].keys())[0] - backup_copy = 's3://' + s3_client.bucket + '/' + key - dst_dir = str(tmpdir.mkdir('dst')) + backup_copy = "s3://" + s3_client.bucket + "/" + key + dst_dir = str(tmpdir.mkdir("dst")) ret, cout = docker_execute( - docker_client, - master1['Id'], - ["mkdir", "-p", str(dst_dir)] + docker_client, master1["Id"], ["mkdir", "-p", str(dst_dir)] ) print(cout) assert ret == 0 cmd = [ - 'twindb-backup', '--debug', - '--config', str(twindb_config_guest), - 'restore', 'mysql', + "twindb-backup", + "--debug", + "--config", + str(twindb_config_guest), + "restore", + "mysql", backup_copy, - '--dst', dst_dir + "--dst", + dst_dir, ] - # LOG.debug('Test paused') - # LOG.debug(' '.join(cmd)) - # import time - # time.sleep(36000) - - ret, cout = docker_execute( - docker_client, - master1['Id'], - cmd - ) + ret, cout = docker_execute(docker_client, master1["Id"], cmd) print(cout) assert ret == 0 - print('Files in restored datadir:') - ret, cout = docker_execute( - docker_client, - master1['Id'], - ["find", dst_dir] - ) + print("Files in restored datadir:") + ret, cout = docker_execute(docker_client, master1["Id"], ["find", dst_dir]) print(cout) assert ret == 0 files_to_test = [] - for datadir_file in ['ibdata1', 'ib_logfile0', 'ib_logfile1', - 'mysql/user.MYD', - 'backup-my.cnf', - 'xtrabackup_logfile']: - files_to_test += [ - "test -f %s/%s" % (dst_dir, datadir_file) - ] - cmd = [ - "bash", - "-c", - " && ".join(files_to_test) - ] + for datadir_file in [ + "ibdata1", + "ib_logfile0", + "ib_logfile1", + "mysql/user.MYD", + "backup-my.cnf", + "xtrabackup_logfile", + ]: + files_to_test += ["test -f %s/%s" % (dst_dir, datadir_file)] + cmd = ["bash", "-c", " && ".join(files_to_test)] print(cmd) - ret, cout = docker_execute( - docker_client, - master1['Id'], - cmd - ) + ret, cout = docker_execute(docker_client, master1["Id"], cmd) print(cout) assert ret == 0 cmd = [ - "bash", "-c", + "bash", + "-c", "test -f {datadir}/_config/etc/my.cnf " - "|| test -f {datadir}/_config/etc/mysql/my.cnf".format( - datadir=dst_dir - ) + "|| test -f {datadir}/_config/etc/mysql/my.cnf".format(datadir=dst_dir), ] - ret, cout = docker_execute( - docker_client, - master1['Id'], - cmd - ) + ret, cout = docker_execute(docker_client, master1["Id"], cmd) print(cout) assert ret == 0 diff --git a/tests/integration/backup/s3/test_restore.py b/tests/integration/backup/s3/test_restore.py index 4efd64bbb..40dff8dd6 100644 --- a/tests/integration/backup/s3/test_restore.py +++ b/tests/integration/backup/s3/test_restore.py @@ -4,105 +4,123 @@ from tests.integration.conftest import get_twindb_config_dir, docker_execute -def test__restore_mysql_inc_creates_log_files(master1, - docker_client, - s3_client, - config_content_mysql_only, - client_my_cnf): - twindb_config_dir = get_twindb_config_dir(docker_client, master1['Id']) +def test__restore_mysql_inc_creates_log_files( + master1, docker_client, s3_client, config_content_mysql_only, client_my_cnf +): + twindb_config_dir = get_twindb_config_dir(docker_client, master1["Id"]) twindb_config_host = "%s/twindb-backup-1.cfg" % twindb_config_dir - twindb_config_guest = '/etc/twindb/twindb-backup-1.cfg' + twindb_config_guest = "/etc/twindb/twindb-backup-1.cfg" my_cnf_path = "%s/my.cnf" % twindb_config_dir with open(my_cnf_path, "w") as my_cnf: my_cnf.write(client_my_cnf) - with open(twindb_config_host, 'w') as fp: + with open(twindb_config_host, "w") as fp: content = config_content_mysql_only.format( - AWS_ACCESS_KEY_ID=os.environ['AWS_ACCESS_KEY_ID'], - AWS_SECRET_ACCESS_KEY=os.environ['AWS_SECRET_ACCESS_KEY'], + AWS_ACCESS_KEY_ID=os.environ["AWS_ACCESS_KEY_ID"], + AWS_SECRET_ACCESS_KEY=os.environ["AWS_SECRET_ACCESS_KEY"], BUCKET=s3_client.bucket, daily_copies=1, hourly_copies=2, - MY_CNF='/etc/twindb/my.cnf' + MY_CNF="/etc/twindb/my.cnf", ) fp.write(content) - cmd = ['ls', '-la', '/var/lib/mysql'] - ret, cout = docker_execute(docker_client, master1['Id'], cmd) + cmd = ["ls", "-la", "/var/lib/mysql"] + ret, cout = docker_execute(docker_client, master1["Id"], cmd) print(cout) assert ret == 0 - cmd = ['twindb-backup', '--debug', '--config', twindb_config_guest, 'backup', 'hourly'] - ret, cout = docker_execute(docker_client, master1['Id'], cmd) + cmd = [ + "twindb-backup", + "--debug", + "--config", + twindb_config_guest, + "backup", + "hourly", + ] + ret, cout = docker_execute(docker_client, master1["Id"], cmd) print(cout) assert ret == 0 - cmd = ['twindb-backup', '--debug', '--config', twindb_config_guest, 'backup', 'daily'] - ret, cout = docker_execute(docker_client, master1['Id'], cmd) + cmd = [ + "twindb-backup", + "--debug", + "--config", + twindb_config_guest, + "backup", + "daily", + ] + ret, cout = docker_execute(docker_client, master1["Id"], cmd) print(cout) assert ret == 0 - cmd = ['twindb-backup', '--config', twindb_config_guest, 'status'] - ret, cout = docker_execute(docker_client, master1['Id'], cmd) + cmd = ["twindb-backup", "--config", twindb_config_guest, "status"] + ret, cout = docker_execute(docker_client, master1["Id"], cmd) print(cout) assert ret == 0 status = json.loads(cout) - key = status['hourly'].keys()[0] - backup_copy = 's3://' + s3_client.bucket + '/' + key - dst_dir = '/tmp/dst_full_log_files' - ret, cout = docker_execute(docker_client, master1['Id'], cmd) + key = list(status["hourly"].keys())[0] + backup_copy = "s3://" + s3_client.bucket + "/" + key + dst_dir = "/tmp/dst_full_log_files" + ret, cout = docker_execute(docker_client, master1["Id"], cmd) print(cout) assert ret == 0 - cmd = ['twindb-backup', '--debug', - '--config', twindb_config_guest, - 'restore', 'mysql', - backup_copy, - '--dst', dst_dir] - ret, cout = docker_execute(docker_client, master1['Id'], cmd) + cmd = [ + "twindb-backup", + "--debug", + "--config", + twindb_config_guest, + "restore", + "mysql", + backup_copy, + "--dst", + dst_dir, + ] + ret, cout = docker_execute(docker_client, master1["Id"], cmd) print(cout) assert ret == 0 - cmd = ['find', dst_dir] - ret, cout = docker_execute(docker_client, master1['Id'], cmd) + cmd = ["find", dst_dir] + ret, cout = docker_execute(docker_client, master1["Id"], cmd) print(cout) assert ret == 0 - cmd = ['test', '-f', '/tmp/dst_full_log_files/backup-my.cnf'] + cmd = ["test", "-f", "/tmp/dst_full_log_files/backup-my.cnf"] print(cmd) - ret, cout = docker_execute(docker_client, master1['Id'], cmd) + ret, cout = docker_execute(docker_client, master1["Id"], cmd) print(cout) assert ret == 0 - cmd = ['test', '-f', '/tmp/dst_full_log_files/ibdata1'] + cmd = ["test", "-f", "/tmp/dst_full_log_files/ibdata1"] print(cmd) - ret, cout = docker_execute(docker_client, master1['Id'], cmd) + ret, cout = docker_execute(docker_client, master1["Id"], cmd) print(cout) assert ret == 0 - cmd = ['test', '-f', '/tmp/dst_full_log_files/ib_logfile0'] + cmd = ["test", "-f", "/tmp/dst_full_log_files/ib_logfile0"] print(cmd) - ret, cout = docker_execute(docker_client, master1['Id'], cmd) + ret, cout = docker_execute(docker_client, master1["Id"], cmd) print(cout) assert ret == 0 - cmd = ['test', '-f', '/tmp/dst_full_log_files/ib_logfile1'] + cmd = ["test", "-f", "/tmp/dst_full_log_files/ib_logfile1"] print(cmd) - ret, cout = docker_execute(docker_client, master1['Id'], cmd) + ret, cout = docker_execute(docker_client, master1["Id"], cmd) print(cout) assert ret == 0 - cmd = ['test', '-f', '/tmp/dst_full_log_files/mysql/user.MYD'] + cmd = ["test", "-f", "/tmp/dst_full_log_files/mysql/user.MYD"] print(cmd) - ret, cout = docker_execute(docker_client, master1['Id'], cmd) + ret, cout = docker_execute(docker_client, master1["Id"], cmd) print(cout) assert ret == 0 - cmd = ['test', '-f', '/tmp/dst_full_log_files/xtrabackup_logfile'] + cmd = ["test", "-f", "/tmp/dst_full_log_files/xtrabackup_logfile"] print(cmd) - ret, cout = docker_execute(docker_client, master1['Id'], cmd) + ret, cout = docker_execute(docker_client, master1["Id"], cmd) print(cout) assert ret == 0 cmd = [ "bash", "-c", - 'test -f /tmp/dst_full_log_files/_config/etc/my.cnf ' - '|| test -f /tmp/dst_full_log_files/_config/etc/mysql/my.cnf' + "test -f /tmp/dst_full_log_files/_config/etc/my.cnf " + "|| test -f /tmp/dst_full_log_files/_config/etc/mysql/my.cnf", ] print(cmd) - ret, cout = docker_execute(docker_client, master1['Id'], cmd) + ret, cout = docker_execute(docker_client, master1["Id"], cmd) print(cout) assert ret == 0 diff --git a/tests/integration/backup/ssh/conftest.py b/tests/integration/backup/ssh/conftest.py index 62012c95c..7e848d4dc 100644 --- a/tests/integration/backup/ssh/conftest.py +++ b/tests/integration/backup/ssh/conftest.py @@ -11,30 +11,29 @@ @pytest.yield_fixture def storage_server(docker_client, container_network): - bootstrap_script = '/twindb-backup/support/bootstrap/storage_server.sh' + bootstrap_script = "/twindb-backup/support/bootstrap/storage_server.sh" container = get_container( - 'storage_server', + "storage_server", docker_client, container_network, bootstrap_script=bootstrap_script, image="centos:centos7", - last_n=2 + last_n=2, ) timeout = time.time() + 30 * 60 while time.time() < timeout: sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - if sock.connect_ex((container['ip'], 22)) == 0: + if sock.connect_ex((container["ip"], 22)) == 0: break time.sleep(1) yield container if container: - LOG.info('Removing container %s', container['Id']) - docker_client.api.remove_container(container=container['Id'], - force=True) + LOG.info("Removing container %s", container["Id"]) + docker_client.api.remove_container(container=container["Id"], force=True) @pytest.fixture diff --git a/tests/integration/backup/ssh/test_backup.py b/tests/integration/backup/ssh/test_backup.py index b3fde4b1b..c1a2e5843 100644 --- a/tests/integration/backup/ssh/test_backup.py +++ b/tests/integration/backup/ssh/test_backup.py @@ -1,21 +1,21 @@ import json +import os -from tests.integration.conftest import get_twindb_config_dir, docker_execute +from tests.integration.conftest import get_twindb_config_dir, docker_execute, assert_and_pause -def test_backup(master1, storage_server, - config_content_ssh, - docker_client, - rsa_private_key): +def test_backup( + master1, storage_server, config_content_ssh, docker_client, rsa_private_key +): - twindb_config_dir = get_twindb_config_dir(docker_client, master1['Id']) + twindb_config_dir = get_twindb_config_dir(docker_client, master1["Id"]) twindb_config_host = "%s/twindb-backup-1.cfg" % twindb_config_dir - twindb_config_guest = '/etc/twindb/twindb-backup-1.cfg' + twindb_config_guest = "/etc/twindb/twindb-backup-1.cfg" my_cnf_path = "%s/my.cnf" % twindb_config_dir ssh_key_host = "%s/id_rsa" % twindb_config_dir - ssh_key_guest = '/etc/twindb/id_rsa' + ssh_key_guest = "/etc/twindb/id_rsa" contents = """ [client] @@ -26,80 +26,83 @@ def test_backup(master1, storage_server, my_cnf.write(contents) with open(ssh_key_host, "w") as ssh_fd: - ssh_fd.write(rsa_private_key) + ssh_fd.write(rsa_private_key.strip()) - with open(twindb_config_host, 'w') as fp: + with open(twindb_config_host, "w") as fp: content = config_content_ssh.format( PRIVATE_KEY=ssh_key_guest, - HOST_IP=storage_server['ip'], - MY_CNF='/etc/twindb/my.cnf' + HOST_IP=storage_server["ip"], + MY_CNF="/etc/twindb/my.cnf", ) fp.write(content) - - cmd = ['twindb-backup', - '--debug', - '--config', twindb_config_guest, - 'backup', 'hourly'] - - # print('Test paused') - # print(' '.join(cmd)) - # import time - # time.sleep(36000) - - ret, cout = docker_execute(docker_client, master1['Id'], cmd) - print(cout) - assert ret == 0 + os.chmod(ssh_key_host, 0o600) cmd = [ - 'twindb-backup', - '--config', twindb_config_guest, - 'status' + "twindb-backup", + "--debug", + "--config", + twindb_config_guest, + "backup", + "hourly", ] - ret, cout = docker_execute(docker_client, master1['Id'], cmd) + ret, cout = docker_execute(docker_client, master1["Id"], cmd) + assert_and_pause((ret == 0, ), cout) + + cmd = ["twindb-backup", "--config", twindb_config_guest, "status"] + + ret, cout = docker_execute(docker_client, master1["Id"], cmd) assert ret == 0 status = json.loads(cout) assert len(status["hourly"]) == 1 - cmd = ['twindb-backup', - '--debug', - '--config', twindb_config_guest, - 'backup', 'hourly'] - ret, cout = docker_execute(docker_client, master1['Id'], cmd) - print(cout) - assert ret == 0 + cmd = [ + "twindb-backup", + "--debug", + "--config", + twindb_config_guest, + "backup", + "hourly", + ] + ret, cout = docker_execute(docker_client, master1["Id"], cmd) + assert_and_pause((ret == 0, ), cout) - cmd = ['test', '-d', '/tmp/backup'] + cmd = ["test", "-d", "/tmp/backup"] + + ret, cout = docker_execute(docker_client, storage_server["Id"], cmd) + assert_and_pause((ret == 0, ), cout) - ret, cout = docker_execute(docker_client, storage_server['Id'], cmd) - print(cout) - assert ret == 0 dir_path = "/var/backup/local/master1_1/hourly/mysql" cmd = ["bash", "-c", "ls %s | wc -l" % dir_path] - ret, cout = docker_execute(docker_client, master1['Id'], cmd, tty=True) - print(cout) - assert ret == 0 - assert '1' in cout + ret, cout = docker_execute(docker_client, master1["Id"], cmd, tty=True) + assert_and_pause((ret == 0, ), cout) + assert_and_pause(("1" in cout, ), cout) - cmd = ['twindb-backup', - '--debug', - '--config', twindb_config_guest, - 'backup', 'daily'] - ret, cout = docker_execute(docker_client, master1['Id'], cmd) - print(cout) - assert ret == 0 + cmd = [ + "twindb-backup", + "--debug", + "--config", + twindb_config_guest, + "backup", + "daily", + ] + ret, cout = docker_execute(docker_client, master1["Id"], cmd) + assert_and_pause((ret == 0, ), cout) - cmd = ['twindb-backup', - '--debug', - '--config', twindb_config_guest, - 'backup', 'daily'] - ret, cout = docker_execute(docker_client, master1['Id'], cmd) - print(cout) - assert ret == 0 + cmd = [ + "twindb-backup", + "--debug", + "--config", + twindb_config_guest, + "backup", + "daily", + ] + ret, cout = docker_execute(docker_client, master1["Id"], cmd) + assert_and_pause((ret == 0, ), cout) dir_path = "/var/backup/local/master1_1/daily/mysql" cmd = ["bash", "-c", "ls %s | wc -l" % dir_path] - ret, cout = docker_execute(docker_client, master1['Id'], cmd, tty=True) + ret, cout = docker_execute(docker_client, master1["Id"], cmd, tty=True) print(cout) - assert ret == 0 - assert '1' in cout + assert_and_pause((ret == 0, ), cout) + assert_and_pause(("1" in cout, ), cout) diff --git a/tests/integration/backup/ssh/test_restore.py b/tests/integration/backup/ssh/test_restore.py index 636b0ee57..e3d7d0e56 100644 --- a/tests/integration/backup/ssh/test_restore.py +++ b/tests/integration/backup/ssh/test_restore.py @@ -1,20 +1,21 @@ -from tests.integration.conftest import get_twindb_config_dir, docker_execute +import os + +from tests.integration.conftest import get_twindb_config_dir, docker_execute, assert_and_pause from twindb_backup import LOG -def test_restore(master1, storage_server, - config_content_ssh, - docker_client, - rsa_private_key): +def test_restore( + master1, storage_server, config_content_ssh, docker_client, rsa_private_key +): - twindb_config_dir = get_twindb_config_dir(docker_client, master1['Id']) + twindb_config_dir = get_twindb_config_dir(docker_client, master1["Id"]) twindb_config_host = "%s/twindb-backup-1.cfg" % twindb_config_dir - twindb_config_guest = '/etc/twindb/twindb-backup-1.cfg' + twindb_config_guest = "/etc/twindb/twindb-backup-1.cfg" my_cnf_path = "%s/my.cnf" % twindb_config_dir ssh_key_host = "%s/id_rsa" % twindb_config_dir - ssh_key_guest = '/etc/twindb/id_rsa' + ssh_key_guest = "/etc/twindb/id_rsa" contents = """ [client] @@ -26,95 +27,87 @@ def test_restore(master1, storage_server, with open(ssh_key_host, "w") as ssh_fd: ssh_fd.write(rsa_private_key) + os.chmod(ssh_key_host, 0o600) - with open(twindb_config_host, 'w') as fp: + with open(twindb_config_host, "w") as fp: content = config_content_ssh.format( PRIVATE_KEY=ssh_key_guest, - HOST_IP=storage_server['ip'], - MY_CNF='/etc/twindb/my.cnf' + HOST_IP=storage_server["ip"], + MY_CNF="/etc/twindb/my.cnf", ) fp.write(content) cmd = [ - 'twindb-backup', - '--debug', - '--config', twindb_config_guest, - 'backup', - 'daily' + "twindb-backup", + "--debug", + "--config", + twindb_config_guest, + "backup", + "daily", ] - ret, cout = docker_execute(docker_client, master1['Id'], cmd) - LOG.info(cout) - - assert ret == 0 + ret, cout = docker_execute(docker_client, master1["Id"], cmd) + assert_and_pause((ret == 0, ), cout) cmd = [ "bash", "-c", "twindb-backup --config %s ls | grep /tmp/backup " - "| grep mysql | sort | tail -1" % twindb_config_guest + "| grep mysql | sort | tail -1" % twindb_config_guest, ] - ret, cout = docker_execute(docker_client, master1['Id'], cmd) + ret, cout = docker_execute(docker_client, master1["Id"], cmd) url = cout.strip() - LOG.info(cout) - assert ret == 0 + assert_and_pause((ret == 0, ), cout) + dst_dir = "/tmp/ssh_dest_restore/" - cmd = ['twindb-backup', - '--debug', - '--config', twindb_config_guest, - 'restore', - 'mysql', url, - "--dst", dst_dir] - - # print('Test paused') - # print(' '.join(cmd)) - # import time - # time.sleep(36000) - - ret, cout = docker_execute(docker_client, master1['Id'], cmd) - LOG.info(cout) - assert ret == 0 - - cmd = ['find', dst_dir] - ret, cout = docker_execute(docker_client, master1['Id'], cmd) - LOG.info(cout) - assert ret == 0 - - cmd = ['test', '-f', '%s/backup-my.cnf' % dst_dir] - ret, cout = docker_execute(docker_client, master1['Id'], cmd) - LOG.info(cout) - assert ret == 0 - - cmd = ['test', '-f', '%s/ibdata1' % dst_dir] - ret, cout = docker_execute(docker_client, master1['Id'], cmd) - LOG.info(cout) - assert ret == 0 - - cmd = ['test', '-f', '%s/ib_logfile0' % dst_dir] - ret, cout = docker_execute(docker_client, master1['Id'], cmd) - LOG.info(cout) - assert ret == 0 - - cmd = ['test', '-f', '%s/ib_logfile1' % dst_dir] - ret, cout = docker_execute(docker_client, master1['Id'], cmd) - LOG.info(cout) - assert ret == 0 - - cmd = ['test', '-f', '%s/mysql/user.MYD' % dst_dir] - ret, cout = docker_execute(docker_client, master1['Id'], cmd) - LOG.info(cout) - assert ret == 0 - - cmd = ['test', '-f', '%s/xtrabackup_logfile' % dst_dir] - ret, cout = docker_execute(docker_client, master1['Id'], cmd) - LOG.info(cout) - assert ret == 0 + cmd = [ + "twindb-backup", + "--debug", + "--config", + twindb_config_guest, + "restore", + "mysql", + url, + "--dst", + dst_dir, + ] + + + ret, cout = docker_execute(docker_client, master1["Id"], cmd) + assert_and_pause((ret == 0, ), cout) + + cmd = ["find", dst_dir] + ret, cout = docker_execute(docker_client, master1["Id"], cmd) + assert_and_pause((ret == 0, ), cout) + + cmd = ["test", "-f", "%s/backup-my.cnf" % dst_dir] + ret, cout = docker_execute(docker_client, master1["Id"], cmd) + assert_and_pause((ret == 0, ), cout) + + cmd = ["test", "-f", "%s/ibdata1" % dst_dir] + ret, cout = docker_execute(docker_client, master1["Id"], cmd) + assert_and_pause((ret == 0, ), cout) + + cmd = ["test", "-f", "%s/ib_logfile0" % dst_dir] + ret, cout = docker_execute(docker_client, master1["Id"], cmd) + assert_and_pause((ret == 0, ), cout) + + cmd = ["test", "-f", "%s/ib_logfile1" % dst_dir] + ret, cout = docker_execute(docker_client, master1["Id"], cmd) + assert_and_pause((ret == 0, ), cout) + + cmd = ["test", "-f", "%s/mysql/user.MYD" % dst_dir] + ret, cout = docker_execute(docker_client, master1["Id"], cmd) + assert_and_pause((ret == 0, ), cout) + + cmd = ["test", "-f", "%s/xtrabackup_logfile" % dst_dir] + ret, cout = docker_execute(docker_client, master1["Id"], cmd) + assert_and_pause((ret == 0, ), cout) cmd = [ "bash", "-c", - 'test -f %s/_config/etc/my.cnf || test -f %s/_config/etc/mysql/my.cnf' - % (dst_dir, dst_dir) + "test -f %s/_config/etc/my.cnf || test -f %s/_config/etc/mysql/my.cnf" + % (dst_dir, dst_dir), ] - ret, cout = docker_execute(docker_client, master1['Id'], cmd) - LOG.info(cout) - assert ret == 0 + ret, cout = docker_execute(docker_client, master1["Id"], cmd) + assert_and_pause((ret == 0, ), cout) diff --git a/tests/integration/clone/test_clone.py b/tests/integration/clone/test_clone.py index 34f6402b4..a8de87bb0 100644 --- a/tests/integration/clone/test_clone.py +++ b/tests/integration/clone/test_clone.py @@ -1,23 +1,23 @@ import time -from tests.integration.conftest import get_twindb_config_dir, docker_execute, \ - pause_test +from tests.integration.conftest import get_twindb_config_dir, docker_execute, pause_test, assert_and_pause from twindb_backup import INTERVALS, LOG from twindb_backup.source.mysql_source import MySQLConnectInfo from twindb_backup.source.remote_mysql_source import RemoteMySQLSource def test_clone( - runner, - master1, - slave, - docker_client, - config_content_clone, - client_my_cnf, - rsa_private_key): + runner, + master1, + slave, + docker_client, + config_content_clone, + client_my_cnf, + rsa_private_key, +): - twindb_config_dir = get_twindb_config_dir(docker_client, runner['Id']) + twindb_config_dir = get_twindb_config_dir(docker_client, runner["Id"]) twindb_config_host = "%s/twindb-backup-1.cfg" % twindb_config_dir - twindb_config_guest = '/etc/twindb/twindb-backup-1.cfg' + twindb_config_guest = "/etc/twindb/twindb-backup-1.cfg" my_cnf_path = "%s/my.cnf" % twindb_config_dir private_key_host = "%s/private_key" % twindb_config_dir @@ -29,55 +29,54 @@ def test_clone( with open(private_key_host, "w") as key_fd: key_fd.write(rsa_private_key) - with open(twindb_config_host, 'w') as fp: + with open(twindb_config_host, "w") as fp: content = config_content_clone.format( - PRIVATE_KEY=private_key_guest, - MY_CNF='/etc/twindb/my.cnf' + PRIVATE_KEY=private_key_guest, MY_CNF="/etc/twindb/my.cnf" ) fp.write(content) - cmd = '/usr/sbin/sshd' - LOG.info('Run SSH daemon on master1_1') - ret, cout = docker_execute(docker_client, master1['Id'], cmd) - print(cout) + cmd = "/usr/sbin/sshd" + LOG.info("Run SSH daemon on master1_1") + ret, cout = docker_execute(docker_client, master1["Id"], cmd) + assert_and_pause((ret == 0, ), cout) cmd = [ - 'twindb-backup', - '--debug', - '--config', twindb_config_guest, - 'clone', - 'mysql', - '%s:3306' % master1['ip'], - '%s:3306' % slave['ip'] + "twindb-backup", + "--debug", + "--config", + twindb_config_guest, + "clone", + "mysql", + "%s:3306" % master1["ip"], + "%s:3306" % slave["ip"], ] - pause_test(' '.join(cmd)) - ret, cout = docker_execute(docker_client, runner['Id'], cmd) - print(cout) + ret, cout = docker_execute(docker_client, runner["Id"], cmd) + assert_and_pause((ret == 0, ), cout) - assert ret == 0 - sql_master_2 = RemoteMySQLSource({ - "ssh_host": slave['ip'], - "ssh_user": 'root', - "ssh_key": private_key_guest, - "mysql_connect_info": MySQLConnectInfo( - my_cnf_path, - hostname=slave['ip'] - ), - "run_type": INTERVALS[0], - "backup_type": 'full' - }) + sql_master_2 = RemoteMySQLSource( + { + "ssh_host": slave["ip"], + "ssh_user": "root", + "ssh_key": private_key_guest, + "mysql_connect_info": MySQLConnectInfo(my_cnf_path, hostname=slave["ip"]), + "run_type": INTERVALS[0], + "backup_type": "full", + } + ) timeout = time.time() + 30 while time.time() < timeout: with sql_master_2.get_connection() as conn: with conn.cursor() as cursor: - cursor.execute('SHOW SLAVE STATUS') + cursor.execute("SHOW SLAVE STATUS") row = cursor.fetchone() - if row['Slave_IO_Running'] == 'Yes' \ - and row['Slave_SQL_Running'] == 'Yes': + if ( + row["Slave_IO_Running"] == "Yes" + and row["Slave_SQL_Running"] == "Yes" + ): - LOG.info('Replication is up and running') + LOG.info("Replication is up and running") return - LOG.error('Replication is not running after 30 seconds timeout') + LOG.error("Replication is not running after 30 seconds timeout") assert False diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 197976f8b..c50fcc59c 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -13,18 +13,21 @@ from twindb_backup.util import mkdir_p try: - NODE_IMAGE = os.environ['DOCKER_IMAGE'] + NODE_IMAGE = os.environ["DOCKER_IMAGE"] except KeyError: - raise EnvironmentError("""You must define the DOCKER_IMAGE environment + raise EnvironmentError( + """You must define the DOCKER_IMAGE environment variable. Valid values are: * twindb/backup-test:centos-7 * twindb/backup-test:centos-6 * twindb/backup-test:jessie * twindb/backup-test:trusty * twindb/backup-test:xenial - """) + * twindb/backup-test:bionic + """ + ) -NETWORK_NAME = 'test_network' +NETWORK_NAME = "test_network" setup_logging(LOG, debug=True) ensure_aws_creds() @@ -73,21 +76,18 @@ def rsa_private_key(): 8Wt4BxW6kFA7+Su7n8o4DxCqhZYmK9ZUhNjE+uUhxJCJaGr4 -----END RSA PRIVATE KEY----- """ - ) + ).strip() @pytest.fixture(scope="module") def docker_client(): - for _ in xrange(5): + for _ in range(5): try: - return docker.DockerClient( - version="auto", - timeout=600, - ) + return docker.DockerClient(version="auto", timeout=600,) except DockerException as err: LOG.error(err) time.sleep(5) - raise DockerException('Failed to get a docker client') + raise DockerException("Failed to get a docker client") # noinspection PyShadowingNames @@ -97,19 +97,15 @@ def node_image(docker_client): def _ipam_config(): - for octet in xrange(16, 31): + for octet in range(16, 31): subnet = "172.%d.0.0/16" % octet try: - ipam_pool = IPAMPool( - subnet=subnet - ) - ipam_config = IPAMConfig( - pool_configs=[ipam_pool] - ) + ipam_pool = IPAMPool(subnet=subnet) + ipam_config = IPAMConfig(pool_configs=[ipam_pool]) return ipam_config except APIError as err: if err.status_code == 409: - LOG.info('Subnet %s already exists', subnet) + LOG.info("Subnet %s already exists", subnet) continue else: raise @@ -120,113 +116,98 @@ def _ipam_config(): def container_network(docker_client): api = docker_client.api network = None - network_params = { - 'NAME': NETWORK_NAME, - 'subnet': None, - 'second_octet': None - } + network_params = {"NAME": NETWORK_NAME, "subnet": None, "second_octet": None} ipam_config = _ipam_config() - subnet = ipam_config['Config'][0]['Subnet'] - network_params['subnet'] = subnet - network_params['second_octet'] = int(subnet.split('.')[1]) + subnet = ipam_config["Config"][0]["Subnet"] + network_params["subnet"] = subnet + network_params["second_octet"] = int(subnet.split(".")[1]) try: network = api.create_network( - name=NETWORK_NAME, - driver="bridge", - ipam=ipam_config, - check_duplicate=True + name=NETWORK_NAME, driver="bridge", ipam=ipam_config, check_duplicate=True ) - LOG.info('Created subnet %s', network_params['subnet']) + LOG.info("Created subnet %s", network_params["subnet"]) LOG.debug(network) except APIError as err: if err.status_code == 500: - LOG.info('Network %r already exists', network) + LOG.info("Network %r already exists", network) else: raise yield network_params if network: - api.remove_network(net_id=network['Id']) - - -def get_container(name, client, network, - datadir=None, - bootstrap_script=None, - last_n=1, - twindb_config_dir=None, - image=NODE_IMAGE): + api.remove_network(net_id=network["Id"]) + + +def get_container( + name, + client, + network, + datadir=None, + bootstrap_script=None, + last_n=1, + twindb_config_dir=None, + image=NODE_IMAGE, +): api = client.api api.pull(image) cwd = os.getcwd() - LOG.debug('Current directory: %s', cwd) + LOG.debug("Current directory: %s", cwd) - binds = { - cwd: { - 'bind': '/twindb-backup', - 'mode': 'rw', - } - } + binds = {cwd: {"bind": "/twindb-backup", "mode": "rw",}} if twindb_config_dir: - LOG.debug('TwinDB config directory: %s', twindb_config_dir) - mkdir_p(twindb_config_dir, mode=0755) + LOG.debug("TwinDB config directory: %s", twindb_config_dir) + mkdir_p(twindb_config_dir, mode=0o755) binds[twindb_config_dir] = { - 'bind': '/etc/twindb', - 'mode': 'rw', + "bind": "/etc/twindb", + "mode": "rw", } if datadir: binds[datadir] = { - 'bind': '/var/lib/mysql', - 'mode': 'rw', + "bind": "/var/lib/mysql", + "mode": "rw", } host_config = api.create_host_config( - binds=binds, - dns=[ - '8.8.8.8', - '208.67.222.222', - '208.67.220.220' - ] + binds=binds, dns=["8.8.8.8", "208.67.222.222", "208.67.220.220"] ) - ip = '172.%d.3.%d' % (network['second_octet'], last_n) - networking_config = api.create_networking_config({ - network['NAME']: api.create_endpoint_config(ipv4_address=ip) - }) + ip = "172.%d.3.%d" % (network["second_octet"], last_n) + networking_config = api.create_networking_config( + {network["NAME"]: api.create_endpoint_config(ipv4_address=ip)} + ) LOG.debug(networking_config) - container_hostname = '%s_%d' % (name, last_n) + container_hostname = "%s_%d" % (name, last_n) kwargs = { - 'image': image, - 'name': container_hostname, - 'ports': [22, 3306], - 'hostname': container_hostname, - 'host_config': host_config, - 'networking_config': networking_config, - 'volumes': ['/twindb-backup'], - 'environment': {} + "image": image, + "name": container_hostname, + "ports": [22, 3306], + "hostname": container_hostname, + "host_config": host_config, + "networking_config": networking_config, + "volumes": ["/twindb-backup"], + "environment": {}, } try: - kwargs['environment'] = { - 'DEV': os.environ['DEV'] - } + kwargs["environment"] = {"DEV": os.environ["DEV"]} except KeyError: pass if bootstrap_script: - kwargs['command'] = 'bash %s' % bootstrap_script + kwargs["command"] = "bash %s" % bootstrap_script container = api.create_container(**kwargs) - container['ip'] = ip - LOG.info('Created container %r', container) + container["ip"] = ip + LOG.info("Created container %r", container) try: - api.start(container['Id']) - LOG.info('Started %r', container) + api.start(container["Id"]) + LOG.info("Started %r", container) return container except APIError as err: LOG.error(err) - client.api.remove_container(container=container['Id'], force=True) + client.api.remove_container(container=container["Id"], force=True) # noinspection PyShadowingNames @@ -234,55 +215,57 @@ def get_container(name, client, network, def master1(docker_client, container_network, tmpdir_factory): try: - platform = os.environ['PLATFORM'] + platform = os.environ["PLATFORM"] except KeyError: - raise EnvironmentError("""The environment variable PLATFORM + raise EnvironmentError( + """The environment variable PLATFORM must be defined. Allowed values are: * centos * debian * ubuntu - """) + """ + ) - bootstrap_script = '/twindb-backup/support/bootstrap/master/' \ - '%s/master1.sh' % platform - datadir = tmpdir_factory.mktemp('mysql') - twindb_config_dir = tmpdir_factory.mktemp('twindb') + bootstrap_script = ( + "/twindb-backup/support/bootstrap/master/" "%s/master1.sh" % platform + ) + datadir = tmpdir_factory.mktemp("mysql") + twindb_config_dir = tmpdir_factory.mktemp("twindb") container = get_container( - 'master1', + "master1", docker_client, container_network, str(datadir), twindb_config_dir=str(twindb_config_dir), - last_n=1 + last_n=1, ) try: timeout = time.time() + 30 * 60 - LOG.info('Waiting until port TCP/3306 becomes available') + LOG.info("Waiting until port TCP/3306 becomes available") while time.time() < timeout: sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - if sock.connect_ex((container['ip'], 3306)) == 0: + if sock.connect_ex((container["ip"], 3306)) == 0: break time.sleep(1) - LOG.info('Still waiting') + LOG.info("Still waiting") - LOG.info('Port TCP/3306 is ready') + LOG.info("Port TCP/3306 is ready") - privileges_file = "/twindb-backup/vagrant/environment/puppet/" \ - "modules/profile/files/mysql_grants.sql" - cmd = ["bash", "-c", - "mysql -uroot mysql < %s" % privileges_file] - ret, cout = docker_execute(docker_client, container['Id'], cmd) + privileges_file = ( + "/twindb-backup/vagrant/environment/puppet/" + "modules/profile/files/mysql_grants.sql" + ) + cmd = ["bash", "-c", "mysql -uroot mysql < %s" % privileges_file] + ret, cout = docker_execute(docker_client, container["Id"], cmd) print(cout) assert ret == 0 - ret, _ = docker_execute(docker_client, container['Id'], ['ls']) + ret, _ = docker_execute(docker_client, container["Id"], ["ls"]) assert ret == 0 ret, cout = docker_execute( - docker_client, - container['Id'], - ['bash', bootstrap_script] + docker_client, container["Id"], ["bash", bootstrap_script] ) print(cout) assert ret == 0 @@ -290,57 +273,58 @@ def master1(docker_client, container_network, tmpdir_factory): yield container finally: - LOG.info('Removing container %s', container['Id']) - docker_client.api.remove_container( - container=container['Id'], - force=True - ) + if container: + LOG.info("Removing container %s", container["Id"]) + docker_client.api.remove_container(container=container["Id"], force=True) # noinspection PyShadowingNames @pytest.yield_fixture(scope="module") def slave(docker_client, container_network, tmpdir_factory): try: - platform = os.environ['PLATFORM'] + platform = os.environ["PLATFORM"] except KeyError: - raise EnvironmentError("""The environment variable PLATFORM + raise EnvironmentError( + """The environment variable PLATFORM must be defined. Allowed values are: * centos * debian * ubuntu - """) - bootstrap_script = '/twindb-backup/support/bootstrap/master/' \ - '%s/slave.sh' % platform - separator_pos = NODE_IMAGE.find(':') - image_name = NODE_IMAGE[:separator_pos+1] + 'slave_' + NODE_IMAGE[separator_pos+1:] - datadir = tmpdir_factory.mktemp('mysql') - twindb_config_dir = tmpdir_factory.mktemp('twindb') + """ + ) + bootstrap_script = ( + "/twindb-backup/support/bootstrap/master/" "%s/slave.sh" % platform + ) + separator_pos = NODE_IMAGE.find(":") + image_name = ( + NODE_IMAGE[: separator_pos + 1] + "slave_" + NODE_IMAGE[separator_pos + 1 :] + ) + datadir = tmpdir_factory.mktemp("mysql") + twindb_config_dir = tmpdir_factory.mktemp("twindb") container = get_container( - 'slave', + "slave", docker_client, container_network, str(datadir), twindb_config_dir=str(twindb_config_dir), last_n=2, - image=image_name + image=image_name, ) try: timeout = time.time() + 30 * 60 - LOG.info('Waiting until port TCP/22 becomes available') + LOG.info("Waiting until port TCP/22 becomes available") while time.time() < timeout: sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - if sock.connect_ex((container['ip'], 22)) == 0: + if sock.connect_ex((container["ip"], 22)) == 0: break time.sleep(1) - LOG.info('Still waiting') - LOG.info('Port TCP/22 is ready') - ret, _ = docker_execute(docker_client, container['Id'], ['ls']) + LOG.info("Still waiting") + LOG.info("Port TCP/22 is ready") + ret, _ = docker_execute(docker_client, container["Id"], ["ls"]) assert ret == 0 ret, cout = docker_execute( - docker_client, - container['Id'], - ['bash', bootstrap_script] + docker_client, container["Id"], ["bash", bootstrap_script] ) print(cout) assert ret == 0 @@ -349,18 +333,15 @@ def slave(docker_client, container_network, tmpdir_factory): finally: - LOG.info('Removing container %s', container['Id']) - docker_client.api.remove_container( - container=container['Id'], - force=True - ) + LOG.info("Removing container %s", container["Id"]) + docker_client.api.remove_container(container=container["Id"], force=True) # noinspection PyShadowingNames @pytest.yield_fixture(scope="module") def runner(docker_client, container_network, tmpdir_factory): try: - platform = os.environ['PLATFORM'] + platform = os.environ["PLATFORM"] except KeyError: raise EnvironmentError( """The environment variable PLATFORM @@ -370,11 +351,12 @@ def runner(docker_client, container_network, tmpdir_factory): * ubuntu """ ) - bootstrap_script = '/twindb-backup/support/bootstrap/master/' \ - '%s/master1.sh' % platform + bootstrap_script = ( + "/twindb-backup/support/bootstrap/master/" "%s/master1.sh" % platform + ) - datadir = tmpdir_factory.mktemp('mysql') - twindb_config_dir = tmpdir_factory.mktemp('twindb') + datadir = tmpdir_factory.mktemp("mysql") + twindb_config_dir = tmpdir_factory.mktemp("twindb") container = get_container( name="runner", client=docker_client, @@ -382,16 +364,14 @@ def runner(docker_client, container_network, tmpdir_factory): bootstrap_script=bootstrap_script, last_n=3, twindb_config_dir=str(twindb_config_dir), - datadir=datadir + datadir=datadir, ) try: - ret, _ = docker_execute(docker_client, container['Id'], ['ls']) + ret, _ = docker_execute(docker_client, container["Id"], ["ls"]) assert ret == 0 ret, cout = docker_execute( - docker_client, - container['Id'], - ['bash', bootstrap_script] + docker_client, container["Id"], ["bash", bootstrap_script] ) print(cout) assert ret == 0 @@ -400,11 +380,8 @@ def runner(docker_client, container_network, tmpdir_factory): finally: - LOG.info('Removing container %s', container['Id']) - docker_client.api.remove_container( - container=container['Id'], - force=True - ) + LOG.info("Removing container %s", container["Id"]) + docker_client.api.remove_container(container=container["Id"], force=True) def docker_execute(client, container_id, cmd, tty=False): @@ -422,13 +399,13 @@ def docker_execute(client, container_id, cmd, tty=False): :type tty: bool :rtype: tuple(int, str) """ - LOG.debug('Running %s', ' '.join(cmd)) + LOG.debug("Running %s", " ".join(cmd)) api = client.api executor = api.exec_create(container_id, cmd, tty=tty) - exec_id = executor['Id'] + exec_id = executor["Id"] cout = api.exec_start(exec_id) - ret = api.exec_inspect(exec_id)['ExitCode'] - return ret, cout + ret = api.exec_inspect(exec_id)["ExitCode"] + return ret, cout.decode("utf-8") def get_twindb_config_dir(client, container_id): @@ -443,25 +420,37 @@ def get_twindb_config_dir(client, container_id): :rtype: str """ api = client.api - host_config = api.inspect_container(container_id)['HostConfig'] - binds = host_config['Binds'] + host_config = api.inspect_container(container_id)["HostConfig"] + binds = host_config["Binds"] for bind_pair in binds: print(bind_pair) - bind = bind_pair.split(':') + bind = bind_pair.split(":") host_dir = bind[0] guest_dir = bind[1] - if guest_dir == '/etc/twindb': + if guest_dir == "/etc/twindb": return host_dir - raise RuntimeError('Could not find binding for /etc/twindb') + raise RuntimeError("Could not find binding for /etc/twindb") def pause_test(msg): """Pause """ try: - if os.environ['PAUSE_TEST']: - print('Test paused') - print(msg) + if os.environ["PAUSE_TEST"]: + LOG.debug("Test paused") + LOG.debug(msg) import time + time.sleep(36000) except KeyError: + LOG.debug("Define the PAUSE_TEST environment variable if you'd like to pause the test") + LOG.debug("export PAUSE_TEST=1") pass + + +def assert_and_pause(condition, msg): + try: + assert all(condition), msg + except AssertionError as err: + LOG.error(err) + pause_test(msg) + raise err diff --git a/tests/integration/verify/conftest.py b/tests/integration/verify/conftest.py index c77c0e334..3fd3fda1e 100644 --- a/tests/integration/verify/conftest.py +++ b/tests/integration/verify/conftest.py @@ -11,30 +11,29 @@ @pytest.yield_fixture def storage_server(docker_client, container_network): - bootstrap_script = '/twindb-backup/support/bootstrap/storage_server.sh' + bootstrap_script = "/twindb-backup/support/bootstrap/storage_server.sh" container = get_container( - 'storage_server', + "storage_server", docker_client, container_network, bootstrap_script=bootstrap_script, image="centos:centos7", - last_n=3 + last_n=3, ) timeout = time.time() + 30 * 60 while time.time() < timeout: sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - if sock.connect_ex((container['ip'], 22)) == 0: + if sock.connect_ex((container["ip"], 22)) == 0: break time.sleep(1) yield container if container: - LOG.info('Removing container %s', container['Id']) - docker_client.api.remove_container(container=container['Id'], - force=True) + LOG.info("Removing container %s", container["Id"]) + docker_client.api.remove_container(container=container["Id"], force=True) @pytest.fixture diff --git a/tests/integration/verify/test_verify.py b/tests/integration/verify/test_verify.py index f27f971b9..652230bba 100644 --- a/tests/integration/verify/test_verify.py +++ b/tests/integration/verify/test_verify.py @@ -5,22 +5,19 @@ def test_verify_on_master( - master1, slave, - storage_server, - config_content_ssh, - docker_client, - rsa_private_key): + master1, slave, storage_server, config_content_ssh, docker_client, rsa_private_key +): - twindb_config_guest = '/etc/twindb/twindb-backup-1.cfg' + twindb_config_guest = "/etc/twindb/twindb-backup-1.cfg" for cont in master1, slave: - twindb_config_dir = get_twindb_config_dir(docker_client, cont['Id']) + twindb_config_dir = get_twindb_config_dir(docker_client, cont["Id"]) twindb_config_host = "%s/twindb-backup-1.cfg" % twindb_config_dir my_cnf_path = "%s/my.cnf" % twindb_config_dir ssh_key_host = "%s/id_rsa" % twindb_config_dir - ssh_key_guest = '/etc/twindb/id_rsa' + ssh_key_guest = "/etc/twindb/id_rsa" contents = dedent( """ @@ -38,23 +35,24 @@ def test_verify_on_master( ssh_fd.write(rsa_private_key) ssh_fd.flush() - with open(twindb_config_host, 'w') as fp: + with open(twindb_config_host, "w") as fp: content = config_content_ssh.format( PRIVATE_KEY=ssh_key_guest, - HOST_IP=storage_server['ip'], - MY_CNF='/etc/twindb/my.cnf' + HOST_IP=storage_server["ip"], + MY_CNF="/etc/twindb/my.cnf", ) fp.write(content) fp.flush() cmd = [ - 'twindb-backup', - '--debug', - '--config', twindb_config_guest, - 'backup', - 'daily' + "twindb-backup", + "--debug", + "--config", + twindb_config_guest, + "backup", + "daily", ] - ret, cout = docker_execute(docker_client, master1['Id'], cmd) + ret, cout = docker_execute(docker_client, master1["Id"], cmd) LOG.info(cout) assert ret == 0 @@ -63,49 +61,54 @@ def test_verify_on_master( "bash", "-c", "twindb-backup --config %s ls | grep /tmp/backup " - "| grep mysql | sort | tail -1" % twindb_config_guest + "| grep mysql | sort | tail -1" % twindb_config_guest, ] - ret, cout = docker_execute(docker_client, master1['Id'], cmd) + ret, cout = docker_execute(docker_client, master1["Id"], cmd) url = cout.strip() LOG.info(cout) assert ret == 0 cmd = [ - 'twindb-backup', - '--debug', - '--config', twindb_config_guest, - 'verify', - 'mysql', url, + "twindb-backup", + "--debug", + "--config", + twindb_config_guest, + "verify", + "mysql", + url, ] - ret, cout = docker_execute(docker_client, master1['Id'], cmd) + ret, cout = docker_execute(docker_client, master1["Id"], cmd) LOG.info(cout) assert ret == 0 cmd = [ - 'twindb-backup', - '--debug', - '--config', twindb_config_guest, - 'verify', - 'mysql', - 'latest', + "twindb-backup", + "--debug", + "--config", + twindb_config_guest, + "verify", + "mysql", + "latest", ] - ret, cout = docker_execute(docker_client, master1['Id'], cmd) + ret, cout = docker_execute(docker_client, master1["Id"], cmd) LOG.info(cout) assert ret == 0 cmd = [ - 'twindb-backup', - '--debug', - '--config', twindb_config_guest, - 'verify', - 'mysql', - '--hostname', 'master1_1', - 'latest', + "twindb-backup", + "--debug", + "--config", + twindb_config_guest, + "verify", + "mysql", + "--hostname", + "master1_1", + "latest", ] - ret, cout = docker_execute(docker_client, master1['Id'], cmd) + ret, cout = docker_execute(docker_client, master1["Id"], cmd) LOG.info(cout) assert ret == 0 diff --git a/tests/unit/source/test_remote_mysql_source.py b/tests/unit/source/test_remote_mysql_source.py index 5eae2ecfd..5203cf72c 100644 --- a/tests/unit/source/test_remote_mysql_source.py +++ b/tests/unit/source/test_remote_mysql_source.py @@ -1,14 +1,11 @@ import os -from configparser import ConfigParser from os import path as osp -from io import StringIO from pathlib import PurePath, Path from pprint import pformat from textwrap import dedent import mock -import py import pytest from twindb_backup import INTERVALS, LOG diff --git a/twindb_backup/__init__.py b/twindb_backup/__init__.py index 137563f9e..ed110501a 100644 --- a/twindb_backup/__init__.py +++ b/twindb_backup/__init__.py @@ -154,7 +154,7 @@ def get_timeout(run_type): 'monthly': 30 * 24 * 3600 / 2, 'yearly': 365 * 24 * 3600 / 2 } - return timeouts[run_type] + return int(timeouts[run_type]) def save_measures(start_time, end_time, log_path=LOG_FILE): diff --git a/twindb_backup/backup.py b/twindb_backup/backup.py index 7563a860a..c7066cdf2 100644 --- a/twindb_backup/backup.py +++ b/twindb_backup/backup.py @@ -327,7 +327,7 @@ def backup_everything(run_type, twindb_config, binlogs_only=False): save_measures(backup_start, end) else: backup_binlogs(run_type, twindb_config) - except CONFIGPARSER.NoSectionError as err: + except configparser.NoSectionError as err: LOG.debug(traceback.format_exc()) LOG.error(err) exit(1) diff --git a/twindb_backup/destination/local.py b/twindb_backup/destination/local.py index 2b3ad3087..d08f10d9a 100644 --- a/twindb_backup/destination/local.py +++ b/twindb_backup/destination/local.py @@ -84,9 +84,9 @@ def _list_files(self, prefix=None, recursive=False, files_only=False): with run_command(cmd) as cout: if files_only: - return cout.read().split() + return cout.read().decode("utf-8").split() else: - return cout.read().split()[1:] + return cout.read().decode("utf-8").split()[1:] def delete(self, path): cmd = ["rm", path] diff --git a/twindb_backup/source/remote_mysql_source.py b/twindb_backup/source/remote_mysql_source.py index 3d1fd8d3c..a0d30f5ee 100644 --- a/twindb_backup/source/remote_mysql_source.py +++ b/twindb_backup/source/remote_mysql_source.py @@ -142,7 +142,7 @@ def _find_server_id_by_path(cfg): def _save_cfg(self, dst, root_cfg): """Save configs on destination recursively""" - files = self._find_all_cnf(root_cfg) + files = self._find_all_cnf(Path(root_cfg)) server_id = self._get_server_id(dst.host) is_server_id_set = False valid_cfg = [] diff --git a/twindb_backup/ssh/client.py b/twindb_backup/ssh/client.py index 2ef96898d..1e69f398a 100644 --- a/twindb_backup/ssh/client.py +++ b/twindb_backup/ssh/client.py @@ -62,6 +62,7 @@ def _shell(self): shell = SSHClient() shell.set_missing_host_key_policy(AutoAddPolicy()) try: + LOG.debug("Connecting to %s:%d as %s with key %s", self._host, self._port, self._user, self._key) shell.connect( hostname=self._host, key_filename=self._key, @@ -69,7 +70,10 @@ def _shell(self): username=self._user ) yield shell + except FileNotFoundError: + raise except (AuthenticationException, SSHException, socket.error) as err: + # print(type(err)) raise SshClientException(err) finally: shell.close() @@ -119,11 +123,11 @@ def execute(self, cmd, quiet=False, background=False): or channel.recv_stderr_ready(): if channel.recv_ready(): stdout_chunks.append( - channel.recv(max_chunk_size) + channel.recv(max_chunk_size).decode("utf-8") ) if channel.recv_stderr_ready(): stderr_chunks.append( - channel.recv_stderr(max_chunk_size) + channel.recv_stderr(max_chunk_size).decode("utf-8") ) exit_code = channel.recv_exit_status() @@ -211,10 +215,11 @@ def get_text_content(self, path): :return: File content :rtype: str """ + LOG.debug("Reading remote file %s", path) with self._shell() as ssh_client: sftp_client = ssh_client.open_sftp() with sftp_client.open(path) as remote_file: - return remote_file.read() + return remote_file.read().decode("utf-8") def write_content(self, path, content): """ diff --git a/twindb_backup/status/base_status.py b/twindb_backup/status/base_status.py index d2727797d..97e61124b 100644 --- a/twindb_backup/status/base_status.py +++ b/twindb_backup/status/base_status.py @@ -165,7 +165,7 @@ def _read(self, dst): """ try: return dst.read(self.status_path) - except FileNotFound: + except (FileNotFound, FileNotFoundError): return None def _status_serialize(self): @@ -195,10 +195,7 @@ def __init_from_str(self, content): if content == '': raise CorruptedStatus('Status content cannot be an empty string') try: - print("Content") - print(content) status = json.loads(content) - print(status) md5_stored = status['md5'] md5_calculated = hashlib.md5( status['status'].encode("utf-8") @@ -211,8 +208,6 @@ def __init_from_str(self, content): status['status'] ).decode("utf-8") ) - print(self._status) - # 1/0 self._status.sort( key=lambda cp: cp.created_at ) diff --git a/twindb_backup/verify.py b/twindb_backup/verify.py index 6d37d8413..67f9ce829 100644 --- a/twindb_backup/verify.py +++ b/twindb_backup/verify.py @@ -5,9 +5,9 @@ import json import shutil import time -import ConfigParser import tempfile import traceback +from configparser import ConfigParser, NoOptionError from twindb_backup import LOG from twindb_backup.exceptions import TwinDBBackupError @@ -18,13 +18,13 @@ def edit_backup_my_cnf(dst_path): """Removed options from config(besides MySQL 5.7.8)""" filename = "{dir}/backup-my.cnf".format(dir=dst_path) - backup_cfg = ConfigParser.ConfigParser() + backup_cfg = ConfigParser() backup_cfg.read(filename) for option in ['innodb_log_checksum_algorithm', 'innodb_log_block_size', 'innodb_fast_checksum']: try: backup_cfg.remove_option(section='mysqld', option=option) - except ConfigParser.NoOptionError: + except NoOptionError: pass with open(filename, 'w') as backup_fp: backup_cfg.write(backup_fp) diff --git a/vagrant/Vagrantfile b/vagrant/Vagrantfile index f3a266e91..ebaaadefa 100644 --- a/vagrant/Vagrantfile +++ b/vagrant/Vagrantfile @@ -3,12 +3,12 @@ nodes = [ { :hostname => 'master1', :ip => '192.168.36.250', - :box => 'bento/centos-7.3', + :box => 'bento/centos-7', :ram => 3072 }, { :hostname => 'master2', :ip => '192.168.36.251', - :box => 'bento/centos-7.3', + :box => 'bento/centos-7', :ram => 3072 } ] @@ -43,7 +43,12 @@ which puppet || install_puppet SCRIPT + Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| + config.vm.provider "virtualbox" do |v| + v.memory = 1024 + v.cpus = 4 + end nodes.each do |node| config.vm.define node[:hostname] do |nodeconfig| nodeconfig.vm.box = node[:box] @@ -57,7 +62,7 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config| nodeconfig.vm.provider :virtualbox do |vb| vb.customize [ 'modifyvm', :id, - '--cpuexecutioncap', '40', + '--cpuexecutioncap', '90', '--memory', memory.to_s, ] diff --git a/vagrant/environment/puppet/modules/profile/manifests/base.pp b/vagrant/environment/puppet/modules/profile/manifests/base.pp index b6df6a414..40e01d6aa 100644 --- a/vagrant/environment/puppet/modules/profile/manifests/base.pp +++ b/vagrant/environment/puppet/modules/profile/manifests/base.pp @@ -62,8 +62,8 @@ 'Percona-Server-client-56', 'Percona-Server-server-56', 'Percona-Server-devel-56', 'Percona-Server-shared-56', 'percona-toolkit', 'percona-xtrabackup-24', - 'python2-pip', - 'gcc', 'python-devel', 'zlib-devel', 'openssl-devel', + 'python3-devel', 'python3', 'python36-virtualenv', + 'gcc', 'zlib-devel', 'openssl-devel', 'rpm-build','docker', 'strace', 'jq'] package { $packages: @@ -86,12 +86,6 @@ ensure => installed } - package { ['tox', 'awscli']: - ensure => installed, - provider => pip, - require => Package['python2-pip'] - } - service { 'docker': ensure => running, enable => true, From 08830eeeb320a09522b996eb50ca5a5355884f6b Mon Sep 17 00:00:00 2001 From: Aleksandr Kuzminsky Date: Tue, 7 Apr 2020 20:44:05 -0700 Subject: [PATCH 2/3] =?UTF-8?q?Bump=20version:=202.19.0=20=E2=86=92=202.20?= =?UTF-8?q?.0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- omnibus/config/projects/twindb-backup.rb | 2 +- setup.cfg | 2 +- setup.py | 2 +- twindb_backup/__init__.py | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/omnibus/config/projects/twindb-backup.rb b/omnibus/config/projects/twindb-backup.rb index 456b09b47..20fa9f9f8 100644 --- a/omnibus/config/projects/twindb-backup.rb +++ b/omnibus/config/projects/twindb-backup.rb @@ -23,7 +23,7 @@ # and /opt/twindb-backup on all other platforms install_dir '/opt/twindb-backup' -build_version '2.19.0' +build_version '2.20.0' build_iteration 1 diff --git a/setup.cfg b/setup.cfg index 0228863dd..e553e841a 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 2.19.0 +current_version = 2.20.0 commit = True tag = False diff --git a/setup.py b/setup.py index 08a21fe10..86ded8b77 100644 --- a/setup.py +++ b/setup.py @@ -19,7 +19,7 @@ setup( name='twindb-backup', - version='2.19.0', + version='2.20.0', description="TwinDB Backup tool for files, MySQL et al.", long_description=readme + '\n\n' + history, author="TwinDB Development Team", diff --git a/twindb_backup/__init__.py b/twindb_backup/__init__.py index ed110501a..568048d51 100644 --- a/twindb_backup/__init__.py +++ b/twindb_backup/__init__.py @@ -40,7 +40,7 @@ class and saves the backup copy in something defined in a destination class. __author__ = 'TwinDB Development Team' __email__ = 'dev@twindb.com' -__version__ = '2.19.0' +__version__ = '2.20.0' STATUS_FORMAT_VERSION = 1 LOCK_FILE = '/var/run/twindb-backup.lock' LOG_FILE = '/var/log/twindb-backup-measures.log' From 3b44add754f09a5db02c978a26117c905cd3aa8b Mon Sep 17 00:00:00 2001 From: Aleksandr Kuzminsky Date: Tue, 7 Apr 2020 20:46:15 -0700 Subject: [PATCH 3/3] fix lint --- twindb_backup/ssh/client.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/twindb_backup/ssh/client.py b/twindb_backup/ssh/client.py index 1e69f398a..51106ab48 100644 --- a/twindb_backup/ssh/client.py +++ b/twindb_backup/ssh/client.py @@ -62,7 +62,13 @@ def _shell(self): shell = SSHClient() shell.set_missing_host_key_policy(AutoAddPolicy()) try: - LOG.debug("Connecting to %s:%d as %s with key %s", self._host, self._port, self._user, self._key) + LOG.debug( + "Connecting to %s:%d as %s with key %s", + self._host, + self._port, + self._user, + self._key + ) shell.connect( hostname=self._host, key_filename=self._key, @@ -127,7 +133,9 @@ def execute(self, cmd, quiet=False, background=False): ) if channel.recv_stderr_ready(): stderr_chunks.append( - channel.recv_stderr(max_chunk_size).decode("utf-8") + channel.recv_stderr( + max_chunk_size + ).decode("utf-8") ) exit_code = channel.recv_exit_status()