Adds path stripping to Azure Blob destination, adds hostname to MySQLConfig

README.rst

@@ -126,9 +126,9 @@ Install TwinDB Backup.
 .. code-block:: console
 
     # Download the package
-    wget https://twindb-release.s3.amazonaws.com/twindb-backup/3.3.0/focal/twindb-backup_3.3.0-1_amd64.deb
+    wget https://twindb-release.s3.amazonaws.com/twindb-backup/3.4.0/focal/twindb-backup_3.4.0-1_amd64.deb
     # Install TwinDB Backup
-    apt install ./twindb-backup_3.3.0-1_amd64.deb
+    apt install ./twindb-backup_3.4.0-1_amd64.deb
 
 Configuring TwinDB Backup
 ~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -157,7 +157,7 @@ The package file will be generated in ``omnibus/pkg/``:
 .. code-block:: console
 
     $ ls omnibus/pkg/*.deb
-    omnibus/pkg/twindb-backup_3.3.0-1_amd64.deb
+    omnibus/pkg/twindb-backup_3.4.0-1_amd64.deb
 
 Once the package is built you can install it with rpm/dpkg or upload it to your repository
 and install it with apt or yum.

docs/installation.rst

@@ -38,7 +38,7 @@ The package file will be generated in ``omnibus/pkg/``:
 .. code-block:: console
 
     $ ls omnibus/pkg/*.deb
-    omnibus/pkg/twindb-backup_3.3.0-1_amd64.deb
+    omnibus/pkg/twindb-backup_3.4.0-1_amd64.deb
 
 Once the package is built you can install it with rpm/dpkg or upload it to your repository
 and install it with apt or yum.

docs/usage.rst

@@ -151,6 +151,7 @@ The ``expire_log_days`` options specifies the retention period for MySQL binlogs
     mysql_defaults_file = /etc/twindb/my.cnf
     full_backup = daily
     expire_log_days = 7
+    hostname = localhost # optional, defaults to 127.0.0.1
 
 Backing up MySQL Binlog
 -----------------------

omnibus/config/projects/twindb-backup.rb

@@ -23,7 +23,7 @@
 # and /opt/twindb-backup on all other platforms
 install_dir '/opt/twindb-backup'
 
-build_version '3.3.0'
+build_version '3.4.0'
 
 build_iteration 1
 

setup.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 3.3.0
+current_version = 3.4.0
 commit = True
 tag = False
 

setup.py

@@ -20,7 +20,7 @@
 
 setup(
     name="twindb-backup",
-    version="3.3.0",
+    version="3.4.0",
     description="TwinDB Backup tool for files, MySQL et al.",
     long_description=readme + "\n\n" + history,
     author="TwinDB Development Team",

support/twindb-backup.cfg

@@ -60,8 +60,8 @@ ssh_key=/root/.ssh/id_rsa
 # MySQL
 
 mysql_defaults_file=/etc/twindb/my.cnf
-
 full_backup=daily
+#hostname=localhost # optional, defaults to 127.0.0.1
 
 [retention]
 

tests/unit/configuration/test_mysql.py

@@ -1,10 +1,31 @@
 from twindb_backup.configuration.mysql import MySQLConfig
 
 
-def test_mysql():
+def test_mysql_defaults():
     mc = MySQLConfig()
     assert mc.defaults_file == "/root/.my.cnf"
     assert mc.full_backup == "daily"
+    assert mc.expire_log_days == 7
+    assert mc.xtrabackup_binary is None
+    assert mc.xbstream_binary is None
+    assert mc.hostname == "127.0.0.1"
+
+
+def test_mysql_init():
+    mc = MySQLConfig(
+        mysql_defaults_file="/foo/bar",
+        full_backup="weekly",
+        expire_log_days=3,
+        xtrabackup_binary="/foo/xtrabackup",
+        xbstream_binary="/foo/xbstream",
+        hostname="foo",
+    )
+    assert mc.defaults_file == "/foo/bar"
+    assert mc.full_backup == "weekly"
+    assert mc.expire_log_days == 3
+    assert mc.xtrabackup_binary == "/foo/xtrabackup"
+    assert mc.xbstream_binary == "/foo/xbstream"
+    assert mc.hostname == "foo"
 
 
 def test_mysql_set_xtrabackup_binary():

tests/unit/destination/az/test_init.py

@@ -21,7 +21,7 @@ def test_init_param():
         assert c._connection_string == p.connection_string
         assert c._hostname == p.hostname
         assert c._chunk_size == p.chunk_size
-        assert c._remote_path == p.remote_path
+        assert c._remote_path == p.remote_path.strip("/") if p.remote_path != "/" else p.remote_path
         assert isinstance(c._container_client, ContainerClient)
         az.AZ._connect.assert_called_once()
 

tests/unit/destination/az/test_list_files.py

@@ -35,7 +35,7 @@ def test_list_files_fail():
     with pytest.raises(Exception):
         c._list_files(PREFIX, False, False)
 
-    c._container_client.list_blobs.assert_called_once_with(name_starts_with=PREFIX, include=["metadata"])
+    c._container_client.list_blobs.assert_called_once_with(name_starts_with=PREFIX.strip("/"), include=["metadata"])
 
 
 def test_list_files_files_only():
@@ -47,7 +47,7 @@ def test_list_files_files_only():
 
     assert blob_names == ["blob2", "blob3"]
 
-    c._container_client.list_blobs.assert_called_once_with(name_starts_with=PREFIX, include=["metadata"])
+    c._container_client.list_blobs.assert_called_once_with(name_starts_with=PREFIX.strip("/"), include=["metadata"])
 
 
 def test_list_files_all_files():
@@ -59,7 +59,7 @@ def test_list_files_all_files():
 
     assert blob_names == [b.name for b in BLOBS]
 
-    c._container_client.list_blobs.assert_called_once_with(name_starts_with=PREFIX, include=["metadata"])
+    c._container_client.list_blobs.assert_called_once_with(name_starts_with=PREFIX.strip("/"), include=["metadata"])
 
 
 def test_list_files_recursive():
@@ -71,7 +71,7 @@ def test_list_files_recursive():
     blob_names_recursive = c._list_files(PREFIX, True, False)
 
     assert blob_names == blob_names_recursive
-    c._container_client.list_blobs.assert_called_with(name_starts_with=PREFIX, include=["metadata"])
+    c._container_client.list_blobs.assert_called_with(name_starts_with=PREFIX.strip("/"), include=["metadata"])
 
 
 def test_list_files_prefix():
@@ -84,3 +84,13 @@ def test_list_files_prefix():
     blob_names_recursive = c._list_files(PREFIX, False, False)
 
     assert blob_names == blob_names_recursive
+
+
+def test_list_files_remote_path():
+    """Tests AZ.list_files method, strips remote path from blob names"""
+    c = mocked_az()
+    c._container_client.list_blobs.return_value = BLOBS + [BlobProperties(name="himom/backups/blob4")]
+
+    blob_names = c._list_files(PREFIX, False, True)
+
+    assert blob_names == ["blob2", "blob3", "backups/blob4"]

tests/unit/destination/az/util.py

@@ -14,7 +14,7 @@ def __init__(self, only_required=False) -> None:
         if not only_required:
             self.hostname = "test_host"
             self.chunk_size = 123
-            self.remote_path = "/himom"
+            self.remote_path = "/himom/"
 
     def __iter__(self):
         return iter(self.__dict__)
@@ -33,7 +33,7 @@ def __init__(self, only_required=False) -> None:
 
         if not only_required:
             self.chunk_size = 123
-            self.remote_path = "/himom"
+            self.remote_path = "/himom/"
 
     def __iter__(self):
         return iter(self.__dict__)

twindb_backup/__init__.py

@@ -40,7 +40,7 @@ class and saves the backup copy in something defined in a destination class.
 
 __author__ = "TwinDB Development Team"
 __email__ = "[email protected]"
-__version__ = "3.3.0"
+__version__ = "3.4.0"
 STATUS_FORMAT_VERSION = 1
 LOCK_FILE = "/var/run/twindb-backup.lock"
 LOG_FILE = "/var/log/twindb-backup-measures.log"

twindb_backup/backup.py

@@ -102,7 +102,7 @@ def backup_files(run_type, config: TwinDBBackupConfig):
     )
 
 
-def backup_mysql(run_type, config):
+def backup_mysql(run_type, config: TwinDBBackupConfig):
     """Take backup of local MySQL instance
 
     :param run_type: Run type
@@ -129,8 +129,10 @@ def backup_mysql(run_type, config):
         kwargs["parent_lsn"] = parent.lsn
 
     LOG.debug("Creating source %r", kwargs)
-    mysql_client = MySQLClient(config.mysql.defaults_file)
-    src = MYSQL_SRC_MAP[mysql_client.server_vendor](MySQLConnectInfo(config.mysql.defaults_file), run_type, **kwargs)
+    mysql_client = MySQLClient(defaults_file=config.mysql.defaults_file, hostname=config.mysql.hostname)
+    src = MYSQL_SRC_MAP[mysql_client.server_vendor](
+        MySQLConnectInfo(defaults_file=config.mysql.defaults_file, hostname=config.mysql.hostname), run_type, **kwargs
+    )
 
     callbacks = []
     try:
@@ -173,7 +175,7 @@ def backup_mysql(run_type, config):
         callback[0].callback(**callback[1])
 
 
-def backup_binlogs(run_type, config):  # pylint: disable=too-many-locals
+def backup_binlogs(run_type, config: TwinDBBackupConfig):  # pylint: disable=too-many-locals
     """Copy MySQL binlog files to the backup destination.
 
     :param run_type: Run type
@@ -187,7 +189,7 @@ def backup_binlogs(run_type, config):  # pylint: disable=too-many-locals
 
     dst = config.destination()
     status = BinlogStatus(dst=dst)
-    mysql_client = MySQLClient(defaults_file=config.mysql.defaults_file)
+    mysql_client = MySQLClient(defaults_file=config.mysql.defaults_file, hostname=config.mysql.hostname)
     log_bin_basename = mysql_client.variable("log_bin_basename")
     if log_bin_basename is None:
         return

twindb_backup/configuration/mysql.py

@@ -15,6 +15,7 @@ def __init__(self, **kwargs):
         self._expire_log_days = int(kwargs.get("expire_log_days", 7))
         self._xtrabackup_binary = kwargs.get("xtrabackup_binary")
         self._xbstream_binary = kwargs.get("xbstream_binary")
+        self._hostname = kwargs.get("hostname", "127.0.0.1")
 
     @property
     def defaults_file(self):
@@ -56,3 +57,9 @@ def xbstream_binary(self, path):
         """Set path to xbstream"""
 
         self._xbstream_binary = path
+
+    @property
+    def hostname(self):
+        """MySQL hostname to connect to"""
+
+        return self._hostname

twindb_backup/destination/az.py

@@ -47,7 +47,7 @@ def __init__(
         self._connection_string = connection_string
         self._hostname = hostname
         self._chunk_size = chunk_size
-        self._remote_path = remote_path
+        self._remote_path = remote_path.strip("/") if remote_path != "/" else remote_path
         super(AZ, self).__init__(self._remote_path)
 
         self._container_client = self._connect()
@@ -94,7 +94,7 @@ def render_path(self, path: str) -> str:
         Returns:
             str: Absolute path to the blob in the container
         """
-        return f"{self._remote_path}/{path}"
+        return f"{self._remote_path}/{path}".strip("/")
 
     def _download_to_pipe(self, blob_key: str, pipe_in: int, pipe_out: int) -> None:
         """Downloads a blob from Azure Blob Storage and writes it to a pipe
@@ -226,20 +226,20 @@ def _list_files(self, prefix: str = "", recursive: bool = False, files_only: boo
                 otherwise includes files and directories. Defaults to False.
         """
         LOG.debug(
-            f"""Listing files in container {self._container_name} with prefix={prefix},
+            f"""Listing files in container {self._container_name} with prefix={prefix.strip('/')},
               recursive={recursive}, files_only={files_only}"""
         )
 
         try:
-            blobs = self._container_client.list_blobs(name_starts_with=prefix, include=["metadata"])
+            blobs = self._container_client.list_blobs(name_starts_with=prefix.strip("/"), include=["metadata"])
         except builtins.Exception as err:
             LOG.error(
                 f"Failed to list files in container {self._container_name}. Error: {type(err).__name__}, Reason: {err}"
             )
             raise err
 
         return [
-            blob.name
+            blob.name.strip(self._remote_path).strip("/")
             for blob in blobs
             if not files_only
             or not (bool(blob.get("metadata")) and blob.get("metadata", {}).get("hdi_isfolder") == "true")

twindb_backup/source/mysql_source.py

@@ -216,7 +216,7 @@ def get_stream(self):
             self._xtrabackup,
             "--defaults-file=%s" % self._connect_info.defaults_file,
             "--stream=xbstream",
-            "--host=127.0.0.1",
+            f"--host={self._connect_info.hostname}",
             "--backup",
         ]
         cmd += ["--target-dir", "."]

vagrant/environment/puppet/modules/profile/files/twindb-backup.cfg

@@ -44,6 +44,7 @@ backup_dir=/path/to/twindb-server-backups
 [mysql]
 mysql_defaults_file=/root/.my.cnf
 full_backup=daily
+#hostname=localhost # optional, defaults to 127.0.0.1
 
 # Retention
 [retention]