Adds path stripping to Azure Blob destination, adds hostname to MySQLConfig

README.rst

@@ -126,9 +126,9 @@ Install TwinDB Backup.
 .. code-block:: console
 
     # Download the package
-    wget https://twindb-release.s3.amazonaws.com/twindb-backup/3.3.0/focal/twindb-backup_3.3.0-1_amd64.deb
+    wget https://twindb-release.s3.amazonaws.com/twindb-backup/3.4.1/focal/twindb-backup_3.4.1-1_amd64.deb
     # Install TwinDB Backup
-    apt install ./twindb-backup_3.3.0-1_amd64.deb
+    apt install ./twindb-backup_3.4.1-1_amd64.deb
 
 Configuring TwinDB Backup
 ~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -157,7 +157,7 @@ The package file will be generated in ``omnibus/pkg/``:
 .. code-block:: console
 
     $ ls omnibus/pkg/*.deb
-    omnibus/pkg/twindb-backup_3.3.0-1_amd64.deb
+    omnibus/pkg/twindb-backup_3.4.1-1_amd64.deb
 
 Once the package is built you can install it with rpm/dpkg or upload it to your repository
 and install it with apt or yum.

docs/installation.rst

@@ -38,7 +38,7 @@ The package file will be generated in ``omnibus/pkg/``:
 .. code-block:: console
 
     $ ls omnibus/pkg/*.deb
-    omnibus/pkg/twindb-backup_3.3.0-1_amd64.deb
+    omnibus/pkg/twindb-backup_3.4.1-1_amd64.deb
 
 Once the package is built you can install it with rpm/dpkg or upload it to your repository
 and install it with apt or yum.

docs/usage.rst

@@ -101,7 +101,25 @@ In the ``[az]`` section you specify Azure credentials as well as Azure Blob Stor
     connection_string = "DefaultEndpointsProtocol=https;AccountName=ACCOUNT_NAME;AccountKey=ACCOUNT_KEY;EndpointSuffix=core.windows.net"
     container_name = twindb-backups
     remote_path = /backups/mysql # optional
+    max_concurrency = 1 # optional
 
+In the ``[az.client]`` section you specify optional Azure Blob Storage client options.
+
+.. code-block:: ini
+
+    [az.client]
+
+    api_version = "2019-02-02"
+    secondary_hostname = "ACCOUNT_NAME-secondary.blob.core.windows.net"
+    max_block_size = 4194304
+    max_single_put_size = 67108864
+    min_large_block_upload_threshold = 4194305
+    use_byte_buffer = true
+    max_page_size = 4194304
+    max_single_get_size = 33554432
+    max_chunk_get_size = 4194304
+    audience = "https://storage.azure.com/"
+    connection_timeout = 20
 
 Google Cloud Storage
 ~~~~~~~~~~~~~~~~~~~~
@@ -151,6 +169,7 @@ The ``expire_log_days`` options specifies the retention period for MySQL binlogs
     mysql_defaults_file = /etc/twindb/my.cnf
     full_backup = daily
     expire_log_days = 7
+    hostname = localhost # optional, defaults to 127.0.0.1
 
 Backing up MySQL Binlog
 -----------------------

omnibus/config/projects/twindb-backup.rb

@@ -23,7 +23,7 @@
 # and /opt/twindb-backup on all other platforms
 install_dir '/opt/twindb-backup'
 
-build_version '3.3.0'
+build_version '3.4.1'
 
 build_iteration 1
 

setup.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 3.3.0
+current_version = 3.4.1
 commit = True
 tag = False
 

setup.py

@@ -20,7 +20,7 @@
 
 setup(
     name="twindb-backup",
-    version="3.3.0",
+    version="3.4.1",
     description="TwinDB Backup tool for files, MySQL et al.",
     long_description=readme + "\n\n" + history,
     author="TwinDB Development Team",

support/twindb-backup.cfg

@@ -38,6 +38,23 @@ BUCKET=twindb-backups
 connection_string="DefaultEndpointsProtocol=https;AccountName=ACCOUNT_NAME;AccountKey=ACCOUNT_KEY;EndpointSuffix=core.windows.net"
 container_name=twindb-backups
 #remote_path = /backups/mysql # optional
+#max_concurrency = 1 # optional
+
+[az.client]
+
+# Azure client optional settings
+
+# api_version="2019-02-02" # optional
+# secondary_hostname="ACCOUNT_NAME-secondary.blob.core.windows.net" # optional
+# max_block_size=4194304 # optional
+# max_single_put_size=67108864 # optional
+# min_large_block_upload_threshold=4194305 # optional
+# use_byte_buffer=true # optional
+# max_page_size=4194304 # optional
+# max_single_get_size=33554432 # optional
+# max_chunk_get_size=4194304 # optional
+# audience="https://storage.azure.com/"  # optional
+# connection_timeout=20 # optional
 
 [gcs]
 
@@ -60,8 +77,8 @@ ssh_key=/root/.ssh/id_rsa
 # MySQL
 
 mysql_defaults_file=/etc/twindb/my.cnf
-
 full_backup=daily
+#hostname=localhost # optional, defaults to 127.0.0.1
 
 [retention]
 

tests/unit/configuration/test_mysql.py

@@ -1,10 +1,31 @@
 from twindb_backup.configuration.mysql import MySQLConfig
 
 
-def test_mysql():
+def test_mysql_defaults():
     mc = MySQLConfig()
     assert mc.defaults_file == "/root/.my.cnf"
     assert mc.full_backup == "daily"
+    assert mc.expire_log_days == 7
+    assert mc.xtrabackup_binary is None
+    assert mc.xbstream_binary is None
+    assert mc.hostname == "127.0.0.1"
+
+
+def test_mysql_init():
+    mc = MySQLConfig(
+        mysql_defaults_file="/foo/bar",
+        full_backup="weekly",
+        expire_log_days=3,
+        xtrabackup_binary="/foo/xtrabackup",
+        xbstream_binary="/foo/xbstream",
+        hostname="foo",
+    )
+    assert mc.defaults_file == "/foo/bar"
+    assert mc.full_backup == "weekly"
+    assert mc.expire_log_days == 3
+    assert mc.xtrabackup_binary == "/foo/xtrabackup"
+    assert mc.xbstream_binary == "/foo/xbstream"
+    assert mc.hostname == "foo"
 
 
 def test_mysql_set_xtrabackup_binary():

tests/unit/conftest.py

@@ -32,6 +32,20 @@ def config_content():
 connection_string="DefaultEndpointsProtocol=https;AccountName=ACCOUNT_NAME;AccountKey=ACCOUNT_KEY;EndpointSuffix=core.windows.net"
 container_name="twindb-backups"
 remote_path="/backups/mysql"
+max_concurrency=1
+
+[az.client]
+api_version="2019-02-02"
+secondary_hostname="ACCOUNT_NAME-secondary.blob.core.windows.net"
+max_block_size=4194304
+max_single_put_size=67108864
+min_large_block_upload_threshold=4194305
+use_byte_buffer=true
+max_page_size=4194304
+max_single_get_size=33554432
+max_chunk_get_size=4194304
+audience="https://storage.azure.com/"
+connection_timeout=20
 
 [gcs]
 GC_CREDENTIALS_FILE="XXXXX"

tests/unit/destination/az/test_config.py

@@ -1,37 +1,148 @@
+from dataclasses import asdict
+
 import pytest
 
-from twindb_backup.configuration.destinations.az import AZConfig
+from twindb_backup.configuration.destinations.az import AZClientConfig, AZConfig, drop_empty_dict_factory
 
-from .util import AZConfigParams
+from .util import AZClientConfigParams, AZConfigParams
 
 
 def test_initialization_success():
     """Test initialization of AZConfig with all parameters set."""
-    p = AZConfigParams()
-    c = AZConfig(**dict(p))
-    assert c.connection_string == p.connection_string
-    assert c.container_name == p.container_name
-    assert c.chunk_size == p.chunk_size
-    assert c.remote_path == p.remote_path
+    client_params = AZClientConfigParams()
+    config_params = AZConfigParams()
+    client_config = AZClientConfig(**dict(client_params))
+
+    c = AZConfig(client_config=client_config, **dict(config_params))
+
+    # AZConfig Assertions
+    assert c.client_config == client_config
+    assert c.connection_string == config_params.connection_string
+    assert c.container_name == config_params.container_name
+    assert (
+        c.remote_path == config_params.remote_path.strip("/")
+        if config_params.remote_path != "/"
+        else config_params.remote_path
+    )
+    assert c.max_concurrency == config_params.max_concurrency
+
+    # AZClientConfig Assertions
+    assert c.client_config.api_version == client_params.api_version
+    assert c.client_config.secondary_hostname == client_params.secondary_hostname
+    assert c.client_config.max_block_size == client_params.max_block_size
+    assert c.client_config.max_single_put_size == client_params.max_single_put_size
+    assert c.client_config.min_large_block_upload_threshold == client_params.min_large_block_upload_threshold
+    assert c.client_config.use_byte_buffer == client_params.use_byte_buffer
+    assert c.client_config.max_page_size == client_params.max_page_size
+    assert c.client_config.max_single_get_size == client_params.max_single_get_size
+    assert c.client_config.max_chunk_get_size == client_params.max_chunk_get_size
+    assert c.client_config.audience == client_params.audience
+    assert c.client_config.connection_timeout == client_params.connection_timeout
 
 
 def test_initialization_success_defaults():
     """Test initialization of AZConfig with only required parameters set and ensure default values."""
-    p = AZConfigParams(only_required=True)
-    c = AZConfig(**dict(p))
-    assert c.connection_string == p.connection_string
-    assert c.container_name == p.container_name
-    assert c.chunk_size == 4 * 1024 * 1024
+    client_params = AZClientConfigParams(only_required=True)
+    config_params = AZConfigParams(only_required=True)
+    client_config = AZClientConfig(**dict(client_params))
+
+    c = AZConfig(client_config=client_config, **dict(config_params))
+
+    # AZConfig Assertions
+    assert c.client_config == client_config
+    assert c.connection_string == config_params.connection_string
+    assert c.container_name == config_params.container_name
     assert c.remote_path == "/"
+    assert c.max_concurrency == 1
+
+    # AZClientConfig Assertions
+    assert c.client_config.api_version == None
+    assert c.client_config.secondary_hostname == None
+    assert c.client_config.max_block_size == 4 * 1024 * 1024  # 4MB
+    assert c.client_config.max_single_put_size == 64 * 1024 * 1024  # 64MB
+    assert c.client_config.min_large_block_upload_threshold == (4 * 1024 * 1024) + 1  # 4MB + 1
+    assert c.client_config.use_byte_buffer == False
+    assert c.client_config.max_page_size == 4 * 1024 * 1024  # 4MB
+    assert c.client_config.max_single_get_size == 32 * 1024 * 1024  # 32MB
+    assert c.client_config.max_chunk_get_size == 4 * 1024 * 1024  # 4MB
+    assert c.client_config.audience == None
+    assert c.client_config.connection_timeout == 20
 
 
 def test_invalid_params():
     """Test initialization of AZConfig with invalid parameters."""
-    with pytest.raises(ValueError):
+
+    # Invalidate AZConfig
+    with pytest.raises(ValueError):  # Invalid client_config
+        AZConfig(client_config={}, connection_string="test_connection_string", container_name="test_container")
+    with pytest.raises(ValueError):  # Invalid connection_string
+        AZConfig(client_config=AZClientConfig(), connection_string=123, container_name="test_container")
+    with pytest.raises(ValueError):  # Invalid remote_path
         AZConfig(
-            connection_string="test_connection_string", container_name="test_container", chunk_size="invalid_chunk_size"
+            client_config=AZClientConfig(),
+            connection_string="test_connection_string",
+            container_name="test_container",
+            remote_path=1,
         )
-    with pytest.raises(ValueError):
-        AZConfig(connection_string="test_connection_string", container_name="test_container", remote_path=1)
-    with pytest.raises(TypeError):
-        AZConfig(connection_string="test_connection_string")
+    with pytest.raises(ValueError):  # Invalid container_name
+        AZConfig(client_config=AZClientConfig(), connection_string="test_connection_string", container_name=1)
+    with pytest.raises(ValueError):  # Invalid max_concurrency
+        AZConfig(
+            client_config=AZClientConfig(),
+            connection_string="test_connection_string",
+            container_name="test_container",
+            max_concurrency="1",
+        )
+
+    # Invalidate AZClientConfig
+    with pytest.raises(ValueError):  # Invalid api_version
+        AZClientConfig(api_version=123)
+    with pytest.raises(ValueError):  # Invalid secondary_hostname
+        AZClientConfig(secondary_hostname=123)
+    with pytest.raises(ValueError):  # Invalid max_block_size
+        AZClientConfig(max_block_size="123")
+    with pytest.raises(ValueError):  # Invalid max_single_put_size
+        AZClientConfig(max_single_put_size="123")
+    with pytest.raises(ValueError):  # Invalid min_large_block_upload_threshold
+        AZClientConfig(min_large_block_upload_threshold="123")
+    with pytest.raises(ValueError):  # Invalid use_byte_buffer
+        AZClientConfig(use_byte_buffer="123")
+    with pytest.raises(ValueError):  # Invalid max_page_size
+        AZClientConfig(max_page_size="123")
+    with pytest.raises(ValueError):  # Invalid max_single_get_size
+        AZClientConfig(max_single_get_size="123")
+    with pytest.raises(ValueError):  # Invalid max_chunk_get_size
+        AZClientConfig(max_chunk_get_size="123")
+    with pytest.raises(ValueError):  # Invalid audience
+        AZClientConfig(audience=123)
+    with pytest.raises(ValueError):  # Invalid connection_timeout
+        AZClientConfig(connection_timeout="123")
+
+
+def test_drop_empty_dicts_some_undefined():
+    """Test drop_empty_dict_factory helper function."""
+
+    client_config = AZClientConfig(**dict(AZClientConfigParams(only_required=True)))
+
+    # Convert to dict and drop attributes with None values
+    client_config_dict = asdict(client_config, dict_factory=drop_empty_dict_factory)
+
+    # Assert that the dict does not contain any None values
+    assert "api_version" not in client_config_dict
+    assert "secondary_hostname" not in client_config_dict
+    assert "audience" not in client_config_dict
+
+
+def test_drop_empty_dicts_all_defined():
+    """Test drop_empty_dict_factory helper function doesn't drop any attributes when all are defined."""
+
+    client_config = AZClientConfig(**dict(AZClientConfigParams()))
+
+    # Convert to dict and drop attributes with None values
+    client_config_dict_drop_empty = asdict(client_config, dict_factory=drop_empty_dict_factory)
+
+    # Convert to dict
+    client_config_dict = asdict(client_config)
+
+    # Assert that the dicts are the same
+    assert client_config_dict == client_config_dict_drop_empty

tests/unit/destination/az/test_delete.py

@@ -10,14 +10,14 @@ def test_delete_success():
     c = mocked_az()
 
     c.delete("test")
-    c._container_client.delete_blob.assert_called_once_with(c.render_path("test"))
+    c.container_client.delete_blob.assert_called_once_with(c.render_path("test"))
 
 
 def test_delete_fail():
     """Tests AZ.delete method, re-raising an exception on failure"""
     c = mocked_az()
-    c._container_client.delete_blob.side_effect = Exception()
+    c.container_client.delete_blob.side_effect = Exception()
 
     with pytest.raises(Exception):
         c.delete("test")
-    c._container_client.delete_blob.assert_called_once_with(c.render_path("test"))
+    c.container_client.delete_blob.assert_called_once_with(c.render_path("test"))

tests/unit/destination/az/test_download_to_pipe.py

@@ -15,13 +15,15 @@ def test_download_to_pipe_success():
         c = mocked_az()
 
         mc_dbr = MagicMock()
-        c._container_client.download_blob.return_value = mc_dbr
+        c.container_client.download_blob.return_value = mc_dbr
 
         c._download_to_pipe(c.render_path("foo-key"), 100, 200)
 
         mc_os.close.assert_called_once_with(100)
         mc_os.fdopen.assert_called_once_with(200, "wb")
-        c._container_client.download_blob.assert_called_once_with(c.render_path("foo-key"))
+        c.container_client.download_blob.assert_called_once_with(
+            c.render_path("foo-key"), max_concurrency=c.config.max_concurrency
+        )
         mc_dbr.readinto.assert_called_once_with(mc_fdopen.__enter__())
 
 
@@ -30,11 +32,13 @@ def test_download_to_pipe_fail():
     with patch("twindb_backup.destination.az.os") as mc_os:
         c = mocked_az()
 
-        c._container_client.download_blob.side_effect = ae.HttpResponseError()
+        c.container_client.download_blob.side_effect = ae.HttpResponseError()
 
         with pytest.raises(Exception):
             c._download_to_pipe(c.render_path("foo-key"), 100, 200)
 
         mc_os.close.assert_called_once_with(100)
         mc_os.fdopen.assert_called_once_with(200, "wb")
-        c._container_client.download_blob.assert_called_once_with(c.render_path("foo-key"))
+        c.container_client.download_blob.assert_called_once_with(
+            c.render_path("foo-key"), max_concurrency=c.config.max_concurrency
+        )