Support for MySQL SSL options (--mysql-ssl-cert, --mysql-ssl-key, --mysql-ssl-ca)

.gitignore

@@ -49,6 +49,7 @@ coverage.xml
 *.cover
 .hypothesis/
 .pytest_cache/
+tox.*.ini
 
 # Translations
 *.mo

README.md

@@ -65,6 +65,9 @@ Options:
   --mysql-charset TEXT            MySQL database and table character set
                                   [default: utf8mb4]
   --mysql-collation TEXT          MySQL database and table collation
+  --mysql-ssl-ca PATH             Path to SSL CA certificate file.
+  --mysql-ssl-cert PATH           Path to SSL certificate file.
+  --mysql-ssl-key PATH            Path to SSL key file.
   -S, --skip-ssl                  Disable MySQL connection encryption.
   -c, --chunk INTEGER             Chunk reading/writing SQL records
   -l, --log-file PATH             Log file

docs/README.rst

@@ -45,7 +45,10 @@ Connection Options
 - ``-h, --mysql-host TEXT``: MySQL host. Defaults to localhost.
 - ``-P, --mysql-port INTEGER``: MySQL port. Defaults to 3306.
 - ``--mysql-charset TEXT``: MySQL database and table character set. The default is utf8mb4.
-- ``--mysql-collation TEXT``: MySQL database and table collation
+- ``--mysql-collation TEXT``: MySQL database and table collation.
+- ``--mysql-ssl-ca PATH``: Path to SSL CA certificate file.
+- ``--mysql-ssl-cert PATH``: Path to SSL certificate file.
+- ``--mysql-ssl-key PATH``: Path to SSL key file.
 - ``-S, --skip-ssl``: Disable MySQL connection encryption.
 
 Other Options

src/mysql_to_sqlite3/cli.py

@@ -126,6 +126,9 @@
     default=None,
     help="MySQL database and table collation",
 )
+@click.option("--mysql-ssl-ca", type=click.Path(), help="Path to SSL CA certificate file.")
+@click.option("--mysql-ssl-cert", type=click.Path(), help="Path to SSL certificate file.")
+@click.option("--mysql-ssl-key", type=click.Path(), help="Path to SSL key file.")
 @click.option("-S", "--skip-ssl", is_flag=True, help="Disable MySQL connection encryption.")
 @click.option(
     "-c",
@@ -171,6 +174,9 @@ def cli(
     mysql_port: int,
     mysql_charset: str,
     mysql_collation: str,
+    mysql_ssl_ca: t.Optional[str],
+    mysql_ssl_cert: t.Optional[str],
+    mysql_ssl_key: t.Optional[str],
     skip_ssl: bool,
     chunk: int,
     log_file: t.Union[str, "os.PathLike[t.Any]"],
@@ -219,6 +225,9 @@ def cli(
             mysql_port=mysql_port,
             mysql_charset=mysql_charset,
             mysql_collation=mysql_collation,
+            mysql_ssl_ca=mysql_ssl_ca,
+            mysql_ssl_cert=mysql_ssl_cert,
+            mysql_ssl_key=mysql_ssl_key,
             mysql_ssl_disabled=skip_ssl,
             chunk=chunk,
             json_as_text=json_as_text,

src/mysql_to_sqlite3/transporter.py

@@ -100,6 +100,12 @@ def __init__(self, **kwargs: tx.Unpack[MySQLtoSQLiteParams]) -> None:
         if self._without_tables and self._without_data:
             raise ValueError("Unable to continue without transferring data or creating tables!")
 
+        self._mysql_ssl_ca = kwargs.get("mysql_ssl_ca") or None
+
+        self._mysql_ssl_cert = kwargs.get("mysql_ssl_cert") or None
+
+        self._mysql_ssl_key = kwargs.get("mysql_ssl_key") or None
+
         self._mysql_ssl_disabled = bool(kwargs.get("mysql_ssl_disabled", False))
 
         self._current_chunk_number = 0
@@ -135,6 +141,9 @@ def __init__(self, **kwargs: tx.Unpack[MySQLtoSQLiteParams]) -> None:
                 password=self._mysql_password,
                 host=self._mysql_host,
                 port=self._mysql_port,
+                ssl_ca=self._mysql_ssl_ca,
+                ssl_cert=self._mysql_ssl_cert,
+                ssl_key=self._mysql_ssl_key,
                 ssl_disabled=self._mysql_ssl_disabled,
                 charset=self._mysql_charset,
                 collation=self._mysql_collation,

src/mysql_to_sqlite3/types.py

@@ -26,6 +26,9 @@ class MySQLtoSQLiteParams(tx.TypedDict):
     mysql_port: int
     mysql_charset: t.Optional[str]
     mysql_collation: t.Optional[str]
+    mysql_ssl_ca: t.Optional[str]
+    mysql_ssl_cert: t.Optional[str]
+    mysql_ssl_key: t.Optional[str]
     mysql_ssl_disabled: t.Optional[bool]
     mysql_tables: t.Optional[t.Sequence[str]]
     mysql_user: str

tests/conftest.py

@@ -1,10 +1,13 @@
 import json
 import os
 import socket
+import subprocess
+import threading
 import typing as t
+import uuid
 from codecs import open
 from contextlib import contextmanager
-from os.path import abspath, dirname, isfile, join
+from os.path import abspath, basename, dirname, isfile, join
 from pathlib import Path
 from random import choice
 from string import ascii_lowercase, ascii_uppercase, digits
@@ -31,6 +34,7 @@
 from sqlalchemy_utils import database_exists, drop_database
 
 from . import database, factories, models
+from .utils import generate_ssl_certs, stream_logs
 
 
 def pytest_addoption(parser: "Parser"):
@@ -70,6 +74,27 @@ def pytest_addoption(parser: "Parser"):
         help="The TCP port of the MySQL server.",
     )
 
+    parser.addoption(
+        "--mysql-ssl-ca",
+        dest="mysql_ssl_ca",
+        default=None,
+        help="Path to SSL CA certificate file.",
+    )
+
+    parser.addoption(
+        "--mysql-ssl-cert",
+        dest="mysql_ssl_cert",
+        default=None,
+        help="Path to SSL certificate file.",
+    )
+
+    parser.addoption(
+        "--mysql-ssl-key",
+        dest="mysql_ssl_key",
+        default=None,
+        help="Path to SSL key file.",
+    )
+
     parser.addoption(
         "--no-docker",
         dest="use_docker",
@@ -159,10 +184,35 @@ class MySQLCredentials(t.NamedTuple):
     host: str
     port: int
     database: str
+    ssl_ca: t.Optional[str] = None
+    ssl_cert: t.Optional[str] = None
+    ssl_key: t.Optional[str] = None
 
 
 @pytest.fixture(scope="session")
-def mysql_credentials(pytestconfig: Config) -> MySQLCredentials:
+def mysql_credentials(request, pytestconfig: Config, tmp_path_factory: pytest.TempPathFactory) -> MySQLCredentials:
+    ssl_credentials = {
+        "ssl_ca": pytestconfig.getoption("mysql_ssl_ca") or None,
+        "ssl_cert": pytestconfig.getoption("mysql_ssl_cert") or None,
+        "ssl_key": pytestconfig.getoption("mysql_ssl_key") or None,
+    }
+
+    if hasattr(request, "param") and request.param == "ssl":
+        certs_dir = tmp_path_factory.getbasetemp() / "certs"
+        if not certs_dir.exists():
+            certs_dir.mkdir(parents=True)
+            generate_ssl_certs(certs_dir)
+
+            # FIXED: docker perms
+            subprocess.call(["chmod", "0644", str(certs_dir / "ca-key.pem")])
+            subprocess.call(["chmod", "0644", str(certs_dir / "server-key.pem")])
+
+        ssl_credentials = {
+            "ssl_ca": str(certs_dir / "ca.pem"),
+            "ssl_cert": str(certs_dir / "server-cert.pem"),
+            "ssl_key": str(certs_dir / "server-key.pem"),
+        }
+
     db_credentials_file: str = abspath(join(dirname(__file__), "db_credentials.json"))
     if isfile(db_credentials_file):
         with open(db_credentials_file, "r", "utf-8") as fh:
@@ -173,6 +223,9 @@ def mysql_credentials(pytestconfig: Config) -> MySQLCredentials:
                 database=db_credentials["mysql_database"],
                 host=db_credentials["mysql_host"],
                 port=db_credentials["mysql_port"],
+                ssl_ca=db_credentials.get("mysql_ssl_ca") or ssl_credentials["ssl_ca"],
+                ssl_cert=db_credentials.get("mysql_ssl_cert") or ssl_credentials["ssl_cert"],
+                ssl_key=db_credentials.get("mysql_ssl_key") or ssl_credentials["ssl_key"],
             )
 
     port: int = pytestconfig.getoption("mysql_port") or 3306
@@ -188,6 +241,7 @@ def mysql_credentials(pytestconfig: Config) -> MySQLCredentials:
         database=pytestconfig.getoption("mysql_database") or "test_db",
         host=pytestconfig.getoption("mysql_host") or "0.0.0.0",
         port=port,
+        **ssl_credentials,
     )
 
 
@@ -197,6 +251,7 @@ def mysql_instance(mysql_credentials: MySQLCredentials, pytestconfig: Config) ->
     mysql_connection: t.Optional[t.Union[PooledMySQLConnection, MySQLConnection, CMySQLConnection]] = None
     mysql_available: bool = False
     mysql_connection_retries: int = 15  # failsafe
+    ssl_args = {}
 
     db_credentials_file = abspath(join(dirname(__file__), "db_credentials.json"))
     if isfile(db_credentials_file):
@@ -222,33 +277,71 @@ def mysql_instance(mysql_credentials: MySQLCredentials, pytestconfig: Config) ->
             except (HTTPError, NotFound) as err:
                 pytest.fail(str(err))
 
+        ssl_cmds = []
+        ssl_volumes = {}
+        host_certs_dir = None
+        container_certs_dir = "/etc/mysql/certs"
+
+        if mysql_credentials.ssl_ca:
+            host_certs_dir = dirname(mysql_credentials.ssl_ca)
+            ssl_cmds.append(f"--ssl-ca={container_certs_dir}/{basename(mysql_credentials.ssl_ca)}")
+            ssl_args["ssl_ca"] = mysql_credentials.ssl_ca
+
+        if mysql_credentials.ssl_cert:
+            host_certs_dir = dirname(mysql_credentials.ssl_cert)
+            ssl_cmds.append(f"--ssl-cert={container_certs_dir}/{basename(mysql_credentials.ssl_cert)}")
+            ssl_args["ssl_cert"] = f"{host_certs_dir}/client-cert.pem"
+
+        if mysql_credentials.ssl_key:
+            host_certs_dir = dirname(mysql_credentials.ssl_key)
+            ssl_cmds.append(f"--ssl-key={container_certs_dir}/{basename(mysql_credentials.ssl_key)}")
+            ssl_args["ssl_key"] = f"{host_certs_dir}/client-key.pem"
+
+        if host_certs_dir:
+            ssl_volumes[host_certs_dir] = {"bind": container_certs_dir, "mode": "ro"}
+
+        if ssl_args:
+            ssl_args["ssl_verify_cert"] = True
+
+        container_name = f"pytest_mysql_to_sqlite3_{uuid.uuid4().hex[:10]}"
+
         container = client.containers.run(
             image=docker_mysql_image,
-            name="pytest_mysql_to_sqlite3",
+            name=container_name,
             ports={"3306/tcp": (mysql_credentials.host, f"{mysql_credentials.port}/tcp")},
             environment={
                 "MYSQL_RANDOM_ROOT_PASSWORD": "yes",
                 "MYSQL_USER": mysql_credentials.user,
                 "MYSQL_PASSWORD": mysql_credentials.password,
                 "MYSQL_DATABASE": mysql_credentials.database,
             },
+            volumes=ssl_volumes,
             command=[
                 "--character-set-server=utf8mb4",
                 "--collation-server=utf8mb4_unicode_ci",
-            ],
+            ]
+            + ssl_cmds,
             detach=True,
             auto_remove=True,
         )
 
+        log_thread = threading.Thread(target=stream_logs, args=(container,))
+        # The thread will terminate when the main program terminates
+        log_thread.daemon = True
+        log_thread.start()
+
     while not mysql_available and mysql_connection_retries > 0:
         try:
+            print(f"Attempt #{mysql_connection_retries} to connect to MySQL...")
+
             mysql_connection = mysql.connector.connect(
                 user=mysql_credentials.user,
                 password=mysql_credentials.password,
                 host=mysql_credentials.host,
                 port=mysql_credentials.port,
                 charset="utf8mb4",
                 collation="utf8mb4_unicode_ci",
+                **ssl_args,
             )
         except mysql.connector.Error as err:
             if err.errno == errorcode.CR_SERVER_LOST:
@@ -270,6 +363,10 @@ def mysql_instance(mysql_credentials: MySQLCredentials, pytestconfig: Config) ->
     if use_docker and container is not None:
         container.kill()
 
+        # Wait for the log thread to finish (optional)
+        if "log_thread" in locals() and log_thread.is_alive():
+            log_thread.join(timeout=5)
+
 
 @pytest.fixture(scope="session")
 def mysql_database(