Skip to content

Commit

Permalink
Update test for S3 SSL certificate verification (#814)
Browse files Browse the repository at this point in the history
Updated the test case for S3 SSL certificate verification to use a self-signed certificate generated during the test. This change ensures the test is more reliable, as it doesn't depend on external, possibly changing, data. Improved the test setup by creating and using a temporary certificate file, which gets cleaned up after use.

Signed-off-by: Marcel Coetzee <[email protected]>
  • Loading branch information
Pipboyguy committed Dec 24, 2023
1 parent f620745 commit c998216
Show file tree
Hide file tree
Showing 2 changed files with 55 additions and 13 deletions.
46 changes: 44 additions & 2 deletions tests/load/conftest.py
Original file line number Diff line number Diff line change
@@ -1,9 +1,16 @@
import datetime
import os
import pytest
import tempfile
from typing import Iterator

import pytest
from cryptography import x509
from cryptography.hazmat.backends import default_backend
from cryptography.hazmat.primitives import hashes, serialization
from cryptography.hazmat.primitives.asymmetric import rsa
from cryptography.x509.oid import NameOID

from tests.load.utils import DEFAULT_BUCKETS, ALL_BUCKETS
from tests.utils import preserve_environ


@pytest.fixture(scope="function", params=DEFAULT_BUCKETS)
Expand All @@ -24,3 +31,38 @@ def all_buckets_env(request) -> Iterator[str]:
bucket_url = request.param
os.environ["DESTINATION__FILESYSTEM__BUCKET_URL"] = bucket_url
yield bucket_url


@pytest.fixture(scope="function")
def self_signed_cert() -> Iterator[str]:
key = rsa.generate_private_key(public_exponent=65537, key_size=2048, backend=default_backend())

subject = issuer = x509.Name([
x509.NameAttribute(NameOID.COUNTRY_NAME, "DE"),
x509.NameAttribute(NameOID.STATE_OR_PROVINCE_NAME, "Berlin"),
x509.NameAttribute(NameOID.LOCALITY_NAME, "Britz"),
x509.NameAttribute(NameOID.ORGANIZATION_NAME, "dltHub"),
x509.NameAttribute(NameOID.COMMON_NAME, "localhost"),
])
cert = (
x509.CertificateBuilder()
.subject_name(subject)
.issuer_name(issuer)
.public_key(key.public_key())
.serial_number(x509.random_serial_number())
.not_valid_before(datetime.datetime.now(datetime.timezone.utc))
.not_valid_after(datetime.datetime.now(datetime.timezone.utc) + datetime.timedelta(days=1))
.add_extension(
x509.SubjectAlternativeName([x509.DNSName("localhost")]),
critical=False,
)
.sign(key, hashes.SHA256(), default_backend())
)

with tempfile.NamedTemporaryFile(suffix=".crt", delete=False) as cert_file:
cert_file.write(cert.public_bytes(serialization.Encoding.PEM))
cert_path = cert_file.name

yield cert_path

os.remove(cert_path)
22 changes: 11 additions & 11 deletions tests/load/filesystem/test_filesystem_common.py
Original file line number Diff line number Diff line change
@@ -1,22 +1,19 @@
import os
import posixpath
from ssl import SSLError, SSLCertVerificationError
from typing import Union, Dict

import pytest
from fsspec import AbstractFileSystem

from dlt.common import pendulum
from dlt.common.configuration.inject import with_config
from dlt.common.configuration.specs import AzureCredentials, AzureCredentialsWithoutDefaults
from dlt.common.storages import fsspec_from_config, FilesystemConfiguration
from dlt.common.storages.fsspec_filesystem import MTIME_DISPATCH, glob_files
from dlt.common.utils import uniq_id
from unittest.mock import patch
from tests.common.configuration.utils import environment
from tests.common.storages.utils import assert_sample_files
from tests.load.utils import ALL_FILESYSTEM_DRIVERS
from tests.utils import preserve_environ, autouse_test_storage
from tests.common.configuration.utils import environment


@with_config(spec=FilesystemConfiguration, sections=("destination", "filesystem"))
Expand Down Expand Up @@ -134,22 +131,25 @@ def test_client_kwargs_propagate_to_instance(default_buckets_env: str) -> None:
assert ("foo", "bar") in filesystem.client_kwargs.items()


def test_wrong_client_certificate(default_buckets_env: str) -> None:
@pytest.mark.skipif("s3" not in ALL_FILESYSTEM_DRIVERS, reason="s3 destination not configured")
def test_s3_wrong_client_certificate(default_buckets_env: str, self_signed_cert: str) -> None:
"""Test whether filesystem raises an SSLError when trying to establish
a connection with the wrong client certificate."""

config = get_config()

if config.protocol != "s3":
pytest.skip(f"Not configured to use {config.protocol} protocol.")

config = FilesystemConfiguration(
bucket_url=config.bucket_url,
credentials=config.credentials,
kwargs={"use_ssl": True},
client_kwargs={"verify": "public.crt"},
client_kwargs={"verify": self_signed_cert},
)

from s3fs import S3FileSystem

filesystem: S3FileSystem
filesystem, _ = fsspec_from_config(config)

with pytest.raises(SSLCertVerificationError):
from botocore.exceptions import SSLError

with pytest.raises(SSLError, match="SSL: CERTIFICATE_VERIFY_FAILED"):
print(filesystem.ls("", detail=False))

0 comments on commit c998216

Please sign in to comment.