From a0cb7dac230202ff216da25680572634a0cdc1c1 Mon Sep 17 00:00:00 2001 From: Skynet Date: Mon, 9 Oct 2023 08:18:00 +0200 Subject: [PATCH 1/6] Fix: read pkey as DER fmt instead of PEM --- dlt/destinations/snowflake/configuration.py | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/dlt/destinations/snowflake/configuration.py b/dlt/destinations/snowflake/configuration.py index 9124fa4f9f..a2e8fb7b4a 100644 --- a/dlt/destinations/snowflake/configuration.py +++ b/dlt/destinations/snowflake/configuration.py @@ -23,21 +23,24 @@ def _read_private_key(private_key: str, password: Optional[str] = None) -> bytes from cryptography.hazmat.primitives.asymmetric import rsa from cryptography.hazmat.primitives.asymmetric import dsa from cryptography.hazmat.primitives import serialization + from cryptography.hazmat.primitives.asymmetric.types import PrivateKeyTypes except ModuleNotFoundError as e: raise MissingDependencyException("SnowflakeCredentials with private key", dependencies=[f"{version.DLT_PKG_NAME}[snowflake]"]) from e + load_key_func = serialization.load_pem_private_key + private_key_bytes_decoded = private_key.encode() try: # decode base64 encoded private key - private_key_decoded = base64.b64decode(private_key).decode('utf-8') + private_key_bytes_decoded = base64.b64decode(private_key) + load_key_func = serialization.load_der_private_key except binascii.Error: - # private key is not in base64 -> assume it's already provided in plain-text - private_key_decoded = private_key - except UnicodeDecodeError: - # private key cannot be decoded into utf-8 -> assume it's already provided in plain-text - private_key_decoded = private_key - - pkey = serialization.load_pem_private_key( - private_key_decoded.encode(), password.encode() if password is not None else None, backend=default_backend() + # cannot base64-decode private key -> assume it's been provided as plain text + pass + + pkey: PrivateKeyTypes = load_key_func( + private_key_bytes_decoded, + password=password.encode() if password is not None else None, + backend=default_backend(), ) return pkey.private_bytes( From 4bb541b389b51d44f49c48e9d469224fa35f43cc Mon Sep 17 00:00:00 2001 From: Skynet Date: Mon, 9 Oct 2023 08:18:14 +0200 Subject: [PATCH 2/6] Fix: update tests --- tests/common/cases/secrets/encrypted-private-key-base64 | 2 +- tests/load/snowflake/test_snowflake_configuration.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/common/cases/secrets/encrypted-private-key-base64 b/tests/common/cases/secrets/encrypted-private-key-base64 index 374021bd5d..b4eda0cfd2 100644 --- a/tests/common/cases/secrets/encrypted-private-key-base64 +++ b/tests/common/cases/secrets/encrypted-private-key-base64 @@ -1 +1 @@ -LS0tLS1CRUdJTiBFTkNSWVBURUQgUFJJVkFURSBLRVktLS0tLQpNSUlGSERCT0Jna3Foa2lHOXcwQkJRMHdRVEFwQmdrcWhraUc5dzBCQlF3d0hBUUlWd3E3cFY5Vm93MENBZ2dBCk1Bd0dDQ3FHU0liM0RRSUpCUUF3RkFZSUtvWklodmNOQXdjRUNKUENXSXFNQXo1dkJJSUV5Q2EyNFpoeE9xSGEKUHJWeHF0Y1lzdTQvY1ZvUWZGMVlTcnB1eUh5WDlNcjZaVU5NYitRSkRFNTY4R0ZTVU82SndGRG1yc2U2K2V2RQovZUpDSUVYdHJ0OC90N0JrWU5MSlFVeS9LMWlxSEhFZENRYXZmYUkzU0E1bEZ1RElwTzlyM2w5LzM5czZWaFFoCnlMOGIwU2tYdGFnS2hqbGhJcFlXdHMxcFJTU2VreFpKTnk5N0pKVHRJRDRRUUtJRTF1TExBQ1JVQlRwczlENVYKdmNNN1d1dXBVbEdwSmZzclltSHBYZ2lPMzBLd2VZSVFuc0NMcHgyNmgzd1BhN3FLYzdTc2xNTitRWW51WEt5MQp6bFFGRW9YWnN1ZkM5bEE5bTVhbG40RDJvaGdob1J4ckRZQlMwVmFWb1dEM1ZUK1A1bG5FNHpyK2xjQi9Ud2FDCkZlWUJoT0x2Tm15eXloZTB1cGpGaVV6YVdqZGZyTFJnSXRpUUNjd3J2T1ZaUmp3MFVPTGlDN0E3VGR6cjZhVk0KNjYwUzZZaTlXRXVIN1RvZHBGMW15YjQrTDgwVjVEclVSd0lDTzcxNGQ3MFFtMTUvQ2Q5Y3hFVHlFb2g1UDdGQworMjBDQ2R6SW0vQTBsVTgwZjg1bDZyOVJIaEFpVWtSbjhaZXhqeU5QUUlpRUloYVlYSXFwNEhJS1BocmVmUTZQCjgwdEM4Rm5NRzVqaXhnbTZBNmt3RTh1NGVoaTlWeE1FQklTaS9WWW9Bdm9hSUFIWVJqOFZGM29MaWxhUjJ2Y0EKRkQ5UDlrZVdwWjFiOEVtc2N3ZnBWWGszK1U0ZWEyeE0zdXVBSEFnOG5FQ0wxNVdxNURFRWxtbmlPTkJMOEE0NAp5RDhFcmcwYWdTeWNKM3ZROFRLQTNiby81K0NFOWZLbWJlZ01JZkhpSXpKZGNIWGFDUlRWM2wrLzlNSHNIQmEvCjZURTdib2JTTFIxUkxVSm5Fd0VmNHJBWjZIbWxMR1ZJdGRsZlZHS3FSSXRKZWdrLzY0RjMvblozeGN2R2Fvc2YKQ0l1TzF5S3FiMUxKWm5ZNDcyeTVRbnpzMmF2L1E0ZG5YL3BCMDdQbUwwUFVsTFArK3BXUTNJM2NTOFQrVTZQRgpMSE9RY2pFY2doMUdpVk1zbU9DQlNKMHBHbVFhWXlaS0E5bkp1aFFHblFrRzFDQlhqNTZLV3JvcXpIWlVhdVI0CkpscUZuaUZXeUZPT2hsd2J6KzRDYlIxYkpFKzZra05xUFcvb003b05WT0ozS1pFMlVNcGxrWVRLR1U5MEdlUEgKNEdpcVYxbjA2QVRnWG53TjRXUE1EV09NY0NmY1RjcGE4VGk3VTNIbTViQU9NZ3lXektqUUxZTkRabEhFUDhTVgpIYWtqNkU0cmc1bkpQcllLWHgrMURhS3FQZS9DMEdvaDRNZ0VmSTJUSHhkYmVvdy9MU3NkZDV5dUlEcnNvd3hGCmEwNm1HdXB0Ym5JSXpRVkJjcHBCSysyTHhuL0RFN0xOb3ZKT2d3cWs2VzhLa3NQSnRUYU9JYndUcmM4N1dmSnQKVDFiWTAzWGJCak8rRkd4L3pESmxwT1l2NmlNZkJ6WlM2Q3lYZXR4Z2pwZGxEbGMyUjc3N3ZaY2lKQkJhazJqbApEUXZiV05OZVZGdGVDVEFEN3c4WWpCd2FrWEdZNUxKYy9icC9MZzZIUG5vMkd0bCtnRWUvbkhUcElBbHptNWI1CjBPZ083eXNxb0F3Rm83cVVhbStGQ21ZUHpoVW5ZS2FqSHVZb3g2bm1aaTFqSkVPNVc2Y1QxdVJBRmdRb2czM1MKTUpBNEM3S0xHejlPcmRTZkQ2bE9vMllBdmxIeE1lS3Y4OTJXUVh0Q2xTZ1hrOVBTcGpwbjJ1S2Q1UThsd2c5QQpiYy9jMVo3Nm9PL3FadW05TTdyMDQzbFprT2RkeEd1VksyNlNPMTE0M21XaEV5U2VGbEtySU84dHV3bVN2dHJRCkU5WE1DeVJkd0YrKzY4ZmpEbHRwTm5CMmpldm1wbjRIQ1c3WGd6eEVWeGUzN0JPRGkwRTM5VTJiTndwN05seFcKQ1gra2w5Z0pRY25iMk1tVWpMNXFRNXk4b1U1aWo4L3FZb0YySHBuZWJqYjlpd1RJRDBsMFJyZS9hSEFqU01vVgpQTmxPZWFSZTZZK0QrT2RjWlhPL1ZnPT0KLS0tLS1FTkQgRU5DUllQVEVEIFBSSVZBVEUgS0VZLS0tLS0= +MIIBVAIBADANBgkqhkiG9w0BAQEFAASCAT4wggE6AgEAAkEA2S4VyM76HJroL3w1bGa42RPFSRZpIJ8nck+xUmtEKAr/hHa6U420NviFd4HixWN58bapaV1G/ARlYPt3O5H1zwIDAQABAkEAgYTGYRdueZi4V3lPsvIf0vXDdi7zSw0fTUDRk0F9SBMNp9QKLDdMFD//4vOsk7vfw+Xq8UFHwH6imjY26xFT4QIhAPMd8czsaq99AdTuhAMoUXg5n+1I8QE2crpFe1VrnSwTAiEA5LBJi4tV5ZqTVp+oVWmcUdcShjZnPE1LpEo1sMb3ztUCIHdwPavkdiFS88RtjZOTijrbbJOBo01qN4EFw6ranLPbAiB3r1deQ9b5qnNwA3D6+qHIJzJXsYNICQAMhJYi5Y7t3QIgHVwCvPP/FyI6Pt9be3nowWrawqmdlClf5pzxRkYDWYE= \ No newline at end of file diff --git a/tests/load/snowflake/test_snowflake_configuration.py b/tests/load/snowflake/test_snowflake_configuration.py index e097575936..7af2af2067 100644 --- a/tests/load/snowflake/test_snowflake_configuration.py +++ b/tests/load/snowflake/test_snowflake_configuration.py @@ -65,7 +65,7 @@ def test_to_connector_params() -> None: creds = SnowflakeCredentials() creds.private_key = pkey_str # type: ignore[assignment] - creds.private_key_passphrase = '12345' # type: ignore[assignment] + creds.private_key_passphrase = 'insecure' # type: ignore[assignment] creds.username = 'user1' creds.database = 'db1' creds.host = 'host1' From 1f12bd6e26ec2c5e56f4f42c8edfe8344ef69a62 Mon Sep 17 00:00:00 2001 From: Skynet Date: Mon, 9 Oct 2023 22:04:30 +0200 Subject: [PATCH 3/6] Fix: clean up key-loading logic --- dlt/destinations/snowflake/configuration.py | 27 ++++++++++----------- 1 file changed, 13 insertions(+), 14 deletions(-) diff --git a/dlt/destinations/snowflake/configuration.py b/dlt/destinations/snowflake/configuration.py index a2e8fb7b4a..981ee1df91 100644 --- a/dlt/destinations/snowflake/configuration.py +++ b/dlt/destinations/snowflake/configuration.py @@ -27,22 +27,21 @@ def _read_private_key(private_key: str, password: Optional[str] = None) -> bytes except ModuleNotFoundError as e: raise MissingDependencyException("SnowflakeCredentials with private key", dependencies=[f"{version.DLT_PKG_NAME}[snowflake]"]) from e - load_key_func = serialization.load_pem_private_key - private_key_bytes_decoded = private_key.encode() try: - # decode base64 encoded private key - private_key_bytes_decoded = base64.b64decode(private_key) - load_key_func = serialization.load_der_private_key - except binascii.Error: - # cannot base64-decode private key -> assume it's been provided as plain text - pass + # load key from base64-encoded DER key + pkey = serialization.load_der_private_key( + base64.b64decode(private_key), + password=password.encode() if password is not None else None, + backend=default_backend(), + ) + except Exception: + # loading base64-encoded DER key failed -> assume it's a plain-text PEM key + serialization.load_pem_private_key( + private_key.encode(), + password=password.encode() if password is not None else None, + backend=default_backend(), + ) - pkey: PrivateKeyTypes = load_key_func( - private_key_bytes_decoded, - password=password.encode() if password is not None else None, - backend=default_backend(), - ) - return pkey.private_bytes( encoding=serialization.Encoding.DER, format=serialization.PrivateFormat.PKCS8, From 49908eae9ae7fdd819b154d1155e70c175115ba4 Mon Sep 17 00:00:00 2001 From: Skynet Date: Mon, 9 Oct 2023 23:31:09 +0200 Subject: [PATCH 4/6] Fix: please linter --- dlt/destinations/snowflake/configuration.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dlt/destinations/snowflake/configuration.py b/dlt/destinations/snowflake/configuration.py index 981ee1df91..b8af1a4a91 100644 --- a/dlt/destinations/snowflake/configuration.py +++ b/dlt/destinations/snowflake/configuration.py @@ -35,13 +35,13 @@ def _read_private_key(private_key: str, password: Optional[str] = None) -> bytes backend=default_backend(), ) except Exception: - # loading base64-encoded DER key failed -> assume it's a plain-text PEM key + # loading base64-encoded DER key failed -> assume it's a plain-text PEM key serialization.load_pem_private_key( private_key.encode(), password=password.encode() if password is not None else None, backend=default_backend(), ) - + return pkey.private_bytes( encoding=serialization.Encoding.DER, format=serialization.PrivateFormat.PKCS8, From c8c4f21b413265faddcac631cc7f8aa525f89bc7 Mon Sep 17 00:00:00 2001 From: Skynet Date: Tue, 10 Oct 2023 10:36:47 +0200 Subject: [PATCH 5/6] Fix: missing variable assignment --- dlt/destinations/snowflake/configuration.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dlt/destinations/snowflake/configuration.py b/dlt/destinations/snowflake/configuration.py index b8af1a4a91..473c6c5cb2 100644 --- a/dlt/destinations/snowflake/configuration.py +++ b/dlt/destinations/snowflake/configuration.py @@ -36,7 +36,7 @@ def _read_private_key(private_key: str, password: Optional[str] = None) -> bytes ) except Exception: # loading base64-encoded DER key failed -> assume it's a plain-text PEM key - serialization.load_pem_private_key( + pkey = serialization.load_pem_private_key( private_key.encode(), password=password.encode() if password is not None else None, backend=default_backend(), From c4e64af263150793ff4906d2e0cedaa9fe6a7b70 Mon Sep 17 00:00:00 2001 From: Marcin Rudolf Date: Tue, 10 Oct 2023 13:54:24 +0200 Subject: [PATCH 6/6] provides valid DER key in test --- dlt/destinations/snowflake/configuration.py | 2 +- tests/common/cases/secrets/encrypted-private-key-base64 | 2 +- tests/load/snowflake/test_snowflake_configuration.py | 4 +++- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/dlt/destinations/snowflake/configuration.py b/dlt/destinations/snowflake/configuration.py index 473c6c5cb2..829b492127 100644 --- a/dlt/destinations/snowflake/configuration.py +++ b/dlt/destinations/snowflake/configuration.py @@ -37,7 +37,7 @@ def _read_private_key(private_key: str, password: Optional[str] = None) -> bytes except Exception: # loading base64-encoded DER key failed -> assume it's a plain-text PEM key pkey = serialization.load_pem_private_key( - private_key.encode(), + private_key.encode(encoding="ascii"), password=password.encode() if password is not None else None, backend=default_backend(), ) diff --git a/tests/common/cases/secrets/encrypted-private-key-base64 b/tests/common/cases/secrets/encrypted-private-key-base64 index b4eda0cfd2..919d803230 100644 --- a/tests/common/cases/secrets/encrypted-private-key-base64 +++ b/tests/common/cases/secrets/encrypted-private-key-base64 @@ -1 +1 @@ -MIIBVAIBADANBgkqhkiG9w0BAQEFAASCAT4wggE6AgEAAkEA2S4VyM76HJroL3w1bGa42RPFSRZpIJ8nck+xUmtEKAr/hHa6U420NviFd4HixWN58bapaV1G/ARlYPt3O5H1zwIDAQABAkEAgYTGYRdueZi4V3lPsvIf0vXDdi7zSw0fTUDRk0F9SBMNp9QKLDdMFD//4vOsk7vfw+Xq8UFHwH6imjY26xFT4QIhAPMd8czsaq99AdTuhAMoUXg5n+1I8QE2crpFe1VrnSwTAiEA5LBJi4tV5ZqTVp+oVWmcUdcShjZnPE1LpEo1sMb3ztUCIHdwPavkdiFS88RtjZOTijrbbJOBo01qN4EFw6ranLPbAiB3r1deQ9b5qnNwA3D6+qHIJzJXsYNICQAMhJYi5Y7t3QIgHVwCvPP/FyI6Pt9be3nowWrawqmdlClf5pzxRkYDWYE= \ No newline at end of file +MIIFHDBOBgkqhkiG9w0BBQ0wQTApBgkqhkiG9w0BBQwwHAQIVwq7pV9Vow0CAggAMAwGCCqGSIb3DQIJBQAwFAYIKoZIhvcNAwcECJPCWIqMAz5vBIIEyCa24ZhxOqHaPrVxqtcYsu4/cVoQfF1YSrpuyHyX9Mr6ZUNMb+QJDE568GFSUO6JwFDmrse6+evE/eJCIEXtrt8/t7BkYNLJQUy/K1iqHHEdCQavfaI3SA5lFuDIpO9r3l9/39s6VhQhyL8b0SkXtagKhjlhIpYWts1pRSSekxZJNy97JJTtID4QQKIE1uLLACRUBTps9D5VvcM7WuupUlGpJfsrYmHpXgiO30KweYIQnsCLpx26h3wPa7qKc7SslMN+QYnuXKy1zlQFEoXZsufC9lA9m5aln4D2ohghoRxrDYBS0VaVoWD3VT+P5lnE4zr+lcB/TwaCFeYBhOLvNmyyyhe0upjFiUzaWjdfrLRgItiQCcwrvOVZRjw0UOLiC7A7Tdzr6aVM660S6Yi9WEuH7TodpF1myb4+L80V5DrURwICO714d70Qm15/Cd9cxETyEoh5P7FC+20CCdzIm/A0lU80f85l6r9RHhAiUkRn8ZexjyNPQIiEIhaYXIqp4HIKPhrefQ6P80tC8FnMG5jixgm6A6kwE8u4ehi9VxMEBISi/VYoAvoaIAHYRj8VF3oLilaR2vcAFD9P9keWpZ1b8EmscwfpVXk3+U4ea2xM3uuAHAg8nECL15Wq5DEElmniONBL8A44yD8Erg0agSycJ3vQ8TKA3bo/5+CE9fKmbegMIfHiIzJdcHXaCRTV3l+/9MHsHBa/6TE7bobSLR1RLUJnEwEf4rAZ6HmlLGVItdlfVGKqRItJegk/64F3/nZ3xcvGaosfCIuO1yKqb1LJZnY472y5Qnzs2av/Q4dnX/pB07PmL0PUlLP++pWQ3I3cS8T+U6PFLHOQcjEcgh1GiVMsmOCBSJ0pGmQaYyZKA9nJuhQGnQkG1CBXj56KWroqzHZUauR4JlqFniFWyFOOhlwbz+4CbR1bJE+6kkNqPW/oM7oNVOJ3KZE2UMplkYTKGU90GePH4GiqV1n06ATgXnwN4WPMDWOMcCfcTcpa8Ti7U3Hm5bAOMgyWzKjQLYNDZlHEP8SVHakj6E4rg5nJPrYKXx+1DaKqPe/C0Goh4MgEfI2THxdbeow/LSsdd5yuIDrsowxFa06mGuptbnIIzQVBcppBK+2Lxn/DE7LNovJOgwqk6W8KksPJtTaOIbwTrc87WfJtT1bY03XbBjO+FGx/zDJlpOYv6iMfBzZS6CyXetxgjpdlDlc2R777vZciJBBak2jlDQvbWNNeVFteCTAD7w8YjBwakXGY5LJc/bp/Lg6HPno2Gtl+gEe/nHTpIAlzm5b50OgO7ysqoAwFo7qUam+FCmYPzhUnYKajHuYox6nmZi1jJEO5W6cT1uRAFgQog33SMJA4C7KLGz9OrdSfD6lOo2YAvlHxMeKv892WQXtClSgXk9PSpjpn2uKd5Q8lwg9Abc/c1Z76oO/qZum9M7r043lZkOddxGuVK26SO1143mWhEySeFlKrIO8tuwmSvtrQE9XMCyRdwF++68fjDltpNnB2jevmpn4HCW7XgzxEVxe37BODi0E39U2bNwp7NlxWCX+kl9gJQcnb2MmUjL5qQ5y8oU5ij8/qYoF2Hpnebjb9iwTID0l0Rre/aHAjSMoVPNlOeaRe6Y+D+OdcZXO/Vg== \ No newline at end of file diff --git a/tests/load/snowflake/test_snowflake_configuration.py b/tests/load/snowflake/test_snowflake_configuration.py index 7af2af2067..7108ad06e5 100644 --- a/tests/load/snowflake/test_snowflake_configuration.py +++ b/tests/load/snowflake/test_snowflake_configuration.py @@ -36,6 +36,7 @@ def test_connection_string_with_all_params() -> None: def test_to_connector_params() -> None: + # PEM key pkey_str = Path('./tests/common/cases/secrets/encrypted-private-key').read_text('utf8') creds = SnowflakeCredentials() @@ -61,11 +62,12 @@ def test_to_connector_params() -> None: role='role1', ) + # base64 encoded DER key pkey_str = Path('./tests/common/cases/secrets/encrypted-private-key-base64').read_text('utf8') creds = SnowflakeCredentials() creds.private_key = pkey_str # type: ignore[assignment] - creds.private_key_passphrase = 'insecure' # type: ignore[assignment] + creds.private_key_passphrase = '12345' # type: ignore[assignment] creds.username = 'user1' creds.database = 'db1' creds.host = 'host1'