From a7da42ba2e8a161ad329f9f9e785b8b31bcb971f Mon Sep 17 00:00:00 2001 From: Andrew Zhang Date: Fri, 1 Sep 2023 11:36:22 -0400 Subject: [PATCH 1/4] Remove unused TLS context (#15700) * Remove unused TLS context * Update test_tls_config_legacy * Fix test for legacy tls_validate_hostname config * Fix style --- .../datadog_checks/kafka_consumer/client.py | 3 +- .../datadog_checks/kafka_consumer/config.py | 15 ++++--- .../kafka_consumer/kafka_consumer.py | 11 +---- kafka_consumer/tests/test_unit.py | 42 +++++++++++++++---- 4 files changed, 45 insertions(+), 26 deletions(-) diff --git a/kafka_consumer/datadog_checks/kafka_consumer/client.py b/kafka_consumer/datadog_checks/kafka_consumer/client.py index 404d5e7440627..ef175a638699e 100644 --- a/kafka_consumer/datadog_checks/kafka_consumer/client.py +++ b/kafka_consumer/datadog_checks/kafka_consumer/client.py @@ -10,11 +10,10 @@ class KafkaClient: - def __init__(self, config, tls_context, log) -> None: + def __init__(self, config, log) -> None: self.config = config self.log = log self._kafka_client = None - self._tls_context = tls_context @property def kafka_client(self): diff --git a/kafka_consumer/datadog_checks/kafka_consumer/config.py b/kafka_consumer/datadog_checks/kafka_consumer/config.py index 57271d532b8c3..b99781bf2eb44 100644 --- a/kafka_consumer/datadog_checks/kafka_consumer/config.py +++ b/kafka_consumer/datadog_checks/kafka_consumer/config.py @@ -44,11 +44,16 @@ def __init__(self, init_config, instance, log) -> None: self._sasl_kerberos_keytab = instance.get('sasl_kerberos_keytab', os.environ.get("KRB5_CLIENT_KTNAME")) self._sasl_kerberos_principal = instance.get('sasl_kerberos_principal', 'kafkaclient') self._sasl_oauth_token_provider = instance.get('sasl_oauth_token_provider') - self._tls_ca_cert = instance.get("tls_ca_cert") - self._tls_cert = instance.get("tls_cert") - self._tls_private_key = instance.get("tls_private_key") - self._tls_private_key_password = instance.get("tls_private_key_password") - self._tls_validate_hostname = is_affirmative(instance.get("tls_validate_hostname", True)) + self._tls_ca_cert = instance.get("tls_ca_cert") or instance.get("ssl_cafile") + self._tls_cert = instance.get("tls_cert") or instance.get("ssl_certfile") + self._tls_private_key = instance.get("tls_private_key") or instance.get("ssl_keyfile") + self._tls_private_key_password = instance.get("tls_private_key_password") or instance.get("ssl_password") + # Note: Remapped field is ignored if standard field is already used + self._tls_validate_hostname = ( + is_affirmative(instance.get("tls_validate_hostname", True)) + if "tls_validate_hostname" in instance + else is_affirmative(instance.get("ssl_check_hostname", True)) + ) if self._tls_cert or self._tls_ca_cert or self._tls_private_key or self._tls_private_key_password: self._tls_verify = True diff --git a/kafka_consumer/datadog_checks/kafka_consumer/kafka_consumer.py b/kafka_consumer/datadog_checks/kafka_consumer/kafka_consumer.py index 08ebf03d6a8a4..81bad046b5225 100644 --- a/kafka_consumer/datadog_checks/kafka_consumer/kafka_consumer.py +++ b/kafka_consumer/datadog_checks/kafka_consumer/kafka_consumer.py @@ -12,20 +12,11 @@ class KafkaCheck(AgentCheck): __NAMESPACE__ = 'kafka' - # This remapper is used to support legacy config values - TLS_CONFIG_REMAPPER = { - 'ssl_check_hostname': {'name': 'tls_validate_hostname'}, - 'ssl_cafile': {'name': 'tls_ca_cert'}, - 'ssl_certfile': {'name': 'tls_cert'}, - 'ssl_keyfile': {'name': 'tls_private_key'}, - 'ssl_password': {'name': 'tls_private_key_password'}, - } - def __init__(self, name, init_config, instances): super(KafkaCheck, self).__init__(name, init_config, instances) self.config = KafkaConfig(self.init_config, self.instance, self.log) self._context_limit = self.config._context_limit - self.client = KafkaClient(self.config, self.get_tls_context(), self.log) + self.client = KafkaClient(self.config, self.log) self.check_initializations.append(self.config.validate_config) def check(self, _): diff --git a/kafka_consumer/tests/test_unit.py b/kafka_consumer/tests/test_unit.py index 25d7381660d1c..c3537b2f7f5a1 100644 --- a/kafka_consumer/tests/test_unit.py +++ b/kafka_consumer/tests/test_unit.py @@ -13,21 +13,45 @@ @pytest.mark.parametrize( - 'extra_config, expected_http_kwargs', + 'legacy_config, kafka_client_config, value', [ + pytest.param("ssl_check_hostname", "_tls_validate_hostname", False, id='legacy validate_hostname param false'), + pytest.param("ssl_check_hostname", "_tls_validate_hostname", True, id='legacy validate_hostname param true'), + pytest.param("ssl_cafile", "_tls_ca_cert", "ca_file", id='legacy tls_ca_cert param'), + pytest.param("ssl_certfile", "_tls_cert", "cert", id='legacy tls_cert param'), + pytest.param("ssl_keyfile", "_tls_private_key", "private_key", id='legacy tls_private_key param'), pytest.param( - {'ssl_check_hostname': False}, {'tls_validate_hostname': False}, id='legacy validate_hostname param' + "ssl_password", + "_tls_private_key_password", + "private_key_password", + id='legacy tls_private_key_password param', ), ], ) -def test_tls_config_legacy(extra_config, expected_http_kwargs, check, kafka_instance): - kafka_instance.update(extra_config) +def test_tls_config_legacy(legacy_config, kafka_client_config, value, check): + kafka_consumer_check = check({legacy_config: value}) + assert getattr(kafka_consumer_check.config, kafka_client_config) == value + + +@pytest.mark.parametrize( + 'ssl_check_hostname_value, tls_validate_hostname_value, expected_value', + [ + pytest.param(True, True, True, id='Both true'), + pytest.param(False, False, False, id='Both false'), + pytest.param(False, True, True, id='only tls_validate_hostname_value true'), + pytest.param(True, False, False, id='only tls_validate_hostname_value false'), + pytest.param(False, "true", True, id='tls_validate_hostname true as string'), + pytest.param(False, "false", False, id='tls_validate_hostname false as string'), + ], +) +def test_tls_validate_hostname_conflict( + ssl_check_hostname_value, tls_validate_hostname_value, expected_value, check, kafka_instance +): + kafka_instance.update( + {"ssl_check_hostname": ssl_check_hostname_value, "tls_validate_hostname": tls_validate_hostname_value} + ) kafka_consumer_check = check(kafka_instance) - kafka_consumer_check.get_tls_context() - actual_options = { - k: v for k, v in kafka_consumer_check._tls_context_wrapper.config.items() if k in expected_http_kwargs - } - assert expected_http_kwargs == actual_options + assert kafka_consumer_check.config._tls_validate_hostname == expected_value @pytest.mark.parametrize( From 124034d98bee383e5a4fc17f4af8d373eb9bd876 Mon Sep 17 00:00:00 2001 From: Andrew Zhang Date: Fri, 1 Sep 2023 13:05:21 -0400 Subject: [PATCH 2/4] Set tls_verify to string rather than boolean (#15699) * Set tls_verify to string rather than boolean * Remove unused TLS context * Add Changelog entry * Update test_tls_config_legacy * Fix test for legacy tls_validate_hostname config * Fix style * Fix style * Move script.py to separate PR --- kafka_consumer/CHANGELOG.md | 6 ++++ .../datadog_checks/kafka_consumer/config.py | 5 +-- kafka_consumer/tests/test_unit.py | 31 +++++++++++++++++++ 3 files changed, 40 insertions(+), 2 deletions(-) diff --git a/kafka_consumer/CHANGELOG.md b/kafka_consumer/CHANGELOG.md index 0f665e71dcf27..fc20c68ba4e4c 100644 --- a/kafka_consumer/CHANGELOG.md +++ b/kafka_consumer/CHANGELOG.md @@ -1,5 +1,11 @@ # CHANGELOG - kafka_consumer +## Unreleased + +***Fixed***: + +* Set tls_verify to string rather than boolean ([#15699](https://github.com/DataDog/integrations-core/pull/15699)) + ## 3.1.3 / 2023-08-14 ***Fixed***: diff --git a/kafka_consumer/datadog_checks/kafka_consumer/config.py b/kafka_consumer/datadog_checks/kafka_consumer/config.py index b99781bf2eb44..b337ca4fe017d 100644 --- a/kafka_consumer/datadog_checks/kafka_consumer/config.py +++ b/kafka_consumer/datadog_checks/kafka_consumer/config.py @@ -55,10 +55,11 @@ def __init__(self, init_config, instance, log) -> None: else is_affirmative(instance.get("ssl_check_hostname", True)) ) + # tls_verify/enable.ssl.certificate.verification is required to be a string when passed into if self._tls_cert or self._tls_ca_cert or self._tls_private_key or self._tls_private_key_password: - self._tls_verify = True + self._tls_verify = "true" else: - self._tls_verify = is_affirmative(instance.get("tls_verify", True)) + self._tls_verify = "true" if is_affirmative(instance.get("tls_verify", True)) else "false" def validate_config(self): if not self._kafka_connect_str: diff --git a/kafka_consumer/tests/test_unit.py b/kafka_consumer/tests/test_unit.py index c3537b2f7f5a1..bc70aca471b1d 100644 --- a/kafka_consumer/tests/test_unit.py +++ b/kafka_consumer/tests/test_unit.py @@ -54,6 +54,37 @@ def test_tls_validate_hostname_conflict( assert kafka_consumer_check.config._tls_validate_hostname == expected_value +@pytest.mark.parametrize( + 'tls_verify, expected', + [ + pytest.param({}, "true", id='given empty tls_verify, expect default string true'), + pytest.param({'tls_verify': True}, "true", id='given True tls_verify, expect string true'), + pytest.param( + { + 'tls_verify': False, + "tls_cert": None, + "tls_ca_cert": None, + "tls_private_key": None, + "tls_private_key_password": None, + }, + "false", + id='given False tls_verify and other TLS options none, expect string false', + ), + pytest.param( + {'tls_verify': False, "tls_private_key_password": "password"}, + "true", + id='given False tls_verify but TLS password, expect string true', + ), + ], +) +def test_tls_verify_is_string(tls_verify, expected, check, kafka_instance): + kafka_instance.update(tls_verify) + kafka_consumer_check = check(kafka_instance) + config = kafka_consumer_check.config + + assert config._tls_verify == expected + + @pytest.mark.parametrize( 'sasl_oauth_token_provider, expected_exception, mocked_admin_client', [ From 735f7cc99762853310a9e3f3db29f2359a6d941b Mon Sep 17 00:00:00 2001 From: Andrew Zhang Date: Fri, 8 Sep 2023 16:16:59 -0400 Subject: [PATCH 3/4] [Release] Bumped kafka_consumer version to 3.1.4-rc.1 --- kafka_consumer/CHANGELOG.md | 6 ++++++ kafka_consumer/datadog_checks/kafka_consumer/__about__.py | 2 +- requirements-agent-release.txt | 2 +- 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/kafka_consumer/CHANGELOG.md b/kafka_consumer/CHANGELOG.md index fc20c68ba4e4c..c943a58a75833 100644 --- a/kafka_consumer/CHANGELOG.md +++ b/kafka_consumer/CHANGELOG.md @@ -1,5 +1,11 @@ # CHANGELOG - kafka_consumer +## 3.1.4-rc.1 / 2023-09-08 + +***Fixed***: + +* Set tls_verify to string rather than boolean. See [#15699](https://github.com/DataDog/integrations-core/pull/15699). + ## Unreleased ***Fixed***: diff --git a/kafka_consumer/datadog_checks/kafka_consumer/__about__.py b/kafka_consumer/datadog_checks/kafka_consumer/__about__.py index a7ba430e5e216..c7e2f8f979508 100644 --- a/kafka_consumer/datadog_checks/kafka_consumer/__about__.py +++ b/kafka_consumer/datadog_checks/kafka_consumer/__about__.py @@ -2,4 +2,4 @@ # All rights reserved # Licensed under a 3-clause BSD style license (see LICENSE) -__version__ = "3.1.3" +__version__ = "3.1.4-rc.1" diff --git a/requirements-agent-release.txt b/requirements-agent-release.txt index 7cfcba458974a..e17c4592be8a0 100644 --- a/requirements-agent-release.txt +++ b/requirements-agent-release.txt @@ -79,7 +79,7 @@ datadog-impala==1.1.2 datadog-istio==4.4.1 datadog-jboss-wildfly==2.0.2 datadog-journald==1.1.1 -datadog-kafka-consumer==3.1.3 +datadog-kafka-consumer==3.1.4-rc.1 datadog-kafka==2.13.2 datadog-kong==2.4.1 datadog-kube-apiserver-metrics==3.6.2 From b4a4e075734ff1beb900c07d332fc9d9fab44108 Mon Sep 17 00:00:00 2001 From: Andrew Zhang Date: Fri, 8 Sep 2023 16:17:02 -0400 Subject: [PATCH 4/4] [Release] Update metadata --- .in-toto/tag.eabca4ac.link | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.in-toto/tag.eabca4ac.link b/.in-toto/tag.eabca4ac.link index 4f6d499c65483..5953b07e282e7 100644 --- a/.in-toto/tag.eabca4ac.link +++ b/.in-toto/tag.eabca4ac.link @@ -1 +1 @@ -{"signatures":[{"keyid":"eabca4ac14acbaf006cc03847c6739cb3a799f86","other_headers":"04000108001d162104eabca4ac14acbaf006cc03847c6739cb3a799f86050264fb7d87","signature":"618234153e92dcd4772003ff7c6e0b9e1cc38cebd5b07180a06bb25d91229486f4cfc55aceafc713aa1905bd6ffec725362d1dbd220189769220c5e3d682b473b1709f2edf017cdd52d8a41c6a554bf1b4dbd8197f8e071c247527a9d737cda37b49ae0860d84c24dfddf8b7518d40f136d89c51f3299dad0c8c48020dd7dec8f5b6a0cddf04196fd6fc18330ef21c3d03c50bb0e2284a3bafa02d79ea028813d3d32c5f8269dae0c08f06e554e3b6b0b9d742d2815a9c9aaa9e8ade61aa4298fe9f56d00b6fed76b91bb8979c145c677d34ce9715f30fc35c208a890cd2867a600704ff16f3d1f12f52821d845f07eb3debef224d377d44565819242b73969f3b2b9e712a0c6ef74cba6c94afd0eab71fb3e6534d607a4381b4cd608d33a16a2051ef93c701e8a0ec61fcfc1aae50a5b02c570afb5177afa0f5dda4f5b2e9a78521ffe53b4fe846a604138b9d543569c7a2dba55ad28b291d6599b93d4bc85423001a301e1e9333b363d08018596b99e1bfa3e3925891e7f1e7d5701fc79ceadbd89e34440bcf4036137ab5207cb8e4a2c1dad17f528120a4ea0a1958b7cf764867db6f4d5bd9bdac37bbb39daed690ab574420d985c02ae558fb1922ccda65e40fcf44ef6bbf2a877eec296bf0f1f33df10350b4cb3d1f067a32415e589105f7f5d31caa05e0a25183fbadc0e9282fc25e58b8c2e6fb30b7d4bd28b13b690d"}],"signed":{"_type":"link","byproducts":{},"command":[],"environment":{},"materials":{},"name":"tag","products":{"kafka_consumer/datadog_checks/__init__.py":{"sha256":"9a3c64b8b00c94da4b4f34618d803d3255808caf21b8afa9195c84b61da66b6a"},"kafka_consumer/datadog_checks/kafka_consumer/__about__.py":{"sha256":"3a639680ccbb2cccef79083a0aec04800ee80cecf88dbcc732f502b3da2f5032"},"kafka_consumer/datadog_checks/kafka_consumer/__init__.py":{"sha256":"5d5a2f991c64a95c96c8713f3179f9d0944682b6bed037b8c34d9f85463ca57c"},"kafka_consumer/datadog_checks/kafka_consumer/client.py":{"sha256":"d9ed648be374c67ee2e842698bfb72aa91ff3ef3c6f11bb7e344b8aa083bae20"},"kafka_consumer/datadog_checks/kafka_consumer/config.py":{"sha256":"55a1007ed7374fafb63b3b29eabdc854e13b9bfc41d13e9be347774af230aec0"},"kafka_consumer/datadog_checks/kafka_consumer/config_models/__init__.py":{"sha256":"c9cf5c66894430e7edbb00d00613b58ccfd38360f2fe490a23c17cf71ed294dc"},"kafka_consumer/datadog_checks/kafka_consumer/config_models/defaults.py":{"sha256":"5c46e0f757deee89e8ffb54fb9c06fb21ddb7fdbf253dcf80f42ae63e0c516fa"},"kafka_consumer/datadog_checks/kafka_consumer/config_models/instance.py":{"sha256":"10ad3f00ae389d32c47d79d2be6de4e85f6d33d616afda64b4220be45ced9be1"},"kafka_consumer/datadog_checks/kafka_consumer/config_models/shared.py":{"sha256":"aaac974b2272d540203aa9822a55bf3601c3d824e03e6ea9bb0ffa3bc316e76a"},"kafka_consumer/datadog_checks/kafka_consumer/config_models/validators.py":{"sha256":"0424fe17778b76e1b589b9564d0d543d1b71dba1edd6e5d71a7c528dddf68e0b"},"kafka_consumer/datadog_checks/kafka_consumer/constants.py":{"sha256":"df9bbecdb8bf8009ed20402963ebea6104bba418d7a1894034438bc0e3c91b44"},"kafka_consumer/datadog_checks/kafka_consumer/data/conf.yaml.example":{"sha256":"4258bd81a13bc7a6a1b5754946155efe3e9a328a2c081945cbac94dd430e7bf9"},"kafka_consumer/datadog_checks/kafka_consumer/kafka_consumer.py":{"sha256":"287fc5bb631cb665c86cae382bff3a6a00a7aec473cb09aa31d77e148091e266"},"kafka_consumer/pyproject.toml":{"sha256":"fd98e849fff6554cb9e19e25546adb89563f0c7ee417b03bcb1dc8e4ed0c9569"}}}} \ No newline at end of file +{"signatures":[{"keyid":"eabca4ac14acbaf006cc03847c6739cb3a799f86","other_headers":"04000108001d162104eabca4ac14acbaf006cc03847c6739cb3a799f86050264fb813d","signature":"5d288613fbff364c201b1b84352bc075da1e782e1b6c659f9d019c670821d66bebb3a574dd373ce365991913e08abaee77d434e5ee6fd4806a334b9c1e475e21f691c5300e59bd8a7f95a3bc9ccadd378046d0854b377592f0205a4d5a5b1e6bf31f0d897cf004f79c86696eddc770a9ca32807e01dd1220196cd71191ff261a0416040ac135597d166e593172fb1cb4bff7ef047b7dbeb572492f4fc60bee5db56fdc30394220bb7d38f2ecd49fe8495f551ce658258c59f4ead5dfd452089c096aff23359920c2fc72dbbd2119279477d7e888c92306366a9be0a9f0dfd3e0637c10a48d89875f9c7dbb05ddd0647a0aa8deab4269e2fdcb4ebc210939d70cef8c3ab8175a6f9126a61b45e1649d7e74eb09e64299e19bff64e3437a51b5988f05951f878bcc37cbf3462ca1f8398f91d106f6bb17f7074409e8808f0478b9dd08be34cc658597e1acea7ab13a331449f29103a95ac26fd58d8202b0daf530f6d78b8e6cc02e262b9a88ffe37e2fa89ab4e1036425ec78239c158d9bad810e6009dc34115f800c7691582bae3db0172272c1d7d7c800eb278a609509c901a791875e5780398e735e413cc81b064707ab0f9428168bb85cbd67f156c46b849974218119022b1f7751c38e0e152e86464fbb19eaac378a55f8246c0773b54b35863ff25837808e918507b116cf2c107b0ca7f8bd561424c0322f66d3730e3933"}],"signed":{"_type":"link","byproducts":{},"command":[],"environment":{},"materials":{},"name":"tag","products":{"kafka_consumer/datadog_checks/__init__.py":{"sha256":"9a3c64b8b00c94da4b4f34618d803d3255808caf21b8afa9195c84b61da66b6a"},"kafka_consumer/datadog_checks/kafka_consumer/__about__.py":{"sha256":"6fb17bea16a11704dab0c76fc4fe1aa33bed17926444a6bb08b60dee13355e0a"},"kafka_consumer/datadog_checks/kafka_consumer/__init__.py":{"sha256":"5d5a2f991c64a95c96c8713f3179f9d0944682b6bed037b8c34d9f85463ca57c"},"kafka_consumer/datadog_checks/kafka_consumer/client.py":{"sha256":"70e4fdaee38809d9b8c0caabe90edf8af9cd23fca91a11075c5d3255a128fe59"},"kafka_consumer/datadog_checks/kafka_consumer/config.py":{"sha256":"3c584ba56df9f17c1b7322e3ac2202bb126e41de6ca6b87cb16298e2c3dedb78"},"kafka_consumer/datadog_checks/kafka_consumer/config_models/__init__.py":{"sha256":"c9cf5c66894430e7edbb00d00613b58ccfd38360f2fe490a23c17cf71ed294dc"},"kafka_consumer/datadog_checks/kafka_consumer/config_models/defaults.py":{"sha256":"5c46e0f757deee89e8ffb54fb9c06fb21ddb7fdbf253dcf80f42ae63e0c516fa"},"kafka_consumer/datadog_checks/kafka_consumer/config_models/instance.py":{"sha256":"10ad3f00ae389d32c47d79d2be6de4e85f6d33d616afda64b4220be45ced9be1"},"kafka_consumer/datadog_checks/kafka_consumer/config_models/shared.py":{"sha256":"aaac974b2272d540203aa9822a55bf3601c3d824e03e6ea9bb0ffa3bc316e76a"},"kafka_consumer/datadog_checks/kafka_consumer/config_models/validators.py":{"sha256":"0424fe17778b76e1b589b9564d0d543d1b71dba1edd6e5d71a7c528dddf68e0b"},"kafka_consumer/datadog_checks/kafka_consumer/constants.py":{"sha256":"df9bbecdb8bf8009ed20402963ebea6104bba418d7a1894034438bc0e3c91b44"},"kafka_consumer/datadog_checks/kafka_consumer/data/conf.yaml.example":{"sha256":"4258bd81a13bc7a6a1b5754946155efe3e9a328a2c081945cbac94dd430e7bf9"},"kafka_consumer/datadog_checks/kafka_consumer/kafka_consumer.py":{"sha256":"b89d3f70e775023fdd181b8f532f1cdc2752f78bf76dea3b734ee52e80e5dbba"},"kafka_consumer/pyproject.toml":{"sha256":"fd98e849fff6554cb9e19e25546adb89563f0c7ee417b03bcb1dc8e4ed0c9569"}}}} \ No newline at end of file