[DO-NOT-MERGE] Release kafka_consumer 3.1.4 beta

.in-toto/tag.eabca4ac.link

@@ -1 +1 @@
-{"signatures":[{"keyid":"eabca4ac14acbaf006cc03847c6739cb3a799f86","other_headers":"04000108001d162104eabca4ac14acbaf006cc03847c6739cb3a799f86050264fb7d87","signature":"618234153e92dcd4772003ff7c6e0b9e1cc38cebd5b07180a06bb25d91229486f4cfc55aceafc713aa1905bd6ffec725362d1dbd220189769220c5e3d682b473b1709f2edf017cdd52d8a41c6a554bf1b4dbd8197f8e071c247527a9d737cda37b49ae0860d84c24dfddf8b7518d40f136d89c51f3299dad0c8c48020dd7dec8f5b6a0cddf04196fd6fc18330ef21c3d03c50bb0e2284a3bafa02d79ea028813d3d32c5f8269dae0c08f06e554e3b6b0b9d742d2815a9c9aaa9e8ade61aa4298fe9f56d00b6fed76b91bb8979c145c677d34ce9715f30fc35c208a890cd2867a600704ff16f3d1f12f52821d845f07eb3debef224d377d44565819242b73969f3b2b9e712a0c6ef74cba6c94afd0eab71fb3e6534d607a4381b4cd608d33a16a2051ef93c701e8a0ec61fcfc1aae50a5b02c570afb5177afa0f5dda4f5b2e9a78521ffe53b4fe846a604138b9d543569c7a2dba55ad28b291d6599b93d4bc85423001a301e1e9333b363d08018596b99e1bfa3e3925891e7f1e7d5701fc79ceadbd89e34440bcf4036137ab5207cb8e4a2c1dad17f528120a4ea0a1958b7cf764867db6f4d5bd9bdac37bbb39daed690ab574420d985c02ae558fb1922ccda65e40fcf44ef6bbf2a877eec296bf0f1f33df10350b4cb3d1f067a32415e589105f7f5d31caa05e0a25183fbadc0e9282fc25e58b8c2e6fb30b7d4bd28b13b690d"}],"signed":{"_type":"link","byproducts":{},"command":[],"environment":{},"materials":{},"name":"tag","products":{"kafka_consumer/datadog_checks/__init__.py":{"sha256":"9a3c64b8b00c94da4b4f34618d803d3255808caf21b8afa9195c84b61da66b6a"},"kafka_consumer/datadog_checks/kafka_consumer/__about__.py":{"sha256":"3a639680ccbb2cccef79083a0aec04800ee80cecf88dbcc732f502b3da2f5032"},"kafka_consumer/datadog_checks/kafka_consumer/__init__.py":{"sha256":"5d5a2f991c64a95c96c8713f3179f9d0944682b6bed037b8c34d9f85463ca57c"},"kafka_consumer/datadog_checks/kafka_consumer/client.py":{"sha256":"d9ed648be374c67ee2e842698bfb72aa91ff3ef3c6f11bb7e344b8aa083bae20"},"kafka_consumer/datadog_checks/kafka_consumer/config.py":{"sha256":"55a1007ed7374fafb63b3b29eabdc854e13b9bfc41d13e9be347774af230aec0"},"kafka_consumer/datadog_checks/kafka_consumer/config_models/__init__.py":{"sha256":"c9cf5c66894430e7edbb00d00613b58ccfd38360f2fe490a23c17cf71ed294dc"},"kafka_consumer/datadog_checks/kafka_consumer/config_models/defaults.py":{"sha256":"5c46e0f757deee89e8ffb54fb9c06fb21ddb7fdbf253dcf80f42ae63e0c516fa"},"kafka_consumer/datadog_checks/kafka_consumer/config_models/instance.py":{"sha256":"10ad3f00ae389d32c47d79d2be6de4e85f6d33d616afda64b4220be45ced9be1"},"kafka_consumer/datadog_checks/kafka_consumer/config_models/shared.py":{"sha256":"aaac974b2272d540203aa9822a55bf3601c3d824e03e6ea9bb0ffa3bc316e76a"},"kafka_consumer/datadog_checks/kafka_consumer/config_models/validators.py":{"sha256":"0424fe17778b76e1b589b9564d0d543d1b71dba1edd6e5d71a7c528dddf68e0b"},"kafka_consumer/datadog_checks/kafka_consumer/constants.py":{"sha256":"df9bbecdb8bf8009ed20402963ebea6104bba418d7a1894034438bc0e3c91b44"},"kafka_consumer/datadog_checks/kafka_consumer/data/conf.yaml.example":{"sha256":"4258bd81a13bc7a6a1b5754946155efe3e9a328a2c081945cbac94dd430e7bf9"},"kafka_consumer/datadog_checks/kafka_consumer/kafka_consumer.py":{"sha256":"287fc5bb631cb665c86cae382bff3a6a00a7aec473cb09aa31d77e148091e266"},"kafka_consumer/pyproject.toml":{"sha256":"fd98e849fff6554cb9e19e25546adb89563f0c7ee417b03bcb1dc8e4ed0c9569"}}}}
+{"signatures":[{"keyid":"eabca4ac14acbaf006cc03847c6739cb3a799f86","other_headers":"04000108001d162104eabca4ac14acbaf006cc03847c6739cb3a799f86050264fb813d","signature":"5d288613fbff364c201b1b84352bc075da1e782e1b6c659f9d019c670821d66bebb3a574dd373ce365991913e08abaee77d434e5ee6fd4806a334b9c1e475e21f691c5300e59bd8a7f95a3bc9ccadd378046d0854b377592f0205a4d5a5b1e6bf31f0d897cf004f79c86696eddc770a9ca32807e01dd1220196cd71191ff261a0416040ac135597d166e593172fb1cb4bff7ef047b7dbeb572492f4fc60bee5db56fdc30394220bb7d38f2ecd49fe8495f551ce658258c59f4ead5dfd452089c096aff23359920c2fc72dbbd2119279477d7e888c92306366a9be0a9f0dfd3e0637c10a48d89875f9c7dbb05ddd0647a0aa8deab4269e2fdcb4ebc210939d70cef8c3ab8175a6f9126a61b45e1649d7e74eb09e64299e19bff64e3437a51b5988f05951f878bcc37cbf3462ca1f8398f91d106f6bb17f7074409e8808f0478b9dd08be34cc658597e1acea7ab13a331449f29103a95ac26fd58d8202b0daf530f6d78b8e6cc02e262b9a88ffe37e2fa89ab4e1036425ec78239c158d9bad810e6009dc34115f800c7691582bae3db0172272c1d7d7c800eb278a609509c901a791875e5780398e735e413cc81b064707ab0f9428168bb85cbd67f156c46b849974218119022b1f7751c38e0e152e86464fbb19eaac378a55f8246c0773b54b35863ff25837808e918507b116cf2c107b0ca7f8bd561424c0322f66d3730e3933"}],"signed":{"_type":"link","byproducts":{},"command":[],"environment":{},"materials":{},"name":"tag","products":{"kafka_consumer/datadog_checks/__init__.py":{"sha256":"9a3c64b8b00c94da4b4f34618d803d3255808caf21b8afa9195c84b61da66b6a"},"kafka_consumer/datadog_checks/kafka_consumer/__about__.py":{"sha256":"6fb17bea16a11704dab0c76fc4fe1aa33bed17926444a6bb08b60dee13355e0a"},"kafka_consumer/datadog_checks/kafka_consumer/__init__.py":{"sha256":"5d5a2f991c64a95c96c8713f3179f9d0944682b6bed037b8c34d9f85463ca57c"},"kafka_consumer/datadog_checks/kafka_consumer/client.py":{"sha256":"70e4fdaee38809d9b8c0caabe90edf8af9cd23fca91a11075c5d3255a128fe59"},"kafka_consumer/datadog_checks/kafka_consumer/config.py":{"sha256":"3c584ba56df9f17c1b7322e3ac2202bb126e41de6ca6b87cb16298e2c3dedb78"},"kafka_consumer/datadog_checks/kafka_consumer/config_models/__init__.py":{"sha256":"c9cf5c66894430e7edbb00d00613b58ccfd38360f2fe490a23c17cf71ed294dc"},"kafka_consumer/datadog_checks/kafka_consumer/config_models/defaults.py":{"sha256":"5c46e0f757deee89e8ffb54fb9c06fb21ddb7fdbf253dcf80f42ae63e0c516fa"},"kafka_consumer/datadog_checks/kafka_consumer/config_models/instance.py":{"sha256":"10ad3f00ae389d32c47d79d2be6de4e85f6d33d616afda64b4220be45ced9be1"},"kafka_consumer/datadog_checks/kafka_consumer/config_models/shared.py":{"sha256":"aaac974b2272d540203aa9822a55bf3601c3d824e03e6ea9bb0ffa3bc316e76a"},"kafka_consumer/datadog_checks/kafka_consumer/config_models/validators.py":{"sha256":"0424fe17778b76e1b589b9564d0d543d1b71dba1edd6e5d71a7c528dddf68e0b"},"kafka_consumer/datadog_checks/kafka_consumer/constants.py":{"sha256":"df9bbecdb8bf8009ed20402963ebea6104bba418d7a1894034438bc0e3c91b44"},"kafka_consumer/datadog_checks/kafka_consumer/data/conf.yaml.example":{"sha256":"4258bd81a13bc7a6a1b5754946155efe3e9a328a2c081945cbac94dd430e7bf9"},"kafka_consumer/datadog_checks/kafka_consumer/kafka_consumer.py":{"sha256":"b89d3f70e775023fdd181b8f532f1cdc2752f78bf76dea3b734ee52e80e5dbba"},"kafka_consumer/pyproject.toml":{"sha256":"fd98e849fff6554cb9e19e25546adb89563f0c7ee417b03bcb1dc8e4ed0c9569"}}}}

kafka_consumer/CHANGELOG.md

@@ -1,5 +1,17 @@
 # CHANGELOG - kafka_consumer
 
+## 3.1.4-rc.1 / 2023-09-08
+
+***Fixed***:
+
+* Set tls_verify to string rather than boolean. See [#15699](https://github.com/DataDog/integrations-core/pull/15699).
+
+## Unreleased
+
+***Fixed***:
+
+* Set tls_verify to string rather than boolean ([#15699](https://github.com/DataDog/integrations-core/pull/15699))
+
 ## 3.1.3 / 2023-08-14
 
 ***Fixed***:

kafka_consumer/datadog_checks/kafka_consumer/__about__.py

@@ -2,4 +2,4 @@
 # All rights reserved
 # Licensed under a 3-clause BSD style license (see LICENSE)
 
-__version__ = "3.1.3"
+__version__ = "3.1.4-rc.1"

kafka_consumer/datadog_checks/kafka_consumer/client.py

@@ -10,11 +10,10 @@
 
 
 class KafkaClient:
-    def __init__(self, config, tls_context, log) -> None:
+    def __init__(self, config, log) -> None:
         self.config = config
         self.log = log
         self._kafka_client = None
-        self._tls_context = tls_context
 
     @property
     def kafka_client(self):

kafka_consumer/datadog_checks/kafka_consumer/config.py

@@ -44,16 +44,22 @@ def __init__(self, init_config, instance, log) -> None:
         self._sasl_kerberos_keytab = instance.get('sasl_kerberos_keytab', os.environ.get("KRB5_CLIENT_KTNAME"))
         self._sasl_kerberos_principal = instance.get('sasl_kerberos_principal', 'kafkaclient')
         self._sasl_oauth_token_provider = instance.get('sasl_oauth_token_provider')
-        self._tls_ca_cert = instance.get("tls_ca_cert")
-        self._tls_cert = instance.get("tls_cert")
-        self._tls_private_key = instance.get("tls_private_key")
-        self._tls_private_key_password = instance.get("tls_private_key_password")
-        self._tls_validate_hostname = is_affirmative(instance.get("tls_validate_hostname", True))
+        self._tls_ca_cert = instance.get("tls_ca_cert") or instance.get("ssl_cafile")
+        self._tls_cert = instance.get("tls_cert") or instance.get("ssl_certfile")
+        self._tls_private_key = instance.get("tls_private_key") or instance.get("ssl_keyfile")
+        self._tls_private_key_password = instance.get("tls_private_key_password") or instance.get("ssl_password")
+        # Note: Remapped field is ignored if standard field is already used
+        self._tls_validate_hostname = (
+            is_affirmative(instance.get("tls_validate_hostname", True))
+            if "tls_validate_hostname" in instance
+            else is_affirmative(instance.get("ssl_check_hostname", True))
+        )
 
+        # tls_verify/enable.ssl.certificate.verification is required to be a string when passed into
         if self._tls_cert or self._tls_ca_cert or self._tls_private_key or self._tls_private_key_password:
-            self._tls_verify = True
+            self._tls_verify = "true"
         else:
-            self._tls_verify = is_affirmative(instance.get("tls_verify", True))
+            self._tls_verify = "true" if is_affirmative(instance.get("tls_verify", True)) else "false"
 
     def validate_config(self):
         if not self._kafka_connect_str:

kafka_consumer/datadog_checks/kafka_consumer/kafka_consumer.py

@@ -12,20 +12,11 @@ class KafkaCheck(AgentCheck):
 
     __NAMESPACE__ = 'kafka'
 
-    # This remapper is used to support legacy config values
-    TLS_CONFIG_REMAPPER = {
-        'ssl_check_hostname': {'name': 'tls_validate_hostname'},
-        'ssl_cafile': {'name': 'tls_ca_cert'},
-        'ssl_certfile': {'name': 'tls_cert'},
-        'ssl_keyfile': {'name': 'tls_private_key'},
-        'ssl_password': {'name': 'tls_private_key_password'},
-    }
-
     def __init__(self, name, init_config, instances):
         super(KafkaCheck, self).__init__(name, init_config, instances)
         self.config = KafkaConfig(self.init_config, self.instance, self.log)
         self._context_limit = self.config._context_limit
-        self.client = KafkaClient(self.config, self.get_tls_context(), self.log)
+        self.client = KafkaClient(self.config, self.log)
         self.check_initializations.append(self.config.validate_config)
 
     def check(self, _):

kafka_consumer/tests/test_unit.py

@@ -13,21 +13,76 @@
 
 
 @pytest.mark.parametrize(
-    'extra_config, expected_http_kwargs',
+    'legacy_config, kafka_client_config, value',
     [
+        pytest.param("ssl_check_hostname", "_tls_validate_hostname", False, id='legacy validate_hostname param false'),
+        pytest.param("ssl_check_hostname", "_tls_validate_hostname", True, id='legacy validate_hostname param true'),
+        pytest.param("ssl_cafile", "_tls_ca_cert", "ca_file", id='legacy tls_ca_cert param'),
+        pytest.param("ssl_certfile", "_tls_cert", "cert", id='legacy tls_cert param'),
+        pytest.param("ssl_keyfile", "_tls_private_key", "private_key", id='legacy tls_private_key param'),
         pytest.param(
-            {'ssl_check_hostname': False}, {'tls_validate_hostname': False}, id='legacy validate_hostname param'
+            "ssl_password",
+            "_tls_private_key_password",
+            "private_key_password",
+            id='legacy tls_private_key_password param',
         ),
     ],
 )
-def test_tls_config_legacy(extra_config, expected_http_kwargs, check, kafka_instance):
-    kafka_instance.update(extra_config)
+def test_tls_config_legacy(legacy_config, kafka_client_config, value, check):
+    kafka_consumer_check = check({legacy_config: value})
+    assert getattr(kafka_consumer_check.config, kafka_client_config) == value
+
+
+@pytest.mark.parametrize(
+    'ssl_check_hostname_value, tls_validate_hostname_value, expected_value',
+    [
+        pytest.param(True, True, True, id='Both true'),
+        pytest.param(False, False, False, id='Both false'),
+        pytest.param(False, True, True, id='only tls_validate_hostname_value true'),
+        pytest.param(True, False, False, id='only tls_validate_hostname_value false'),
+        pytest.param(False, "true", True, id='tls_validate_hostname true as string'),
+        pytest.param(False, "false", False, id='tls_validate_hostname false as string'),
+    ],
+)
+def test_tls_validate_hostname_conflict(
+    ssl_check_hostname_value, tls_validate_hostname_value, expected_value, check, kafka_instance
+):
+    kafka_instance.update(
+        {"ssl_check_hostname": ssl_check_hostname_value, "tls_validate_hostname": tls_validate_hostname_value}
+    )
     kafka_consumer_check = check(kafka_instance)
-    kafka_consumer_check.get_tls_context()
-    actual_options = {
-        k: v for k, v in kafka_consumer_check._tls_context_wrapper.config.items() if k in expected_http_kwargs
-    }
-    assert expected_http_kwargs == actual_options
+    assert kafka_consumer_check.config._tls_validate_hostname == expected_value
+
+
+@pytest.mark.parametrize(
+    'tls_verify, expected',
+    [
+        pytest.param({}, "true", id='given empty tls_verify, expect default string true'),
+        pytest.param({'tls_verify': True}, "true", id='given True tls_verify, expect string true'),
+        pytest.param(
+            {
+                'tls_verify': False,
+                "tls_cert": None,
+                "tls_ca_cert": None,
+                "tls_private_key": None,
+                "tls_private_key_password": None,
+            },
+            "false",
+            id='given False tls_verify and other TLS options none, expect string false',
+        ),
+        pytest.param(
+            {'tls_verify': False, "tls_private_key_password": "password"},
+            "true",
+            id='given False tls_verify but TLS password, expect string true',
+        ),
+    ],
+)
+def test_tls_verify_is_string(tls_verify, expected, check, kafka_instance):
+    kafka_instance.update(tls_verify)
+    kafka_consumer_check = check(kafka_instance)
+    config = kafka_consumer_check.config
+
+    assert config._tls_verify == expected
 
 
 @pytest.mark.parametrize(

requirements-agent-release.txt

@@ -79,7 +79,7 @@ datadog-impala==1.1.2
 datadog-istio==4.4.1
 datadog-jboss-wildfly==2.0.2
 datadog-journald==1.1.1
-datadog-kafka-consumer==3.1.3
+datadog-kafka-consumer==3.1.4-rc.1
 datadog-kafka==2.13.2
 datadog-kong==2.4.1
 datadog-kube-apiserver-metrics==3.6.2