diff --git a/appgate_sdp/CHANGELOG.md b/appgate_sdp/CHANGELOG.md index 7dd14c64baf40..96c18884e43fe 100644 --- a/appgate_sdp/CHANGELOG.md +++ b/appgate_sdp/CHANGELOG.md @@ -2,3 +2,8 @@ +## 1.0.0 / 2024-10-04 + +***Added***: + +* Initial Release ([#18465](https://github.com/DataDog/integrations-core/pull/18465)) diff --git a/appgate_sdp/changelog.d/18465.added b/appgate_sdp/changelog.d/18465.added deleted file mode 100644 index aa949b47b7b41..0000000000000 --- a/appgate_sdp/changelog.d/18465.added +++ /dev/null @@ -1 +0,0 @@ -Initial Release \ No newline at end of file diff --git a/appgate_sdp/datadog_checks/appgate_sdp/__about__.py b/appgate_sdp/datadog_checks/appgate_sdp/__about__.py index e9541ce83e9e5..acbfd1c866b84 100644 --- a/appgate_sdp/datadog_checks/appgate_sdp/__about__.py +++ b/appgate_sdp/datadog_checks/appgate_sdp/__about__.py @@ -1,4 +1,4 @@ # (C) Datadog, Inc. 2024-present # All rights reserved # Licensed under a 3-clause BSD style license (see LICENSE) -__version__ = '0.0.1' +__version__ = '1.0.0' diff --git a/cisco_sdwan/README.md b/cisco_sdwan/README.md index 6fcaca0c15d8f..fa6194285587b 100644 --- a/cisco_sdwan/README.md +++ b/cisco_sdwan/README.md @@ -1,4 +1,4 @@ -
The Cisco SD-WAN NDM integration is in public beta.
+
The Cisco SD-WAN NDM integration is in Preview.
# Agent Check: Cisco SD-WAN diff --git a/cisco_secure_endpoint/README.md b/cisco_secure_endpoint/README.md index 0d4bc372b5718..6f88038d18f0f 100644 --- a/cisco_secure_endpoint/README.md +++ b/cisco_secure_endpoint/README.md @@ -38,10 +38,10 @@ Configure the Datadog endpoint to forward Cisco Secure Endpoint logs to Datadog. | Cisco Secure Endpoint Parameters | Description | | -------------------- | ------------ | -| API Host URL | The API Host URL for Cisco Secure Endpoint Cloud is "https://api.\.apm.cisco.com". Adjust the "region" part based on the region of the Cisco Secure Endpoint server. If Cisco Secure Endpoint is hosted on VPC (Virtual Private Cloud), directly provide the API Host URL. | +| API Domain Name | The API Domain Name for Cisco Secure Endpoint Cloud is "api.\.amp.cisco.com". Adjust the "region" part based on the region of the Cisco Secure Endpoint server. If Cisco Secure Endpoint is hosted on VPC (Virtual Private Cloud), directly provide the API Domain Name. | | Client ID | Client ID from Cisco Secure Endpoint. | | API Key | API Key from Cisco Secure Endpoint. | -| Get Endpoint Details | Keep it "true" to collect endpoint metadata for Cisco Secure Endpoint event logs, otherwise "false". | +| Get Endpoint Details | Keep it "true" to collect endpoint metadata for Cisco Secure Endpoint event logs, otherwise "false". Default value is "true". | ## Data Collected diff --git a/datadog_checks_base/changelog.d/18758.added b/datadog_checks_base/changelog.d/18758.added new file mode 100644 index 0000000000000..426f18362cb36 --- /dev/null +++ b/datadog_checks_base/changelog.d/18758.added @@ -0,0 +1 @@ +Added Postgres cross-org telemetry metrics. diff --git a/datadog_checks_base/datadog_checks/base/stubs/datadog_agent.py b/datadog_checks_base/datadog_checks/base/stubs/datadog_agent.py index 5d355a97c7f72..67f59540d0a80 100644 --- a/datadog_checks_base/datadog_checks/base/stubs/datadog_agent.py +++ b/datadog_checks_base/datadog_checks/base/stubs/datadog_agent.py @@ -29,6 +29,7 @@ def __init__(self): self._process_start_time = 0 self._external_tags = [] self._host_tags = "{}" + self._sent_telemetry = defaultdict(list) def get_default_config(self): return {'enable_metadata_collection': True, 'disable_unsafe_yaml': True} @@ -84,6 +85,12 @@ def assert_external_tags_count(self, count): count, tags_count, repr(self._external_tags) ) + def assert_telemetry(self, check_name, metric_name, metric_type, metric_value): + values = self._sent_telemetry[(check_name, metric_name, metric_type)] + assert metric_value in values, 'Expected value {} for check {}, metric {}, type {}. Found {}.'.format( + metric_value, check_name, metric_name, metric_type, values + ) + def get_hostname(self): return self._hostname @@ -152,6 +159,9 @@ def obfuscate_mongodb_string(self, command): # Passthrough stub: obfuscation implementation is in Go code. return command + def emit_agent_telemetry(self, check_name, metric_name, metric_value, metric_type): + self._sent_telemetry[(check_name, metric_name, metric_type)].append(metric_value) + # Use the stub as a singleton datadog_agent = DatadogAgentStub() diff --git a/kubeflow/CHANGELOG.md b/kubeflow/CHANGELOG.md index 8658345ebf7a3..5c3bf784bb7d8 100644 --- a/kubeflow/CHANGELOG.md +++ b/kubeflow/CHANGELOG.md @@ -2,3 +2,8 @@ +## 1.0.0 / 2024-10-04 + +***Fixed***: + +* Bump the version of datadog-checks-base to 37.0.0 ([#18617](https://github.com/DataDog/integrations-core/pull/18617)) diff --git a/kubeflow/changelog.d/18617.fixed b/kubeflow/changelog.d/18617.fixed deleted file mode 100644 index e759164f82784..0000000000000 --- a/kubeflow/changelog.d/18617.fixed +++ /dev/null @@ -1 +0,0 @@ -Bump the version of datadog-checks-base to 37.0.0 \ No newline at end of file diff --git a/kubeflow/datadog_checks/kubeflow/__about__.py b/kubeflow/datadog_checks/kubeflow/__about__.py index e9541ce83e9e5..acbfd1c866b84 100644 --- a/kubeflow/datadog_checks/kubeflow/__about__.py +++ b/kubeflow/datadog_checks/kubeflow/__about__.py @@ -1,4 +1,4 @@ # (C) Datadog, Inc. 2024-present # All rights reserved # Licensed under a 3-clause BSD style license (see LICENSE) -__version__ = '0.0.1' +__version__ = '1.0.0' diff --git a/kubeflow/manifest.json b/kubeflow/manifest.json index 2252c0462db06..96eebed0294d9 100644 --- a/kubeflow/manifest.json +++ b/kubeflow/manifest.json @@ -2,7 +2,7 @@ "manifest_version": "2.0.0", "app_uuid": "0db7b333-38a1-4e09-af1b-317da2f9f413", "app_id": "kubeflow", - "display_on_public_website": false, + "display_on_public_website": true, "tile": { "overview": "README.md#Overview", "configuration": "README.md#Setup", diff --git a/kubevirt_api/CHANGELOG.md b/kubevirt_api/CHANGELOG.md index fd98fa4837529..41fab8d0f2a51 100644 --- a/kubevirt_api/CHANGELOG.md +++ b/kubevirt_api/CHANGELOG.md @@ -1,3 +1,13 @@ # CHANGELOG - KubeVirt API + +## 1.0.0 / 2024-10-04 + +***Added***: + +* Initial Release ([#18179](https://github.com/DataDog/integrations-core/pull/18179)) + +***Fixed***: + +* Bump the version of datadog-checks-base to 37.0.0 ([#18617](https://github.com/DataDog/integrations-core/pull/18617)) diff --git a/kubevirt_api/changelog.d/18179.added b/kubevirt_api/changelog.d/18179.added deleted file mode 100644 index aa949b47b7b41..0000000000000 --- a/kubevirt_api/changelog.d/18179.added +++ /dev/null @@ -1 +0,0 @@ -Initial Release \ No newline at end of file diff --git a/kubevirt_api/changelog.d/18617.fixed b/kubevirt_api/changelog.d/18617.fixed deleted file mode 100644 index e759164f82784..0000000000000 --- a/kubevirt_api/changelog.d/18617.fixed +++ /dev/null @@ -1 +0,0 @@ -Bump the version of datadog-checks-base to 37.0.0 \ No newline at end of file diff --git a/kubevirt_api/datadog_checks/kubevirt_api/__about__.py b/kubevirt_api/datadog_checks/kubevirt_api/__about__.py index e9541ce83e9e5..acbfd1c866b84 100644 --- a/kubevirt_api/datadog_checks/kubevirt_api/__about__.py +++ b/kubevirt_api/datadog_checks/kubevirt_api/__about__.py @@ -1,4 +1,4 @@ # (C) Datadog, Inc. 2024-present # All rights reserved # Licensed under a 3-clause BSD style license (see LICENSE) -__version__ = '0.0.1' +__version__ = '1.0.0' diff --git a/kubevirt_api/manifest.json b/kubevirt_api/manifest.json index 5db35a9102ba1..68274dbad8c03 100644 --- a/kubevirt_api/manifest.json +++ b/kubevirt_api/manifest.json @@ -42,7 +42,8 @@ }, "service_checks": { "metadata_path": "assets/service_checks.json" - } + }, + "process_signatures": ["virt-api"] }, "dashboards": { "KubeVirt Overview": "assets/dashboards/kubevirt_overview.json" diff --git a/kubevirt_api/metadata.csv b/kubevirt_api/metadata.csv index c2c50890e7599..523b73c43bc18 100644 --- a/kubevirt_api/metadata.csv +++ b/kubevirt_api/metadata.csv @@ -1,5 +1,5 @@ metric_name,metric_type,interval,unit_name,per_unit_name,description,orientation,integration,short_name,curated_metric,sample_tags -kubevirt_api.can_connect,gauge,,,,Whether the check can connect to the KubeVirt API or not.,0,kubevirt_api,api connect,, +kubevirt_api.can_connect,gauge,,,,"Value of 1 if the agent can connect to the KubeVirt Handler, and 0 otherwise.",0,kubevirt_api,api connect,, kubevirt_api.process.cpu_seconds.count,count,,second,,Total user and system CPU time spent in seconds.,0,kubevirt_api,cpu time total,, kubevirt_api.process.max_fds,gauge,,file,,Maximum number of open file descriptors.,0,kubevirt_api,max fds,, kubevirt_api.process.open_fds,gauge,,file,,Number of open file descriptors.,0,kubevirt_api,open fds,, diff --git a/kubevirt_controller/CHANGELOG.md b/kubevirt_controller/CHANGELOG.md index 63ef4768a3096..faadd0fcdb0fc 100644 --- a/kubevirt_controller/CHANGELOG.md +++ b/kubevirt_controller/CHANGELOG.md @@ -2,3 +2,12 @@ +## 1.0.0 / 2024-10-04 + +***Added***: + +* Initial Release ([#18186](https://github.com/DataDog/integrations-core/pull/18186)) + +***Fixed***: + +* Bump the version of datadog-checks-base to 37.0.0 ([#18617](https://github.com/DataDog/integrations-core/pull/18617)) diff --git a/kubevirt_controller/changelog.d/18186.added b/kubevirt_controller/changelog.d/18186.added deleted file mode 100644 index aa949b47b7b41..0000000000000 --- a/kubevirt_controller/changelog.d/18186.added +++ /dev/null @@ -1 +0,0 @@ -Initial Release \ No newline at end of file diff --git a/kubevirt_controller/changelog.d/18617.fixed b/kubevirt_controller/changelog.d/18617.fixed deleted file mode 100644 index e759164f82784..0000000000000 --- a/kubevirt_controller/changelog.d/18617.fixed +++ /dev/null @@ -1 +0,0 @@ -Bump the version of datadog-checks-base to 37.0.0 \ No newline at end of file diff --git a/kubevirt_controller/datadog_checks/kubevirt_controller/__about__.py b/kubevirt_controller/datadog_checks/kubevirt_controller/__about__.py index e9541ce83e9e5..acbfd1c866b84 100644 --- a/kubevirt_controller/datadog_checks/kubevirt_controller/__about__.py +++ b/kubevirt_controller/datadog_checks/kubevirt_controller/__about__.py @@ -1,4 +1,4 @@ # (C) Datadog, Inc. 2024-present # All rights reserved # Licensed under a 3-clause BSD style license (see LICENSE) -__version__ = '0.0.1' +__version__ = '1.0.0' diff --git a/kubevirt_controller/datadog_checks/kubevirt_controller/check.py b/kubevirt_controller/datadog_checks/kubevirt_controller/check.py index 4a4063870f68c..591857bc6964a 100644 --- a/kubevirt_controller/datadog_checks/kubevirt_controller/check.py +++ b/kubevirt_controller/datadog_checks/kubevirt_controller/check.py @@ -63,6 +63,10 @@ def _parse_config(self): "namespace": self.__NAMESPACE__, "enable_health_service_check": False, "tls_verify": self.tls_verify, + "rename_labels": { + "namespace": "vm_namespace", + # both kubevirt_vm_* and kubevirt_vmi_* metrics share the same name and namespace labels + }, } self.scraper_configs.append(instance) diff --git a/kubevirt_controller/manifest.json b/kubevirt_controller/manifest.json index 8600a63ef8a5e..552896c23a325 100644 --- a/kubevirt_controller/manifest.json +++ b/kubevirt_controller/manifest.json @@ -42,7 +42,8 @@ }, "service_checks": { "metadata_path": "assets/service_checks.json" - } + }, + "process_signatures": ["virt-controller"] } }, "author": { diff --git a/kubevirt_controller/metadata.csv b/kubevirt_controller/metadata.csv index bc3261e7b18af..0f1b194405586 100644 --- a/kubevirt_controller/metadata.csv +++ b/kubevirt_controller/metadata.csv @@ -1,35 +1,35 @@ metric_name,metric_type,interval,unit_name,per_unit_name,description,orientation,integration,short_name,curated_metric,sample_tags -kubevirt_controller.can_connect,gauge,,,,Whether the check can connect to the KubeVirt Controller or not.,0,kubevirt_controller,api connect,, +kubevirt_controller.can_connect,gauge,,,,"Value of 1 if the agent can connect to the KubeVirt Controller, and 0 otherwise.",0,kubevirt_controller,api connect,, kubevirt_controller.virt_controller.leading_status,gauge,,,,Indication for an operating virt-controller.,0,kubevirt_controller,leading_status,, kubevirt_controller.virt_controller.ready_status,gauge,,,,Indication for a virt-controller that is ready to take the lead.,0,kubevirt_controller,ready_status,, -kubevirt_controller.vm.error_status_last_transition_timestamp_seconds.count,count,,,,Virtual Machine last transition timestamp to error status.,0,kubevirt_controller,,, -kubevirt_controller.vm.migrating_status_last_transition_timestamp_seconds.count,count,,,,Virtual Machine last transition timestamp to migrating status.,0,kubevirt_controller,,, -kubevirt_controller.vm.non_running_status_last_transition_timestamp_seconds.count,count,,,,Virtual Machine last transition timestamp to paused/stopped status.,0,kubevirt_controller,,, -kubevirt_controller.vm.running_status_last_transition_timestamp_seconds.count,count,,,,Virtual Machine last transition timestamp to running status.,0,kubevirt_controller,,, -kubevirt_controller.vm.starting_status_last_transition_timestamp_seconds.count,count,,,,Virtual Machine last transition timestamp to starting status.,0,kubevirt_controller,,, +kubevirt_controller.vm.error_status_last_transition_timestamp_seconds.count,count,,second,,Virtual Machine last transition timestamp to error status.,0,kubevirt_controller,,, +kubevirt_controller.vm.migrating_status_last_transition_timestamp_seconds.count,count,,second,,Virtual Machine last transition timestamp to migrating status.,0,kubevirt_controller,,, +kubevirt_controller.vm.non_running_status_last_transition_timestamp_seconds.count,count,,second,,Virtual Machine last transition timestamp to paused/stopped status.,0,kubevirt_controller,,, +kubevirt_controller.vm.running_status_last_transition_timestamp_seconds.count,count,,second,,Virtual Machine last transition timestamp to running status.,0,kubevirt_controller,,, +kubevirt_controller.vm.starting_status_last_transition_timestamp_seconds.count,count,,second,,Virtual Machine last transition timestamp to starting status.,0,kubevirt_controller,,, kubevirt_controller.vmi.migrations_in_pending_phase,gauge,,,,Number of current pending migrations.,0,kubevirt_controller,,, kubevirt_controller.vmi.migrations_in_running_phase,gauge,,,,Number of current running migrations.,0,kubevirt_controller,,, kubevirt_controller.vmi.migrations_in_scheduling_phase,gauge,,,,Number of current scheduling migrations.,0,kubevirt_controller,,, kubevirt_controller.vmi.non_evictable,gauge,,,,Indication for a VirtualMachine that its eviction strategy is set to Live Migration but is not migratable.,0,kubevirt_controller,,, kubevirt_controller.vmi.number_of_outdated,gauge,,,,Indication for the total number of VirtualMachineInstance workloads that are not running within the most up-to-date version of the virt-launcher environment.,0,kubevirt_controller,,, kubevirt_controller.vmi.phase_count,gauge,,,,"Sum of VMIs per phase and node. phase can be one of the following: [Pending, Scheduling, Scheduled, Running, Succeeded, Failed, Unknown].",0,kubevirt_controller,,, -kubevirt_controller.vmi.phase_transition_time_from_creation_seconds.bucket,count,,,,Histogram of VM phase transitions duration from creation time in seconds.,0,kubevirt_controller,,, -kubevirt_controller.vmi.phase_transition_time_from_creation_seconds.count,count,,,,Histogram of VM phase transitions duration from creation time in seconds.,0,kubevirt_controller,,, -kubevirt_controller.vmi.phase_transition_time_from_creation_seconds.sum,count,,,,Histogram of VM phase transitions duration from creation time in seconds.,0,kubevirt_controller,,, -kubevirt_controller.vmi.phase_transition_time_from_deletion_seconds.bucket,count,,,,Histogram of VM phase transitions duration from deletion time in seconds.,0,kubevirt_controller,,, -kubevirt_controller.vmi.phase_transition_time_from_deletion_seconds.count,count,,,,Histogram of VM phase transitions duration from deletion time in seconds.,0,kubevirt_controller,,, -kubevirt_controller.vmi.phase_transition_time_from_deletion_seconds.sum,count,,,,Histogram of VM phase transitions duration from deletion time in seconds.,0,kubevirt_controller,,, -kubevirt_controller.vmi.phase_transition_time_seconds.bucket,count,,,,Histogram of VM phase transitions duration between different phases in seconds.,0,kubevirt_controller,,, -kubevirt_controller.vmi.phase_transition_time_seconds.count,count,,,,Histogram of VM phase transitions duration between different phases in seconds.,0,kubevirt_controller,,, -kubevirt_controller.vmi.phase_transition_time_seconds.sum,count,,,,Histogram of VM phase transitions duration between different phases in seconds.,0,kubevirt_controller,,, -kubevirt_controller.workqueue.adds.count,count,,,,Total number of adds handled by workqueue,0,kubevirt_controller,,, -kubevirt_controller.workqueue.depth,gauge,,,,Current depth of workqueue,0,kubevirt_controller,,, -kubevirt_controller.workqueue.longest_running_processor_seconds,gauge,,,,How many seconds has the longest running processor for workqueue been running.,0,kubevirt_controller,,, -kubevirt_controller.workqueue.queue_duration_seconds.bucket,count,,,,How long an item stays in workqueue before being requested.,0,kubevirt_controller,,, -kubevirt_controller.workqueue.queue_duration_seconds.count,count,,,,How long an item stays in workqueue before being requested.,0,kubevirt_controller,,, -kubevirt_controller.workqueue.queue_duration_seconds.sum,count,,,,How long an item stays in workqueue before being requested.,0,kubevirt_controller,,, +kubevirt_controller.vmi.phase_transition_time_from_creation_seconds.bucket,count,,second,,Histogram of VM phase transitions duration from creation time in seconds.,0,kubevirt_controller,,, +kubevirt_controller.vmi.phase_transition_time_from_creation_seconds.count,count,,second,,Histogram of VM phase transitions duration from creation time in seconds.,0,kubevirt_controller,,, +kubevirt_controller.vmi.phase_transition_time_from_creation_seconds.sum,count,,second,,Histogram of VM phase transitions duration from creation time in seconds.,0,kubevirt_controller,,, +kubevirt_controller.vmi.phase_transition_time_from_deletion_seconds.bucket,count,,second,,Histogram of VM phase transitions duration from deletion time in seconds.,0,kubevirt_controller,,, +kubevirt_controller.vmi.phase_transition_time_from_deletion_seconds.count,count,,second,,Histogram of VM phase transitions duration from deletion time in seconds.,0,kubevirt_controller,,, +kubevirt_controller.vmi.phase_transition_time_from_deletion_seconds.sum,count,,second,,Histogram of VM phase transitions duration from deletion time in seconds.,0,kubevirt_controller,,, +kubevirt_controller.vmi.phase_transition_time_seconds.bucket,count,,second,,Histogram of VM phase transitions duration between different phases in seconds.,0,kubevirt_controller,,, +kubevirt_controller.vmi.phase_transition_time_seconds.count,count,,second,,Histogram of VM phase transitions duration between different phases in seconds.,0,kubevirt_controller,,, +kubevirt_controller.vmi.phase_transition_time_seconds.sum,count,,second,,Histogram of VM phase transitions duration between different phases in seconds.,0,kubevirt_controller,,, +kubevirt_controller.workqueue.adds.count,count,,item,,Total number of adds handled by workqueue,0,kubevirt_controller,,, +kubevirt_controller.workqueue.depth,gauge,,item,,Current depth of workqueue,0,kubevirt_controller,,, +kubevirt_controller.workqueue.longest_running_processor_seconds,gauge,,second,,How many seconds has the longest running processor for workqueue been running.,0,kubevirt_controller,,, +kubevirt_controller.workqueue.queue_duration_seconds.bucket,count,,second,,How long an item stays in workqueue before being requested.,0,kubevirt_controller,,, +kubevirt_controller.workqueue.queue_duration_seconds.count,count,,second,,How long an item stays in workqueue before being requested.,0,kubevirt_controller,,, +kubevirt_controller.workqueue.queue_duration_seconds.sum,count,,second,,How long an item stays in workqueue before being requested.,0,kubevirt_controller,,, kubevirt_controller.workqueue.retries.count,count,,,,Total number of retries handled by workqueue.,0,kubevirt_controller,,, -kubevirt_controller.workqueue.unfinished_work_seconds,gauge,,,,How many seconds of work has done that is in progress and hasn't been observed by work_duration. Large values indicate stuck threads. One can deduce the number of stuck threads by observing the rate at which this increases.,0,kubevirt_controller,,, -kubevirt_controller.workqueue.work_duration_seconds.bucket,count,,,,How long in seconds processing an item from workqueue takes.,0,kubevirt_controller,,, -kubevirt_controller.workqueue.work_duration_seconds.count,count,,,,How long in seconds processing an item from workqueue takes.,0,kubevirt_controller,,, -kubevirt_controller.workqueue.work_duration_seconds.sum,count,,,,How long in seconds processing an item from workqueue takes.,0,kubevirt_controller,,, +kubevirt_controller.workqueue.unfinished_work_seconds,gauge,,second,,How many seconds of work has done that is in progress and hasn't been observed by work_duration. Large values indicate stuck threads. One can deduce the number of stuck threads by observing the rate at which this increases.,0,kubevirt_controller,,, +kubevirt_controller.workqueue.work_duration_seconds.bucket,count,,second,,How long in seconds processing an item from workqueue takes.,0,kubevirt_controller,,, +kubevirt_controller.workqueue.work_duration_seconds.count,count,,second,,How long in seconds processing an item from workqueue takes.,0,kubevirt_controller,,, +kubevirt_controller.workqueue.work_duration_seconds.sum,count,,second,,How long in seconds processing an item from workqueue takes.,0,kubevirt_controller,,, diff --git a/kubevirt_handler/CHANGELOG.md b/kubevirt_handler/CHANGELOG.md index 934b76a9af53b..efb69c7b7ace0 100644 --- a/kubevirt_handler/CHANGELOG.md +++ b/kubevirt_handler/CHANGELOG.md @@ -1,3 +1,9 @@ # CHANGELOG - KubeVirt Handler + +## 1.0.0 / 2024-10-04 + +***Added***: + +* Initial Release ([#18283](https://github.com/DataDog/integrations-core/pull/18283)) diff --git a/kubevirt_handler/changelog.d/18283.added b/kubevirt_handler/changelog.d/18283.added deleted file mode 100644 index aa949b47b7b41..0000000000000 --- a/kubevirt_handler/changelog.d/18283.added +++ /dev/null @@ -1 +0,0 @@ -Initial Release \ No newline at end of file diff --git a/kubevirt_handler/datadog_checks/kubevirt_handler/__about__.py b/kubevirt_handler/datadog_checks/kubevirt_handler/__about__.py index e9541ce83e9e5..acbfd1c866b84 100644 --- a/kubevirt_handler/datadog_checks/kubevirt_handler/__about__.py +++ b/kubevirt_handler/datadog_checks/kubevirt_handler/__about__.py @@ -1,4 +1,4 @@ # (C) Datadog, Inc. 2024-present # All rights reserved # Licensed under a 3-clause BSD style license (see LICENSE) -__version__ = '0.0.1' +__version__ = '1.0.0' diff --git a/kubevirt_handler/datadog_checks/kubevirt_handler/check.py b/kubevirt_handler/datadog_checks/kubevirt_handler/check.py index f009596b2cc1a..26d77e4e2566f 100644 --- a/kubevirt_handler/datadog_checks/kubevirt_handler/check.py +++ b/kubevirt_handler/datadog_checks/kubevirt_handler/check.py @@ -61,6 +61,9 @@ def _parse_config(self): "namespace": self.__NAMESPACE__, "enable_health_service_check": False, "tls_verify": self.tls_verify, + "rename_labels": { + "namespace": "vm_namespace", + }, } self.scraper_configs.append(instance) diff --git a/kubevirt_handler/images/IMAGES_README.md b/kubevirt_handler/images/IMAGES_README.md deleted file mode 100644 index 70612c2830ffb..0000000000000 --- a/kubevirt_handler/images/IMAGES_README.md +++ /dev/null @@ -1,41 +0,0 @@ -# Marketplace Media Carousel Guidelines - -## Using the media gallery - -To use the media gallery, you must upload a minimum of one image. The gallery -can hold a maximum of 8 pieces of media total, and one of these pieces of media -can be a video (guidelines and submission steps below). Images should be -added to your /images directory and referenced in the manifest.json file. - - -## Image and video requirements - -### Images - -``` -File type : .jpg or .png -File size : ~500 KB per image, with a max of 1 MB per image -File dimensions : The image must be between 1440px and 2880px width, with a 16:9 aspect ratio (for example: 1440x810) -File name : Use only letters, numbers, underscores, and hyphens -Color mode : RGB -Color profile : sRGB -Description : 300 characters maximum -``` - -### Video - -To display a video in your media gallery, please send our team the zipped file -or a link to download the video at `marketplace@datadog.com`. In addition, -please upload a thumbnail image for your video as a part of the pull request. -Once approved, we will upload the file to Vimeo and provide you with the -vimeo_id to add to your manifest.json file. Please note that the gallery can -only hold one video. - -``` -File type : MP4 H.264 -File size : Max 1 video; 1 GB maximum size -File dimensions : The aspect ratio must be exactly 16:9, and the resolution must be 1920x1080 or higher -File name : partnerName-appName.mp4 -Run time : Recommendation of 60 seconds or less -Description : 300 characters maximum -``` diff --git a/kubevirt_handler/manifest.json b/kubevirt_handler/manifest.json index ea197c1090e70..a9a0d6db1620c 100644 --- a/kubevirt_handler/manifest.json +++ b/kubevirt_handler/manifest.json @@ -35,14 +35,15 @@ "metrics": { "prefix": "kubevirt_handler.", "check": [ - "kubevirt_handler.can_connect", + "kubevirt_handler.can_connect", "kubevirt_handler.vmi.cpu_system_usage_seconds.count" ], "metadata_path": "metadata.csv" }, "service_checks": { "metadata_path": "assets/service_checks.json" - } + }, + "process_signatures": ["virt-handler"] } }, "author": { diff --git a/kubevirt_handler/metadata.csv b/kubevirt_handler/metadata.csv index 1b9faba853995..7f697de4cae94 100644 --- a/kubevirt_handler/metadata.csv +++ b/kubevirt_handler/metadata.csv @@ -1,5 +1,5 @@ metric_name,metric_type,interval,unit_name,per_unit_name,description,orientation,integration,short_name,curated_metric,sample_tags -kubevirt_handler.can_connect,gauge,,,,Whether the check can connect to the KubeVirt Handler or not.,0,kubevirt_handler,handler connect,, +kubevirt_handler.can_connect,gauge,,,,"Value of 1 if the agent can connect to the KubeVirt Handler, and 0 otherwise.",0,kubevirt_handler,handler connect,, kubevirt_handler.vmi.cpu_system_usage_seconds.count,count,,second,,"Total CPU time spent in system mode.",0,kubevirt_handler,short_name,, kubevirt_handler.vmi.cpu_usage_seconds.count,count,,second,,"Total CPU time spent in all modes (sum of both vcpu and hypervisor usage).",0,kubevirt_handler,short_name,, kubevirt_handler.vmi.cpu_user_usage_seconds.count,count,,second,,"Total CPU time spent in user mode.",0,kubevirt_handler,short_name,, diff --git a/mimecast/README.md b/mimecast/README.md index 721e1d0a177d7..29266260e8109 100644 --- a/mimecast/README.md +++ b/mimecast/README.md @@ -33,7 +33,7 @@ To find your application's details in Mimecast Email Security: - Application Name: Enter the application name of your choice - Category: Select **SIEM Integration** - Products: Click **Select all** option - - Application Role: Select **SIEM Admin Role** + - Application Role: Select **Basic Administrator** - Description: Enter the description of your choice - In **Notifications**, provide the contact details of your technical administrator and click on **Next** - After clicking on **Add and Generate Keys** there will be pop up window showing Client ID and Client Secret. Please copy those keys to a safe place as they won't be displayed again. diff --git a/mongo/assets/configuration/spec.yaml b/mongo/assets/configuration/spec.yaml index defd4a34db300..762a49a836450 100644 --- a/mongo/assets/configuration/spec.yaml +++ b/mongo/assets/configuration/spec.yaml @@ -218,7 +218,6 @@ files: example: 10 - name: slow_operations description: Configure collection of MongoDB slow operations. - hidden: true options: - name: enabled description: | diff --git a/mongo/changelog.d/18761.added b/mongo/changelog.d/18761.added new file mode 100644 index 0000000000000..9af73df70e76a --- /dev/null +++ b/mongo/changelog.d/18761.added @@ -0,0 +1,4 @@ +Add `resolved_views`, `working_millis`, and `queues` fields to the slow query (dbm only) event payload +- resolved_views: contains view details for slow queries on views (MongoDB 5.0+) +- working_millis: the amount of time that MongoDB spends working on the operation (MongoDB 8.0+) +- queues: contains information about the operation's queues (MongoDB 8.0+) \ No newline at end of file diff --git a/mongo/datadog_checks/mongo/data/conf.yaml.example b/mongo/datadog_checks/mongo/data/conf.yaml.example index 25140de02139f..9d978b840f0e1 100644 --- a/mongo/datadog_checks/mongo/data/conf.yaml.example +++ b/mongo/datadog_checks/mongo/data/conf.yaml.example @@ -182,6 +182,26 @@ instances: # # collection_interval: 10 + ## Configure collection of MongoDB slow operations. + # + # slow_operations: + + ## @param enabled - boolean - optional - default: true + ## Enable collection of slow operations. Requires `dbm: true`. + # + # enabled: true + + ## @param collection_interval - number - optional - default: 10 + ## Set the slow operations collection interval in seconds. Each collection involves capturing + ## profiled slow operations from sys.profile collection or from the Slow Query log. + # + # collection_interval: 10 + + ## @param max_operations - number - optional - default: 1000 + ## Set the maximum number of slow operations to collect per interval. + # + # max_operations: 1000 + ## Configure collection of MongoDB schemas (inferred) by sampling documents. # # schemas: diff --git a/mongo/datadog_checks/mongo/dbm/slow_operations.py b/mongo/datadog_checks/mongo/dbm/slow_operations.py index 40b2f10c91360..aafd587dbb699 100644 --- a/mongo/datadog_checks/mongo/dbm/slow_operations.py +++ b/mongo/datadog_checks/mongo/dbm/slow_operations.py @@ -257,7 +257,7 @@ def _create_slow_operation_event(self, slow_operation): "user": slow_operation.get("user"), # only available with profiling "application": slow_operation.get("appName"), # only available with profiling "statement": slow_operation["obfuscated_command"], - "query_hash": slow_operation.get("queryHash"), # only available with profiling + "query_hash": slow_operation.get("queryHash") or slow_operation.get("planCacheShapeHash"), "plan_cache_key": slow_operation.get("planCacheKey"), # only available with profiling "query_framework": slow_operation.get("queryFramework"), "comment": slow_operation["command"].get("comment"), @@ -290,6 +290,11 @@ def _create_slow_operation_event(self, slow_operation): "cursor": self._get_slow_operation_cursor(slow_operation), "lock_stats": self._get_slow_operation_lock_stats(slow_operation), "flow_control_stats": self._get_slow_operation_flow_control_stats(slow_operation), + # MongoDB 5.0+ specific fields + "resolved_views": self._get_slow_operation_resolved_views(slow_operation), + # MongoDB 8.0+ specific fields + "working_millis": slow_operation.get("workingMillis"), # the amount of time spends working on the operation + "queues": self._get_slow_operation_queues(slow_operation), } return self._sanitize_event(event) @@ -372,6 +377,21 @@ def _get_slow_operation_flow_control_stats(self, slow_operation): return format_key_name(self._check.convert_to_underscore_separated, flow_control_stats) return None + def _get_slow_operation_queues(self, slow_operation): + queues = slow_operation.get("queues") + if queues: + return format_key_name(self._check.convert_to_underscore_separated, queues) + return + + def _get_slow_operation_resolved_views(self, slow_operation): + resolved_views = slow_operation.get("resolvedViews") + result = [] + if resolved_views: + for view in resolved_views: + view.pop("resolvedPipeline", None) + result.append(format_key_name(self._check.convert_to_underscore_separated, view)) + return result or None + def _sanitize_event(self, event): # remove empty fields return {k: v for k, v in event.items() if v is not None} diff --git a/mongo/hatch.toml b/mongo/hatch.toml index 30c0e594ed026..fbd7a3fd7c0b3 100644 --- a/mongo/hatch.toml +++ b/mongo/hatch.toml @@ -8,7 +8,7 @@ dependencies = [ [[envs.default.matrix]] python = ["3.12"] -version = ["4.4", "5.0", "6.0", "7.0"] +version = ["4.4", "5.0", "6.0", "7.0", "8.0"] flavor = ["standalone", "shard", "auth", "tls"] [envs.default.overrides] diff --git a/mongo/tests/fixtures/getLog-standalone b/mongo/tests/fixtures/getLog-standalone index 260f95874a802..4a2f95fca9236 100644 --- a/mongo/tests/fixtures/getLog-standalone +++ b/mongo/tests/fixtures/getLog-standalone @@ -2,14 +2,15 @@ "totalLinesWritten": 1306, "log": [ "{\"t\":{\"$date\":\"2023-04-01T18:45:38.733+00:00\"},\"s\":\"I\", \"c\":\"WTCHKPT\", \"id\":22430, \"ctx\":\"Checkpointer\",\"msg\":\"WiredTiger message\",\"attr\":{\"message\":{\"ts_sec\":1720550738,\"ts_usec\":733701,\"thread\":\"1:0xffff796debc0\",\"session_name\":\"WT_SESSION.checkpoint\",\"category\":\"WT_VERB_CHECKPOINT_PROGRESS\",\"category_id\":6,\"verbose_level\":\"DEBUG_1\",\"verbose_level_id\":1,\"msg\":\"saving checkpoint snapshot min: 1332074, snapshot max: 1332074 snapshot count: 0, oldest timestamp: (0, 0) , meta checkpoint timestamp: (0, 0) base write gen: 7015\"}}}\n", - "{\"t\":{\"$date\":\"2023-04-01T18:45:41.386+00:00\"},\"s\":\"I\", \"c\":\"WRITE\", \"id\":51803, \"ctx\":\"conn50\",\"msg\":\"Slow query\",\"attr\":{\"type\":\"update\",\"ns\":\"test.customers\",\"command\":{\"q\":{\"age\":{\"$gt\":35}},\"u\":{\"$set\":{\"subscribed\":true}},\"multi\":true,\"upsert\":false,\"comment\":\"update customers subscription status by age\"},\"planSummary\":\"IXSCAN { age: -1 }\",\"keysExamined\":487,\"docsExamined\":487,\"nMatched\":487,\"nModified\":487,\"nUpserted\":0,\"keysInserted\":487,\"keysDeleted\":487,\"numYields\":0,\"locks\":{\"ParallelBatchWriterMode\":{\"acquireCount\":{\"r\":1}},\"FeatureCompatibilityVersion\":{\"acquireCount\":{\"w\":1}},\"ReplicationStateTransition\":{\"acquireCount\":{\"w\":1}},\"Global\":{\"acquireCount\":{\"w\":1}},\"Database\":{\"acquireCount\":{\"w\":1}},\"Collection\":{\"acquireCount\":{\"w\":1}}},\"flowControl\":{\"acquireCount\":1},\"storage\":{},\"cpuNanos\":11530375,\"remote\":\"192.168.65.1:58133\",\"durationMillis\":11}}\n", + "{\"t\":{\"$date\":\"2023-04-01T18:45:41.386+00:00\"},\"s\":\"I\", \"c\":\"WRITE\", \"id\":51803, \"ctx\":\"conn50\",\"msg\":\"Slow query\",\"attr\":{\"type\":\"update\",\"ns\":\"test.customers\",\"command\":{\"q\":{\"age\":{\"$gt\":35}},\"u\":{\"$set\":{\"subscribed\":true}},\"multi\":true,\"upsert\":false,\"comment\":\"update customers subscription status by age\"},\"planSummary\":\"IXSCAN { age: -1 }\",\"keysExamined\":487,\"docsExamined\":487,\"nMatched\":487,\"nModified\":487,\"nUpserted\":0,\"keysInserted\":487,\"keysDeleted\":487,\"numYields\":0,\"locks\":{\"ParallelBatchWriterMode\":{\"acquireCount\":{\"r\":1}},\"FeatureCompatibilityVersion\":{\"acquireCount\":{\"w\":1}},\"ReplicationStateTransition\":{\"acquireCount\":{\"w\":1}},\"Global\":{\"acquireCount\":{\"w\":1}},\"Database\":{\"acquireCount\":{\"w\":1}},\"Collection\":{\"acquireCount\":{\"w\":1}}},\"flowControl\":{\"acquireCount\":1},\"storage\":{},\"cpuNanos\":11530375,\"remote\":\"192.168.65.1:58133\",\"durationMillis\":11,\"workingMillis\":10}}\n", "{\"t\":{\"$date\":\"2024-04-01T18:45:41.387+00:00\"},\"s\":\"I\", \"c\":\"COMMAND\", \"id\":51803, \"ctx\":\"conn50\",\"msg\":\"Slow query\",\"attr\":{\"type\":\"command\",\"ns\":\"test.$cmd\",\"command\":{\"update\":\"customers\",\"ordered\":true,\"comment\":\"update customers subscription status by age\",\"$db\":\"test\"},\"numYields\":0,\"reslen\":60,\"locks\":{\"ParallelBatchWriterMode\":{\"acquireCount\":{\"r\":2}},\"FeatureCompatibilityVersion\":{\"acquireCount\":{\"r\":1,\"w\":2}},\"ReplicationStateTransition\":{\"acquireCount\":{\"w\":1}},\"Global\":{\"acquireCount\":{\"r\":1,\"w\":2}},\"Database\":{\"acquireCount\":{\"w\":2}},\"Collection\":{\"acquireCount\":{\"w\":2}}},\"flowControl\":{\"acquireCount\":1},\"storage\":{},\"cpuNanos\":11746542,\"remote\":\"192.168.65.1:58133\",\"protocol\":\"op_msg\",\"durationMillis\":11}}\n", "{\"t\":{\"$date\":\"2024-04-01T18:45:41.387+00:00\"},\"s\":\"I\", \"c\":\"COMMAND\", \"id\":51803, \"ctx\":\"conn50\",\"msg\":\"Slow query\",\"attr\":{\"type\":\"command\",\"ns\":\"test.$cmd\",\"command\":{\"update\":\"customers\",\"ordered\":true,\"comment\":\"update customers subscription status by age\",\"$db\":\"test\"},\"numYields\":0,\"reslen\":60,\"locks\":{\"ParallelBatchWriterMode\":{\"acquireCount\":{\"r\":2}},\"FeatureCompatibilityVersion\":{\"acquireCount\":{\"r\":1,\"w\":2}},\"ReplicationStateTransition\":{\"acquireCount\":{\"w\":1}},\"Global\":{\"acquireCount\":{\"r\":1,\"w\":2}},\"Database\":{\"acquireCount\":{\"w\":2}},\"Collection\":{\"acquireCount\":{\"w\":2}}},\"flowControl\":{\"acquireCount\":1},\"storage\":{},\"cpuNanos\":11746542,\"remote\":\"192.168.65.1:58133\",\"protocol\":\"op_msg\",\"durationMillis\":11}}\n", "{\"t\":{\"$date\":\"2024-04-01T18:45:41.387+00:00\"},\"s\":\"I\", \"c\":\"COMMAND\", \"id\":51803, \"ctx\":\"conn50\",\"msg\":\"Slow query\",\"attr\":{\"type\":\"command\",\"ns\":\"test.$cmd\",\"command\":{\"update\":\"customers\",\"ordered\":true,\"comment\":\"update customers subscription status by age\",\"$db\":\"test\"},\"numYields\":0,\"reslen\":60,\"locks\":{\"ParallelBatchWriterMode\":{\"acquireCount\":{\"r\":2}},\"FeatureCompatibilityVersion\":{\"acquireCount\":{\"r\":1,\"w\":2}},\"ReplicationStateTransition\":{\"acquireCount\":{\"w\":1}},\"Global\":{\"acquireCount\":{\"r\":1,\"w\":2}},\"Database\":{\"acquireCount\":{\"w\":2}},\"Collection\":{\"acquireCount\":{\"w\":2}}},\"flowControl\":{\"acquireCount\":1},\"storage\":{},\"cpuNanos\":11746542,\"remote\":\"192.168.65.1:58133\",\"protocol\":\"op_msg\",\"durationMillis\":11}}\n", - "{\"t\":{\"$date\":\"2999-04-01T18:45:43.991+00:00\"},\"s\":\"I\", \"c\":\"WRITE\", \"id\":51803, \"ctx\":\"conn50\",\"msg\":\"Slow query\",\"attr\":{\"type\":\"update\",\"ns\":\"test.customers\",\"command\":{\"q\":{\"age\":{\"$gt\":18}},\"u\":{\"$set\":{\"subscribed\":false}},\"multi\":true,\"upsert\":false,\"comment\":\"update customers subscription status by age\"},\"planSummary\":\"IXSCAN { age: -1 }\",\"keysExamined\":608,\"docsExamined\":608,\"nMatched\":608,\"nModified\":441,\"nUpserted\":0,\"keysInserted\":441,\"keysDeleted\":441,\"numYields\":4,\"locks\":{\"ParallelBatchWriterMode\":{\"acquireCount\":{\"r\":5}},\"FeatureCompatibilityVersion\":{\"acquireCount\":{\"w\":5}},\"ReplicationStateTransition\":{\"acquireCount\":{\"w\":5}},\"Global\":{\"acquireCount\":{\"w\":5}},\"Database\":{\"acquireCount\":{\"w\":5}},\"Collection\":{\"acquireCount\":{\"w\":5}}},\"flowControl\":{\"acquireCount\":5},\"storage\":{},\"cpuNanos\":74500749,\"remote\":\"192.168.65.1:58133\",\"durationMillis\":74}}\n", + "{\"t\":{\"$date\":\"2999-04-01T18:45:43.991+00:00\"},\"s\":\"I\", \"c\":\"WRITE\", \"id\":51803, \"ctx\":\"conn50\",\"msg\":\"Slow query\",\"attr\":{\"type\":\"update\",\"ns\":\"test.customers\",\"command\":{\"q\":{\"age\":{\"$gt\":18}},\"u\":{\"$set\":{\"subscribed\":false}},\"multi\":true,\"upsert\":false,\"comment\":\"update customers subscription status by age\"},\"planSummary\":\"IXSCAN { age: -1 }\",\"keysExamined\":608,\"docsExamined\":608,\"nMatched\":608,\"nModified\":441,\"nUpserted\":0,\"keysInserted\":441,\"keysDeleted\":441,\"numYields\":4,\"locks\":{\"ParallelBatchWriterMode\":{\"acquireCount\":{\"r\":5}},\"FeatureCompatibilityVersion\":{\"acquireCount\":{\"w\":5}},\"ReplicationStateTransition\":{\"acquireCount\":{\"w\":5}},\"Global\":{\"acquireCount\":{\"w\":5}},\"Database\":{\"acquireCount\":{\"w\":5}},\"Collection\":{\"acquireCount\":{\"w\":5}}},\"flowControl\":{\"acquireCount\":5},\"storage\":{},\"cpuNanos\":74500749,\"remote\":\"192.168.65.1:58133\",\"durationMillis\":74,\"queues\":{\"ingress\":{\"admissions\":7,\"totalTimeQueuedMicros\":0},\"execution\":{\"admissions\":328,\"totalTimeQueuedMicros\":2109}}}}\n", "{\"t\":{\"$date\":\"2999-04-01T18:45:43.991+00:00\"},\"s\":\"I\", \"c\":\"COMMAND\", \"id\":51803, \"ctx\":\"conn50\",\"msg\":\"Slow query\",\"attr\":{\"type\":\"command\",\"ns\":\"test.$cmd\",\"command\":{\"update\":\"customers\",\"ordered\":true,\"comment\":\"update customers subscription status by age\",\"$db\":\"test\"},\"numYields\":4,\"reslen\":60,\"locks\":{\"ParallelBatchWriterMode\":{\"acquireCount\":{\"r\":6}},\"FeatureCompatibilityVersion\":{\"acquireCount\":{\"r\":1,\"w\":6}},\"ReplicationStateTransition\":{\"acquireCount\":{\"w\":5}},\"Global\":{\"acquireCount\":{\"r\":1,\"w\":6}},\"Database\":{\"acquireCount\":{\"w\":6}},\"Collection\":{\"acquireCount\":{\"w\":6}}},\"flowControl\":{\"acquireCount\":5},\"storage\":{},\"cpuNanos\":74715957,\"remote\":\"192.168.65.1:58133\",\"protocol\":\"op_msg\",\"durationMillis\":74}}\n", - "{\"t\":{\"$date\":\"2999-04-01T18:45:45.007+00:00\"},\"s\":\"I\", \"c\":\"WRITE\", \"id\":51803, \"ctx\":\"conn50\",\"msg\":\"Slow query\",\"attr\":{\"type\":\"update\",\"ns\":\"test.customers\",\"command\":{\"q\":{\"age\":{\"$gt\":35}},\"u\":{\"$set\":{\"subscribed\":true}},\"multi\":true,\"upsert\":false,\"comment\":\"update customers subscription status by age\"},\"planSummary\":\"IXSCAN { age: -1 }\",\"keysExamined\":495,\"docsExamined\":495,\"nMatched\":495,\"nModified\":495,\"nUpserted\":0,\"keysInserted\":495,\"keysDeleted\":495,\"numYields\":2,\"locks\":{\"ParallelBatchWriterMode\":{\"acquireCount\":{\"r\":3}},\"FeatureCompatibilityVersion\":{\"acquireCount\":{\"w\":3}},\"ReplicationStateTransition\":{\"acquireCount\":{\"w\":3}},\"Global\":{\"acquireCount\":{\"w\":3}},\"Database\":{\"acquireCount\":{\"w\":3}},\"Collection\":{\"acquireCount\":{\"w\":3}}},\"flowControl\":{\"acquireCount\":3},\"storage\":{},\"cpuNanos\":27416375,\"remote\":\"192.168.65.1:58133\",\"durationMillis\":27}}\n", + "{\"t\":{\"$date\":\"2999-04-01T18:45:45.007+00:00\"},\"s\":\"I\", \"c\":\"WRITE\", \"id\":51803, \"ctx\":\"conn50\",\"msg\":\"Slow query\",\"attr\":{\"type\":\"update\",\"ns\":\"test.customers\",\"command\":{\"q\":{\"age\":{\"$gt\":35}},\"u\":{\"$set\":{\"subscribed\":true}},\"multi\":true,\"upsert\":false,\"comment\":\"update customers subscription status by age\"},\"planSummary\":\"IXSCAN { age: -1 }\",\"keysExamined\":495,\"docsExamined\":495,\"nMatched\":495,\"nModified\":495,\"nUpserted\":0,\"keysInserted\":495,\"keysDeleted\":495,\"numYields\":2,\"locks\":{\"ParallelBatchWriterMode\":{\"acquireCount\":{\"r\":3}},\"FeatureCompatibilityVersion\":{\"acquireCount\":{\"w\":3}},\"ReplicationStateTransition\":{\"acquireCount\":{\"w\":3}},\"Global\":{\"acquireCount\":{\"w\":3}},\"Database\":{\"acquireCount\":{\"w\":3}},\"Collection\":{\"acquireCount\":{\"w\":3}}},\"flowControl\":{\"acquireCount\":3},\"storage\":{},\"cpuNanos\":27416375,\"remote\":\"192.168.65.1:58133\",\"durationMillis\":27,\"workingMillis\":20,\"queues\":{\"ingress\":{\"admissions\":7,\"totalTimeQueuedMicros\":0},\"execution\":{\"admissions\":328,\"totalTimeQueuedMicros\":2109}}}}\n", "{\"t\":{\"$date\":\"2999-04-01T18:45:45.008+00:00\"},\"s\":\"I\", \"c\":\"COMMAND\", \"id\":51803, \"ctx\":\"conn50\",\"msg\":\"Slow query\",\"attr\":{\"type\":\"command\",\"ns\":\"test.$cmd\",\"command\":{\"update\":\"customers\",\"ordered\":true,\"comment\":\"update customers subscription status by age\",\"$db\":\"test\"},\"numYields\":2,\"reslen\":60,\"locks\":{\"ParallelBatchWriterMode\":{\"acquireCount\":{\"r\":4}},\"FeatureCompatibilityVersion\":{\"acquireCount\":{\"r\":1,\"w\":4}},\"ReplicationStateTransition\":{\"acquireCount\":{\"w\":3}},\"Global\":{\"acquireCount\":{\"r\":1,\"w\":4}},\"Database\":{\"acquireCount\":{\"w\":4}},\"Collection\":{\"acquireCount\":{\"w\":4}}},\"flowControl\":{\"acquireCount\":3},\"storage\":{},\"cpuNanos\":27721625,\"remote\":\"192.168.65.1:58133\",\"protocol\":\"op_msg\",\"durationMillis\":27}}\n", + "{\"t\":{\"$date\":\"2999-04-01T18:45:45.009+00:00\"},\"s\":\"I\", \"c\":\"COMMAND\", \"id\":51803, \"ctx\":\"conn50\",\"msg\":\"Slow query\",\"attr\":{\"type\":\"command\",\"ns\":\"test.myview\",\"command\":{\"update\":\"customers\",\"ordered\":true,\"comment\":\"update customers subscription status by age\",\"$db\":\"test\"},\"numYields\":2,\"reslen\":60,\"locks\":{\"ParallelBatchWriterMode\":{\"acquireCount\":{\"r\":4}},\"FeatureCompatibilityVersion\":{\"acquireCount\":{\"r\":1,\"w\":4}},\"ReplicationStateTransition\":{\"acquireCount\":{\"w\":3}},\"Global\":{\"acquireCount\":{\"r\":1,\"w\":4}},\"Database\":{\"acquireCount\":{\"w\":4}},\"Collection\":{\"acquireCount\":{\"w\":4}}},\"flowControl\":{\"acquireCount\":3},\"storage\":{},\"cpuNanos\":27721625,\"remote\":\"192.168.65.1:58133\",\"protocol\":\"op_msg\",\"durationMillis\":27,\"resolvedViews\":[{\"viewNamespace\":\"test.myview\",\"dependencyChain\":[\"myView\",\"myCollection\"],\"resolvedPipeline\":[{\"$sort\":{\"firstName\":1}}]}]}}\n", "{\"t\":{\"$date\":\"2999-04-01T18:45:45.040+00:00\"},\"s\":\"I\", \"c\":\"WRITE\", \"id\":51803, \"ctx\":\"conn50\",\"msg\":\"Slow query\",\"attr\":{\"type\":\"update\",\"ns\":\"test.customers\",\"command\":{\"q\":{\"age\":{\"$gt\":18}},\"u\":{\"$set\":{\"subscribed\":false}},\"multi\":true,\"upsert\":false,\"comment\":\"update customers subscription status by age\"},\"planSummary\":\"IXSCAN { age: -1 }\",\"keysExamined\":495,\"docsExamined\":495,\"nMatched\":495,\"nModified\":495,\"nUpserted\":0,\"keysInserted\":495,\"keysDeleted\":495,\"numYields\":0,\"locks\":{\"ParallelBatchWriterMode\":{\"acquireCount\":{\"r\":1}},\"FeatureCompatibilityVersion\":{\"acquireCount\":{\"w\":1}},\"ReplicationStateTransition\":{\"acquireCount\":{\"w\":1}},\"Global\":{\"acquireCount\":{\"w\":1}},\"Database\":{\"acquireCount\":{\"w\":1}},\"Collection\":{\"acquireCount\":{\"w\":1}}},\"flowControl\":{\"acquireCount\":1},\"storage\":{},\"cpuNanos\":14032417,\"remote\":\"192.168.65.1:58133\",\"durationMillis\":14}}\n", "{\"t\":{\"$date\":\"2999-04-01T18:45:45.041+00:00\"},\"s\":\"I\", \"c\":\"COMMAND\", \"id\":51803, \"ctx\":\"conn50\",\"msg\":\"Slow query\",\"attr\":{\"type\":\"command\",\"ns\":\"test.$cmd\",\"command\":{\"update\":\"customers\",\"ordered\":true,\"comment\":\"update customers subscription status by age\",\"$db\":\"test\"},\"numYields\":0,\"reslen\":60,\"locks\":{\"ParallelBatchWriterMode\":{\"acquireCount\":{\"r\":2}},\"FeatureCompatibilityVersion\":{\"acquireCount\":{\"r\":1,\"w\":2}},\"ReplicationStateTransition\":{\"acquireCount\":{\"w\":1}},\"Global\":{\"acquireCount\":{\"r\":1,\"w\":2}},\"Database\":{\"acquireCount\":{\"w\":2}},\"Collection\":{\"acquireCount\":{\"w\":2}}},\"flowControl\":{\"acquireCount\":1},\"storage\":{},\"cpuNanos\":15198542,\"remote\":\"192.168.65.1:58133\",\"protocol\":\"op_msg\",\"durationMillis\":15}}\n", "{\"t\":{\"$date\":\"2999-04-01T18:45:45.338+00:00\"},\"s\":\"I\", \"c\":\"COMMAND\", \"id\":51803, \"ctx\":\"conn50\",\"msg\":\"Slow query\",\"attr\":{\"type\":\"command\",\"ns\":\"test.customers\",\"command\":{\"find\":\"customers\",\"filter\":{\"subscribed\":false},\"sort\":{\"name\":1},\"comment\":\"query customers by subscription status\",\"limit\":151,\"$db\":\"test\"},\"planSummary\":\"IXSCAN { name: 1 }\",\"planningTimeMicros\":11891,\"cursorid\":5170666760900931828,\"keysExamined\":540,\"docsExamined\":540,\"fromPlanCache\":true,\"nBatches\":1,\"numYields\":0,\"nreturned\":101,\"queryHash\":\"A9774B9D\",\"planCacheKey\":\"0D12DCDF\",\"queryFramework\":\"classic\",\"reslen\":14089,\"locks\":{\"FeatureCompatibilityVersion\":{\"acquireCount\":{\"r\":1}},\"Global\":{\"acquireCount\":{\"r\":1}}},\"storage\":{},\"cpuNanos\":12071416,\"remote\":\"192.168.65.1:58133\",\"protocol\":\"op_msg\",\"durationMillis\":12}}\n", diff --git a/mongo/tests/results/slow-operations-standalone.json b/mongo/tests/results/slow-operations-standalone.json index 9fb87db7ad7c2..bcd604a916f39 100644 --- a/mongo/tests/results/slow-operations-standalone.json +++ b/mongo/tests/results/slow-operations-standalone.json @@ -265,6 +265,16 @@ }, "flow_control_stats": { "acquire_count": 5 + }, + "queues": { + "ingress": { + "admissions": 7, + "total_time_queued_micros": 0 + }, + "execution": { + "admissions": 328, + "total_time_queued_micros": 2109 + } } }, { @@ -375,6 +385,17 @@ }, "flow_control_stats": { "acquire_count": 3 + }, + "working_millis": 20, + "queues": { + "ingress": { + "admissions": 7, + "total_time_queued_micros": 0 + }, + "execution": { + "admissions": 328, + "total_time_queued_micros": 2109 + } } }, { @@ -430,6 +451,68 @@ "acquire_count": 3 } }, + { + "timestamp": 32479987545009.0, + "dbname": "test", + "op": "command", + "ns": "test.myview", + "query_signature": "c1df6c33f6b6c258", + "statement": "{\"update\": \"customers\", \"ordered\": true, \"$db\": \"test\"}", + "comment": "update customers subscription status by age", + "mills": 27, + "num_yields": 2, + "response_length": 60, + "cpu_nanos": 27721625, + "client": { + "hostname": "192.168.65.1:58133" + }, + "lock_stats": { + "parallel_batch_writer_mode": { + "acquire_count": { + "r": 4 + } + }, + "feature_compatibility_version": { + "acquire_count": { + "r": 1, + "w": 4 + } + }, + "replication_state_transition": { + "acquire_count": { + "w": 3 + } + }, + "global": { + "acquire_count": { + "r": 1, + "w": 4 + } + }, + "database": { + "acquire_count": { + "w": 4 + } + }, + "collection": { + "acquire_count": { + "w": 4 + } + } + }, + "flow_control_stats": { + "acquire_count": 3 + }, + "resolved_views": [ + { + "view_namespace": "test.myview", + "dependency_chain": [ + "myView", + "myCollection" + ] + } + ] + }, { "timestamp": 32479987545040.0, "dbname": "test", diff --git a/mongo/tests/test_dbm_slow_operations.py b/mongo/tests/test_dbm_slow_operations.py index ce747d3d28c86..392a6c336f98b 100644 --- a/mongo/tests/test_dbm_slow_operations.py +++ b/mongo/tests/test_dbm_slow_operations.py @@ -36,6 +36,8 @@ def test_mongo_slow_operations_standalone(aggregator, instance_integration_clust dbm_samples = aggregator.get_event_platform_events("dbm-samples") slow_operation_explain_plans_payload = [event for event in dbm_samples if event['dbm_type'] == 'plan'] + print(json.dumps(slow_operation_payload)) + with open(os.path.join(HERE, "results", "slow-operations-standalone.json"), 'r') as f: expected_slow_operation_payload = json.load(f) assert slow_operation_payload == expected_slow_operation_payload diff --git a/network/CHANGELOG.md b/network/CHANGELOG.md index 28d9395b39152..337302b4c0645 100644 --- a/network/CHANGELOG.md +++ b/network/CHANGELOG.md @@ -2,6 +2,12 @@ +## 4.1.0 / 2024-10-04 + +***Fixed***: + +* Fix metric type in new network metrics ([#18764](https://github.com/DataDog/integrations-core/pull/18764)) + ## 4.0.0 / 2024-10-01 ***Changed***: diff --git a/network/datadog_checks/network/__about__.py b/network/datadog_checks/network/__about__.py index c25988b9e6d8b..d83361248d388 100644 --- a/network/datadog_checks/network/__about__.py +++ b/network/datadog_checks/network/__about__.py @@ -2,4 +2,4 @@ # All rights reserved # Licensed under a 3-clause BSD style license (see LICENSE) -__version__ = '4.0.0' +__version__ = '4.1.0' diff --git a/network/datadog_checks/network/check_linux.py b/network/datadog_checks/network/check_linux.py index 986003b763052..7bd7cfc452256 100644 --- a/network/datadog_checks/network/check_linux.py +++ b/network/datadog_checks/network/check_linux.py @@ -304,9 +304,6 @@ def get_expected_metrics(self): ) return expected_metrics - def _submit_netmetric_gauge(self, metric, value, tags=None): - self.gauge(metric, value, tags=tags) - def _read_int_file(self, file_location): try: with open(file_location, 'r') as f: diff --git a/network/datadog_checks/network/check_windows.py b/network/datadog_checks/network/check_windows.py index fa56342dc77b7..3baa292d04cf3 100644 --- a/network/datadog_checks/network/check_windows.py +++ b/network/datadog_checks/network/check_windows.py @@ -128,6 +128,11 @@ def _tcp_stats(self, tags): 'dwOutRsts': '.out_resets', 'dwNumConns': '.connections', } + # similar to the linux check + nstat_metrics_gauge_names = [ + '.connections', + '.current_established', + ] proto_dict = {} tcp4stats = self._get_tcp_stats(socket.AF_INET) @@ -149,7 +154,10 @@ def _tcp_stats(self, tags): for fieldname in tcpstats_dict: fieldvalue = getattr(stats, fieldname) metric_name = "system.net." + str(proto) + tcpstats_dict[fieldname] - self.submit_netmetric(metric_name, fieldvalue, tags) + if tcpstats_dict[fieldname] in nstat_metrics_gauge_names: + self._submit_netmetric_gauge(metric_name, fieldvalue, tags) + else: + self.submit_netmetric(metric_name, fieldvalue, tags) def _parse_protocol_psutil(self, conn): """ diff --git a/network/datadog_checks/network/network.py b/network/datadog_checks/network/network.py index b9f6d5cf97cac..18761deef2846 100644 --- a/network/datadog_checks/network/network.py +++ b/network/datadog_checks/network/network.py @@ -274,6 +274,9 @@ def submit_netmetric(self, metric, value, tags=None): if self._collect_count_metrics: self.monotonic_count('{}.count'.format(metric), value, tags=tags) + def _submit_netmetric_gauge(self, metric, value, tags=None): + self.gauge(metric, value, tags=tags) + def submit_regexed_values(self, output, regex_list, tags): lines = output.splitlines() for line in lines: diff --git a/network/metadata.csv b/network/metadata.csv index 8d0db8d0ba21c..abb42cc3f11ec 100644 --- a/network/metadata.csv +++ b/network/metadata.csv @@ -105,9 +105,7 @@ system.net.tcp.attempt_fails.count,count,,connection,,The number of times TCP co system.net.tcp.established_resets,gauge,,connection,second,The number of times TCP connections have made a direct transition to the CLOSED state from either the ESTABLISHED state or the CLOSE-WAIT state (Linux or Windows only).,-1,system,tcp established rst, system.net.tcp.established_resets.count,count,,connection,,The number of times TCP connections have made a direct transition to the CLOSED state from either the ESTABLISHED state or the CLOSE-WAIT state (Linux or Windows only).,-1,system,tcp established rst count, system.net.tcp.current_established,gauge,,connection,,The number of TCP connections for which the current state is either ESTABLISHED or CLOSE-WAIT (Linux or Windows only).,0,system,tcp current estab, -system.net.tcp.current_established.count,count,,connection,,The number of TCP connections for which the current state is either ESTABLISHED or CLOSE-WAIT (Linux or Windows only).,0,system,tcp current estab count, system.net.tcp.connections,gauge,,segment,second,The number of TCP connections in all states except listening connections.,0,system,tcp connections, -system.net.tcp.connections.count,count,,segment,,Total count of TCP connections in all states except listening connections.,0,system,tcp connections count, system.net.tcp.in_errors,gauge,,packet,second,The total number of segments received in error (e.g. bad TCP checksums) (Linux or Windows only).,-1,system,tcp in errors, system.net.tcp.in_errors.count,count,,packet,,The total number of segments received in error (e.g. bad TCP checksums) (Linux or Windows only).,-1,system,tcp in errors count, system.net.tcp.out_resets,gauge,,packet,second,The number of TCP segments sent containing the RST flag (Linux or Windows only).,-1,system,tcp out rst, @@ -186,7 +184,6 @@ system.net.tcp4.unconn,gauge,,connection,,The number of TCP IPv4 connections in system.net.tcp4.active_opens,gauge,,connection,second,Number of times TCP IPv4 connections have made a direct transition to the SYN-SENT state from the CLOSED state (Windows only).,0,system,tcp4 active opens, system.net.tcp4.active_opens.count,count,,connection,,Number of times TCP IPv4 connections have made a direct transition to the SYN-SENT state from the CLOSED state (Windows only).,0,system,tcp4 active opens count, system.net.tcp4.connections,gauge,,segment,second,The number of TCP IPv4 connections connections in all states except listening connections (Windows only).,0,system,tcp4 connections, -system.net.tcp4.connections.count,count,,segment,,Total count of TCP IPv4 connections connections in all states except listening connections (Windows only).,0,system,tcp4 connections count, system.net.tcp4.passive_opens,gauge,,connection,second,Number of times TCP IPv4 connections have made a direct transition to the SYN-RCVD state from the LISTEN state (Windows only).,0,system,tcp4 passive opens, system.net.tcp4.passive_opens.count,count,,connection,,Number of times TCP IPv4 connections have made a direct transition to the SYN-RCVD state from the LISTEN state (Windows only).,0,system,tcp4 passive opens count, system.net.tcp4.attempt_fails,gauge,,connection,second,The number of times TCP IPv4 connections have made a direct transition to the CLOSED state from either the SYN-SENT state or the SYN-RCVD state plus the number of times TCP IPv4 connections have made a direct transition to the LISTEN state from the SYN-RCVD state (Windows only).,-1,system,tcp4 attempt fails, @@ -194,7 +191,6 @@ system.net.tcp4.attempt_fails.count,count,,connection,,The number of times TCP I system.net.tcp4.established_resets,gauge,,connection,second,The number of times TCP IPv4 connections have made a direct transition to the CLOSED state from either the ESTABLISHED state or the CLOSE-WAIT state (Windows only).,-1,system,tcp4 established rst, system.net.tcp4.established_resets.count,count,,connection,,The number of times TCP IPv4 connections have made a direct transition to the CLOSED state from either the ESTABLISHED state or the CLOSE-WAIT state (Windows only).,-1,system,tcp4 established rst count, system.net.tcp4.current_established,gauge,,segment,second,The number of currently established TCP IPv4 connections (Windows only).,0,system,tcp4 current established, -system.net.tcp4.current_established.count,count,,segment,,The number of currently established TCP IPv4 connections (Windows only).,0,system,tcp4 current established count, system.net.tcp4.in_errors,gauge,,packet,second,The total number of segments received in error (e.g. bad TCP IPv4 checksums) (Windows only).,-1,system,tcp4 in errors, system.net.tcp4.in_errors.count,count,,packet,,The total number of segments received in error (e.g. bad TCP IPv4 checksums) (Windows only).,-1,system,tcp4 in errors count, system.net.tcp4.out_resets,gauge,,packet,second,The number of TCP IPv4 segments sent containing the RST flag (Windows only).,-1,system,tcp4 out rst, @@ -222,7 +218,6 @@ system.net.tcp6.unconn,gauge,,connection,,The number of TCP IPv6 connections in system.net.tcp6.active_opens,gauge,,connection,second,Number of times TCP IPv6 connections have made a direct transition to the SYN-SENT state from the CLOSED state (Windows only).,0,system,tcp6 active opens, system.net.tcp6.active_opens.count,count,,connection,,Number of times TCP IPv6 connections have made a direct transition to the SYN-SENT state from the CLOSED state (Windows only).,0,system,tcp6 active opens count, system.net.tcp6.connections,gauge,,segment,second,The number of TCP IPv6 connections connections in all states except listening connections (Windows only).,0,system,tcp6 connections, -system.net.tcp6.connections.count,count,,segment,,Total count of TCP IPv6 connections connections in all states except listening connections (Windows only).,0,system,tcp6 connections count, system.net.tcp6.passive_opens,gauge,,connection,second,Number of times TCP IPv6 connections have made a direct transition to the SYN-RCVD state from the LISTEN state (Windows only).,0,system,tcp6 passive opens, system.net.tcp6.passive_opens.count,count,,connection,,Number of times TCP IPv6 connections have made a direct transition to the SYN-RCVD state from the LISTEN state (Windows only).,0,system,tcp6 passive opens count, system.net.tcp6.attempt_fails,gauge,,connection,second,The number of times TCP IPv6 connections have made a direct transition to the CLOSED state from either the SYN-SENT state or the SYN-RCVD state plus the number of times TCP IPv6 connections have made a direct transition to the LISTEN state from the SYN-RCVD state (Windows only).,-1,system,tcp6 attempt fails, @@ -230,7 +225,6 @@ system.net.tcp6.attempt_fails.count,count,,connection,,The number of times TCP I system.net.tcp6.established_resets,gauge,,connection,second,The number of times TCP IPv6 connections have made a direct transition to the CLOSED state from either the ESTABLISHED state or the CLOSE-WAIT state (Windows only).,-1,system,tcp6 established rst, system.net.tcp6.established_resets.count,count,,connection,,The number of times TCP IPv6 connections have made a direct transition to the CLOSED state from either the ESTABLISHED state or the CLOSE-WAIT state (Windows only).,-1,system,tcp6 established rst count, system.net.tcp6.current_established,gauge,,segment,second,The number of currently established TCP IPv6 connections (Windows only).,0,system,tcp6 current established, -system.net.tcp6.current_established.count,count,,segment,,The number of currently established TCP IPv6 connections (Windows only).,0,system,tcp6 current established count, system.net.tcp6.in_errors,gauge,,packet,second,The total number of segments received in error (e.g. bad TCP IPv6 checksums) (Windows only).,-1,system,tcp6 in errors, system.net.tcp6.in_errors.count,count,,packet,,The total number of segments received in error (e.g. bad TCP IPv6 checksums) (Windows only).,-1,system,tcp6 in errors count, system.net.tcp6.out_resets,gauge,,packet,second,The number of TCP IPv6 segments sent containing the RST flag (Windows only).,-1,system,tcp6 out rst, diff --git a/network/tests/test_windows.py b/network/tests/test_windows.py index 88c448c99ab25..02de9e0ba027d 100644 --- a/network/tests/test_windows.py +++ b/network/tests/test_windows.py @@ -45,6 +45,7 @@ def test_get_tcp_stats_failure(): def test_get_tcp_stats(aggregator): instance = copy.deepcopy(common.INSTANCE) + instance["collect_count_metrics"] = True check_instance = WindowsNetwork('network', {}, [instance]) mock_stats = TCPSTATS( @@ -99,12 +100,24 @@ def test_get_tcp_stats(aggregator): 'system.net.tcp.out_resets': 28, 'system.net.tcp.connections': 30, } + gauge_mets = [ + 'system.net.tcp4.connections', + 'system.net.tcp4.current_established', + 'system.net.tcp6.connections', + 'system.net.tcp6.current_established', + 'system.net.tcp.connections', + 'system.net.tcp.current_established', + ] with mock.patch('datadog_checks.network.check_windows.WindowsNetwork._get_tcp_stats') as mock_get_tcp_stats: mock_get_tcp_stats.return_value = mock_stats # Make _get_tcp_stats return my mock object check_instance.check({}) for name, value in expected_mets.items(): - aggregator.assert_metric(name, value=value) + if name in gauge_mets: + aggregator.assert_metric(name, value=value, metric_type=aggregator.GAUGE) + else: + aggregator.assert_metric(name, value=value, metric_type=aggregator.RATE) + aggregator.assert_metric(name + '.count', value=value, metric_type=aggregator.MONOTONIC_COUNT) def test_check_psutil_no_collect_connection_state(aggregator): diff --git a/postgres/changelog.d/18754.added b/postgres/changelog.d/18754.added new file mode 100644 index 0000000000000..30a7029f7f2ae --- /dev/null +++ b/postgres/changelog.d/18754.added @@ -0,0 +1 @@ +Adds additional debug logging to help with schema collection investigations diff --git a/postgres/changelog.d/18758.added b/postgres/changelog.d/18758.added new file mode 100644 index 0000000000000..426f18362cb36 --- /dev/null +++ b/postgres/changelog.d/18758.added @@ -0,0 +1 @@ +Added Postgres cross-org telemetry metrics. diff --git a/postgres/changelog.d/18762.fixed b/postgres/changelog.d/18762.fixed new file mode 100644 index 0000000000000..010efda46ccaf --- /dev/null +++ b/postgres/changelog.d/18762.fixed @@ -0,0 +1 @@ +Parse each row of pg_stat_activity separately inside a try/catch to avoid crashing on bad UTF8 data. diff --git a/postgres/datadog_checks/postgres/metadata.py b/postgres/datadog_checks/postgres/metadata.py index 3c870e86a435d..8c859f8d4b01d 100644 --- a/postgres/datadog_checks/postgres/metadata.py +++ b/postgres/datadog_checks/postgres/metadata.py @@ -297,6 +297,8 @@ def report_postgres_metadata(self): # Tuned from experiments on staging, we may want to make this dynamic based on schema size in the future chunk_size = 50 + total_tables = 0 + start_time = time.time() for database in schema_metadata: dbname = database["name"] @@ -310,6 +312,14 @@ def report_postgres_metadata(self): continue tables = self._query_tables_for_schema(cursor, schema["id"], dbname) + self._log.debug( + "Tables found for schema '{schema}' in database '{database}':" + "{tables}".format( + schema=database["schemas"], + database=dbname, + tables=[table["name"] for table in tables], + ) + ) table_chunks = list(get_list_chunks(tables, chunk_size)) buffer_column_count = 0 @@ -324,11 +334,31 @@ def report_postgres_metadata(self): if buffer_column_count >= 100_000: self._flush_schema(base_event, database, schema, tables_buffer) + total_tables += len(tables_buffer) tables_buffer = [] buffer_column_count = 0 if len(tables_buffer) > 0: self._flush_schema(base_event, database, schema, tables_buffer) + total_tables += len(tables_buffer) + elapsed_ms = (time.time() - start_time) * 1000 + self._check.histogram( + "dd.postgres.schema.time", + elapsed_ms, + tags=self._check.tags, + hostname=self._check.resolved_hostname, + raw=True, + ) + self._check.gauge( + "dd.postgres.schema.tables_count", + total_tables, + tags=self._check.tags, + hostname=self._check.resolved_hostname, + raw=True, + ) + datadog_agent.emit_agent_telemetry("postgres", "schema_tables_elapsed_ms", elapsed_ms, "gauge") + datadog_agent.emit_agent_telemetry("postgres", "schema_tables_count", total_tables, "gauge") + self._is_schemas_collection_in_progress = False def _should_collect_metadata(self, name, metadata_type): @@ -415,6 +445,8 @@ def _query_schema_information(self, cursor: psycopg2.extensions.cursor, dbname: schemas = [] for row in rows: schemas.append({"id": str(row["id"]), "name": row["name"], "owner": row["owner"]}) + + self._log.debug("Schemas found for database '{database}': [{schemas}]".format(database=dbname, schemas=rows)) return schemas def _get_table_info(self, cursor, dbname, schema_id): @@ -437,6 +469,13 @@ def _get_table_info(self, cursor, dbname, schema_id): if len(table_info) <= limit: return table_info + self._log.debug( + "{table_count} tables found but max_tables is set to {max_tables}." + "{missing_count} tables will be missing from this collection".format( + table_count=len(table_info), max_tables=limit, missing_count=len(table_info) - limit + ) + ) + if not self._config.relations: self._check.log.warning( "Number of tables exceeds limit of %d set by max_tables but " diff --git a/postgres/datadog_checks/postgres/postgres.py b/postgres/datadog_checks/postgres/postgres.py index 03d77018b93ee..65aeb4d86fcda 100644 --- a/postgres/datadog_checks/postgres/postgres.py +++ b/postgres/datadog_checks/postgres/postgres.py @@ -646,6 +646,8 @@ def _collect_metric_autodiscovery(self, instance_tags, scopes, scope_type): hostname=self.resolved_hostname, raw=True, ) + telemetry_metric = scope_type.replace("_", "", 1) # remove the first underscore to match telemetry convention + datadog_agent.emit_agent_telemetry("postgres", f"{telemetry_metric}_ms", elapsed_ms, "histogram") if elapsed_ms > self._config.min_collection_interval * 1000: self.record_warning( DatabaseConfigurationError.autodiscovered_metrics_exceeds_collection_interval, diff --git a/postgres/datadog_checks/postgres/statement_samples.py b/postgres/datadog_checks/postgres/statement_samples.py index 0a93cad3b5616..3ec1f07fd64d0 100644 --- a/postgres/datadog_checks/postgres/statement_samples.py +++ b/postgres/datadog_checks/postgres/statement_samples.py @@ -268,7 +268,17 @@ def _get_new_pg_stat_activity(self, available_activity_columns): with conn.cursor(cursor_factory=CommenterDictCursor) as cursor: self._log.debug("Running query [%s] %s", query, params) cursor.execute(query, params) - rows = cursor.fetchall() + rows = [] + while True: + try: + row = cursor.fetchone() + if row is None: + break + rows.append(row) + except UnicodeDecodeError: + self._log.debug("Invalid unicode in row from pg_stat_activity") + except: + self._log.warning("Unknown error fetching row from pg_stat_activity") self._report_check_hist_metrics(start_time, len(rows), "get_new_pg_stat_activity") self._log.debug("Loaded %s rows from %s", len(rows), self._config.pg_stat_activity_view) @@ -424,6 +434,18 @@ def _report_check_hist_metrics(self, start_time, row_len, method_name): hostname=self._check.resolved_hostname, raw=True, ) + datadog_agent.emit_agent_telemetry( + "postgres", + f"{method_name}_ms", + (time.time() - start_time) * 1000, + "histogram", + ) + datadog_agent.emit_agent_telemetry( + "postgres", + f"{method_name}_count", + row_len, + "histogram", + ) def run_job(self): # do not emit any dd.internal metrics for DBM specific check code @@ -468,6 +490,13 @@ def _collect_statement_samples(self): tags=self.tags, raw=True, ) + datadog_agent.emit_agent_telemetry( + "postgres", + "collect_activity_snapshot_ms", + (time.time() - start_time) * 1000, + "histogram", + ) + elapsed_ms = (time.time() - start_time) * 1000 self._check.histogram( "dd.postgres.collect_statement_samples.time", @@ -504,6 +533,18 @@ def _collect_statement_samples(self): hostname=self._check.resolved_hostname, raw=True, ) + datadog_agent.emit_agent_telemetry( + "postgres", + "collect_statement_samples_ms", + elapsed_ms, + "histogram", + ) + datadog_agent.emit_agent_telemetry( + "postgres", + "collect_statement_samples_count", + submitted_count, + "gauge", + ) @staticmethod def _to_active_session(row, track_activity_query_size): diff --git a/requirements-agent-release.txt b/requirements-agent-release.txt index d8574097f1e6d..85052975e4984 100644 --- a/requirements-agent-release.txt +++ b/requirements-agent-release.txt @@ -9,6 +9,7 @@ datadog-airflow==6.0.0 datadog-amazon-msk==5.0.0 datadog-ambari==5.0.0; sys_platform != 'win32' datadog-apache==5.0.0 +datadog-appgate-sdp==1.0.0; sys_platform != 'win32' datadog-arangodb==3.0.0 datadog-argo-rollouts==2.0.0 datadog-argo-workflows==2.0.0 @@ -100,9 +101,13 @@ datadog-kube-dns==5.0.0 datadog-kube-metrics-server==4.0.0 datadog-kube-proxy==7.0.0 datadog-kube-scheduler==5.0.0 +datadog-kubeflow==1.0.0 datadog-kubelet==8.0.0 datadog-kubernetes-cluster-autoscaler==2.0.0 datadog-kubernetes-state==9.0.0 +datadog-kubevirt-api==1.0.0 +datadog-kubevirt-controller==1.0.0 +datadog-kubevirt-handler==1.0.0 datadog-kyototycoon==3.0.0 datadog-kyverno==2.0.0 datadog-lighttpd==4.0.0 @@ -118,7 +123,7 @@ datadog-mesos-slave==4.0.0; sys_platform != 'win32' datadog-mongo==7.0.0 datadog-mysql==13.0.0 datadog-nagios==2.0.0 -datadog-network==4.0.0 +datadog-network==4.1.0 datadog-nfsstat==2.0.0; sys_platform == 'linux2' datadog-nginx-ingress-controller==3.0.0 datadog-nginx==7.0.0 diff --git a/sqlserver/assets/configuration/spec.yaml b/sqlserver/assets/configuration/spec.yaml index 46ae98846412e..83817e6c1f253 100644 --- a/sqlserver/assets/configuration/spec.yaml +++ b/sqlserver/assets/configuration/spec.yaml @@ -776,7 +776,6 @@ files: example: false type: boolean - name: deadlocks_collection - hidden: True description: | Configure the collection of deadlock data. The feature is supported for odbc connector only. options: diff --git a/sqlserver/changelog.d/18760.fixed b/sqlserver/changelog.d/18760.fixed new file mode 100644 index 0000000000000..a295600258290 --- /dev/null +++ b/sqlserver/changelog.d/18760.fixed @@ -0,0 +1 @@ +Updated SQL Server Agent job query for completed jobs, significantly reducing query times for large job history logs \ No newline at end of file diff --git a/sqlserver/datadog_checks/sqlserver/agent_history.py b/sqlserver/datadog_checks/sqlserver/agent_history.py index f78d126fdabca..7492dac5ae8ad 100644 --- a/sqlserver/datadog_checks/sqlserver/agent_history.py +++ b/sqlserver/datadog_checks/sqlserver/agent_history.py @@ -18,37 +18,26 @@ DEFAULT_ROW_LIMIT = 10000 AGENT_HISTORY_QUERY = """\ -WITH HISTORY_ENTRIES AS ( +WITH BASE AS ( SELECT {history_row_limit_filter} j.name AS job_name, - CAST(sjh1.job_id AS char(36)) AS job_id, - sjh1.step_name, - sjh1.step_id, - sjh1.instance_id AS step_instance_id, - ( - SELECT MIN(sjh2.instance_id) - FROM msdb.dbo.sysjobhistory AS sjh2 - WHERE sjh2.job_id = sjh1.job_id - AND sjh2.step_id = 0 - AND sjh2.instance_id >= sjh1.instance_id - ) AS completion_instance_id, - ( - SELECT DATEDIFF(SECOND, '19700101', - DATEADD(HOUR, sjh1.run_time / 10000, - DATEADD(MINUTE, (sjh1.run_time / 100) % 100, - DATEADD(SECOND, sjh1.run_time % 100, - CAST(CAST(sjh1.run_date AS CHAR(8)) AS DATETIME) - ) + CAST(sjh.job_id AS CHAR(36)) AS job_id, + sjh.step_name, + sjh.step_id, + sjh.instance_id AS step_instance_id, + DATEDIFF(SECOND, '19700101', + DATEADD(HOUR, sjh.run_time / 10000, + DATEADD(MINUTE, (sjh.run_time / 100) % 100, + DATEADD(SECOND, sjh.run_time % 100, + CAST(CAST(sjh.run_date AS CHAR(8)) AS DATETIME) ) ) - ) - DATEPART(tzoffset, SYSDATETIMEOFFSET()) * 60 - ) AS run_epoch_time, - ( - (sjh1.run_duration / 10000) * 3600 - + ((sjh1.run_duration % 10000) / 100) * 60 - + (sjh1.run_duration % 100) - ) AS run_duration_seconds, - CASE sjh1.run_status + ) + ) - DATEPART(TZOFFSET, SYSDATETIMEOFFSET()) * 60 AS run_epoch_time, + (sjh.run_duration / 10000) * 3600 + + ((sjh.run_duration % 10000) / 100) * 60 + + (sjh.run_duration % 100) AS run_duration_seconds, + CASE sjh.run_status WHEN 0 THEN 'Failed' WHEN 1 THEN 'Succeeded' WHEN 2 THEN 'Retry' @@ -56,32 +45,55 @@ WHEN 4 THEN 'In Progress' ELSE 'Unknown' END AS step_run_status, - sjh1.message - FROM - msdb.dbo.sysjobhistory AS sjh1 - INNER JOIN msdb.dbo.sysjobs AS j - ON j.job_id = sjh1.job_id - ORDER BY completion_instance_id DESC -) -SELECT * FROM HISTORY_ENTRIES -WHERE - EXISTS ( - SELECT 1 - FROM msdb.dbo.sysjobhistory AS sjh2 - WHERE sjh2.instance_id = HISTORY_ENTRIES.completion_instance_id - AND (DATEDIFF(SECOND, '19700101', - DATEADD(HOUR, sjh2.run_time / 10000, - DATEADD(MINUTE, (sjh2.run_time / 100) % 100, - DATEADD(SECOND, sjh2.run_time % 100, - CAST(CAST(sjh2.run_date AS CHAR(8)) AS DATETIME) - ) + sjh.message + FROM msdb.dbo.sysjobhistory AS sjh + INNER JOIN msdb.dbo.sysjobs AS j ON j.job_id = sjh.job_id + ORDER BY step_instance_id DESC +), +COMPLETION_CTE AS ( + SELECT + BASE.*, + MIN(CASE WHEN BASE.step_id = 0 THEN BASE.step_instance_id END) OVER ( + PARTITION BY BASE.job_id + ORDER BY BASE.step_instance_id + ROWS BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING + ) AS completion_instance_id + FROM BASE +), +HISTORY_ENTRIES AS ( + SELECT + C.*, + DATEDIFF(SECOND, '19700101', + DATEADD(HOUR, c_sjh.run_time / 10000, + DATEADD(MINUTE, (c_sjh.run_time / 100) % 100, + DATEADD(SECOND, c_sjh.run_time % 100, + CAST(CAST(c_sjh.run_date AS CHAR(8)) AS DATETIME) ) ) - ) - DATEPART(tzoffset, SYSDATETIMEOFFSET()) * 60 - + (sjh2.run_duration / 10000) * 3600 - + ((sjh2.run_duration % 10000) / 100) * 60 - + (sjh2.run_duration % 100)) > 0 + {last_collection_time_filter} - ) + ) + ) - DATEPART(TZOFFSET, SYSDATETIMEOFFSET()) * 60 + + (c_sjh.run_duration / 10000) * 3600 + + ((c_sjh.run_duration % 10000) / 100) * 60 + + (c_sjh.run_duration % 100) AS completion_epoch_time + FROM COMPLETION_CTE AS C + LEFT JOIN msdb.dbo.sysjobhistory AS c_sjh + ON c_sjh.instance_id = C.completion_instance_id + WHERE C.completion_instance_id IS NOT NULL +) +SELECT + job_name, + job_id, + step_name, + step_id, + step_instance_id, + completion_instance_id, + run_epoch_time, + run_duration_seconds, + step_run_status, + message +FROM HISTORY_ENTRIES +WHERE + completion_epoch_time > {last_collection_time_filter}; """ diff --git a/sqlserver/datadog_checks/sqlserver/data/conf.yaml.example b/sqlserver/datadog_checks/sqlserver/data/conf.yaml.example index ee5316c9a61c7..0aa748b5b3841 100644 --- a/sqlserver/datadog_checks/sqlserver/data/conf.yaml.example +++ b/sqlserver/datadog_checks/sqlserver/data/conf.yaml.example @@ -692,6 +692,25 @@ instances: # # propagate_agent_tags: false + ## Configure the collection of deadlock data. The feature is supported for odbc connector only. + # + # deadlocks_collection: + + ## @param enabled - boolean - optional - default: false + ## Enable the collection of deadlock data. Requires `dbm: true`. Disabled by default. + # + # enabled: false + + ## @param collection_interval - number - optional - default: 600 + ## Set the interval for collecting deadlock data, in seconds. Defaults to 600 seconds. + # + # collection_interval: 600 + + ## @param max_deadlocks - number - optional - default: 100 + ## Set the maximum number of deadlocks to retrieve per collection. + # + # max_deadlocks: 100 + ## @param tags - list of strings - optional ## A list of tags to attach to every metric and service check emitted by this instance. ## diff --git a/sqlserver/tests/test_agent_jobs.py b/sqlserver/tests/test_agent_jobs.py index f2b1ff7d83efa..e265e892b03cb 100644 --- a/sqlserver/tests/test_agent_jobs.py +++ b/sqlserver/tests/test_agent_jobs.py @@ -21,37 +21,26 @@ AGENT_HISTORY_QUERY = """\ -WITH HISTORY_ENTRIES AS ( +WITH BASE AS ( SELECT {history_row_limit_filter} j.name AS job_name, - CAST(sjh1.job_id AS char(36)) AS job_id, - sjh1.step_name, - sjh1.step_id, - sjh1.instance_id AS step_instance_id, - ( - SELECT MIN(sjh2.instance_id) - FROM msdb.dbo.sysjobhistory AS sjh2 - WHERE sjh2.job_id = sjh1.job_id - AND sjh2.step_id = 0 - AND sjh2.instance_id >= sjh1.instance_id - ) AS completion_instance_id, - ( - SELECT DATEDIFF(SECOND, '19700101', - DATEADD(HOUR, sjh1.run_time / 10000, - DATEADD(MINUTE, (sjh1.run_time / 100) % 100, - DATEADD(SECOND, sjh1.run_time % 100, - CAST(CAST(sjh1.run_date AS CHAR(8)) AS DATETIME) - ) + CAST(sjh.job_id AS CHAR(36)) AS job_id, + sjh.step_name, + sjh.step_id, + sjh.instance_id AS step_instance_id, + DATEDIFF(SECOND, '19700101', + DATEADD(HOUR, sjh.run_time / 10000, + DATEADD(MINUTE, (sjh.run_time / 100) % 100, + DATEADD(SECOND, sjh.run_time % 100, + CAST(CAST(sjh.run_date AS CHAR(8)) AS DATETIME) ) ) - ) - DATEPART(tzoffset, SYSDATETIMEOFFSET()) * 60 - ) AS run_epoch_time, - ( - (sjh1.run_duration / 10000) * 3600 - + ((sjh1.run_duration % 10000) / 100) * 60 - + (sjh1.run_duration % 100) - ) AS run_duration_seconds, - CASE sjh1.run_status + ) + ) - DATEPART(TZOFFSET, SYSDATETIMEOFFSET()) * 60 AS run_epoch_time, + (sjh.run_duration / 10000) * 3600 + + ((sjh.run_duration % 10000) / 100) * 60 + + (sjh.run_duration % 100) AS run_duration_seconds, + CASE sjh.run_status WHEN 0 THEN 'Failed' WHEN 1 THEN 'Succeeded' WHEN 2 THEN 'Retry' @@ -59,66 +48,79 @@ WHEN 4 THEN 'In Progress' ELSE 'Unknown' END AS step_run_status, - sjh1.message - FROM - msdb.dbo.sysjobhistory AS sjh1 - INNER JOIN msdb.dbo.sysjobs AS j - ON j.job_id = sjh1.job_id - ORDER BY completion_instance_id DESC -) -SELECT * FROM HISTORY_ENTRIES -WHERE - EXISTS ( - SELECT 1 - FROM msdb.dbo.sysjobhistory AS sjh2 - WHERE sjh2.instance_id = HISTORY_ENTRIES.completion_instance_id - AND (DATEDIFF(SECOND, '19700101', - DATEADD(HOUR, sjh2.run_time / 10000, - DATEADD(MINUTE, (sjh2.run_time / 100) % 100, - DATEADD(SECOND, sjh2.run_time % 100, - CAST(CAST(sjh2.run_date AS CHAR(8)) AS DATETIME) - ) + sjh.message + FROM msdb.dbo.sysjobhistory AS sjh + INNER JOIN msdb.dbo.sysjobs AS j ON j.job_id = sjh.job_id + ORDER BY step_instance_id DESC +), +COMPLETION_CTE AS ( + SELECT + BASE.*, + MIN(CASE WHEN BASE.step_id = 0 THEN BASE.step_instance_id END) OVER ( + PARTITION BY BASE.job_id + ORDER BY BASE.step_instance_id + ROWS BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING + ) AS completion_instance_id + FROM BASE +), +HISTORY_ENTRIES AS ( + SELECT + C.*, + DATEDIFF(SECOND, '19700101', + DATEADD(HOUR, c_sjh.run_time / 10000, + DATEADD(MINUTE, (c_sjh.run_time / 100) % 100, + DATEADD(SECOND, c_sjh.run_time % 100, + CAST(CAST(c_sjh.run_date AS CHAR(8)) AS DATETIME) ) ) - ) - DATEPART(tzoffset, SYSDATETIMEOFFSET()) * 60 - + (sjh2.run_duration / 10000) * 3600 - + ((sjh2.run_duration % 10000) / 100) * 60 - + (sjh2.run_duration % 100)) > 0 + {last_collection_time_filter} - ) + ) + ) - DATEPART(TZOFFSET, SYSDATETIMEOFFSET()) * 60 + + (c_sjh.run_duration / 10000) * 3600 + + ((c_sjh.run_duration % 10000) / 100) * 60 + + (c_sjh.run_duration % 100) AS completion_epoch_time + FROM COMPLETION_CTE AS C + LEFT JOIN msdb.dbo.sysjobhistory AS c_sjh + ON c_sjh.instance_id = C.completion_instance_id + WHERE C.completion_instance_id IS NOT NULL +) +SELECT + job_name, + job_id, + step_name, + step_id, + step_instance_id, + completion_instance_id, + run_epoch_time, + run_duration_seconds, + step_run_status, + message +FROM HISTORY_ENTRIES +WHERE + completion_epoch_time > {last_collection_time_filter}; """ + FORMATTED_HISTORY_QUERY = """\ -WITH HISTORY_ENTRIES AS ( +WITH BASE AS ( SELECT TOP 10000 j.name AS job_name, - CAST(sjh1.job_id AS char(36)) AS job_id, - sjh1.step_name, - sjh1.step_id, - sjh1.instance_id AS step_instance_id, - ( - SELECT MIN(sjh2.instance_id) - FROM msdb.dbo.sysjobhistory AS sjh2 - WHERE sjh2.job_id = sjh1.job_id - AND sjh2.step_id = 0 - AND sjh2.instance_id >= sjh1.instance_id - ) AS completion_instance_id, - ( - SELECT DATEDIFF(SECOND, '19700101', - DATEADD(HOUR, sjh1.run_time / 10000, - DATEADD(MINUTE, (sjh1.run_time / 100) % 100, - DATEADD(SECOND, sjh1.run_time % 100, - CAST(CAST(sjh1.run_date AS CHAR(8)) AS DATETIME) - ) + CAST(sjh.job_id AS CHAR(36)) AS job_id, + sjh.step_name, + sjh.step_id, + sjh.instance_id AS step_instance_id, + DATEDIFF(SECOND, '19700101', + DATEADD(HOUR, sjh.run_time / 10000, + DATEADD(MINUTE, (sjh.run_time / 100) % 100, + DATEADD(SECOND, sjh.run_time % 100, + CAST(CAST(sjh.run_date AS CHAR(8)) AS DATETIME) ) ) - ) - DATEPART(tzoffset, SYSDATETIMEOFFSET()) * 60 - ) AS run_epoch_time, - ( - (sjh1.run_duration / 10000) * 3600 - + ((sjh1.run_duration % 10000) / 100) * 60 - + (sjh1.run_duration % 100) - ) AS run_duration_seconds, - CASE sjh1.run_status + ) + ) - DATEPART(TZOFFSET, SYSDATETIMEOFFSET()) * 60 AS run_epoch_time, + (sjh.run_duration / 10000) * 3600 + + ((sjh.run_duration % 10000) / 100) * 60 + + (sjh.run_duration % 100) AS run_duration_seconds, + CASE sjh.run_status WHEN 0 THEN 'Failed' WHEN 1 THEN 'Succeeded' WHEN 2 THEN 'Retry' @@ -126,32 +128,55 @@ WHEN 4 THEN 'In Progress' ELSE 'Unknown' END AS step_run_status, - sjh1.message - FROM - msdb.dbo.sysjobhistory AS sjh1 - INNER JOIN msdb.dbo.sysjobs AS j - ON j.job_id = sjh1.job_id - ORDER BY completion_instance_id DESC -) -SELECT * FROM HISTORY_ENTRIES -WHERE - EXISTS ( - SELECT 1 - FROM msdb.dbo.sysjobhistory AS sjh2 - WHERE sjh2.instance_id = HISTORY_ENTRIES.completion_instance_id - AND (DATEDIFF(SECOND, '19700101', - DATEADD(HOUR, sjh2.run_time / 10000, - DATEADD(MINUTE, (sjh2.run_time / 100) % 100, - DATEADD(SECOND, sjh2.run_time % 100, - CAST(CAST(sjh2.run_date AS CHAR(8)) AS DATETIME) - ) + sjh.message + FROM msdb.dbo.sysjobhistory AS sjh + INNER JOIN msdb.dbo.sysjobs AS j ON j.job_id = sjh.job_id + ORDER BY step_instance_id DESC +), +COMPLETION_CTE AS ( + SELECT + BASE.*, + MIN(CASE WHEN BASE.step_id = 0 THEN BASE.step_instance_id END) OVER ( + PARTITION BY BASE.job_id + ORDER BY BASE.step_instance_id + ROWS BETWEEN CURRENT ROW AND UNBOUNDED FOLLOWING + ) AS completion_instance_id + FROM BASE +), +HISTORY_ENTRIES AS ( + SELECT + C.*, + DATEDIFF(SECOND, '19700101', + DATEADD(HOUR, c_sjh.run_time / 10000, + DATEADD(MINUTE, (c_sjh.run_time / 100) % 100, + DATEADD(SECOND, c_sjh.run_time % 100, + CAST(CAST(c_sjh.run_date AS CHAR(8)) AS DATETIME) ) ) - ) - DATEPART(tzoffset, SYSDATETIMEOFFSET()) * 60 - + (sjh2.run_duration / 10000) * 3600 - + ((sjh2.run_duration % 10000) / 100) * 60 - + (sjh2.run_duration % 100)) > 0 + 10000 - ) + ) + ) - DATEPART(TZOFFSET, SYSDATETIMEOFFSET()) * 60 + + (c_sjh.run_duration / 10000) * 3600 + + ((c_sjh.run_duration % 10000) / 100) * 60 + + (c_sjh.run_duration % 100) AS completion_epoch_time + FROM COMPLETION_CTE AS C + LEFT JOIN msdb.dbo.sysjobhistory AS c_sjh + ON c_sjh.instance_id = C.completion_instance_id + WHERE C.completion_instance_id IS NOT NULL +) +SELECT + job_name, + job_id, + step_name, + step_id, + step_instance_id, + completion_instance_id, + run_epoch_time, + run_duration_seconds, + step_run_status, + message +FROM HISTORY_ENTRIES +WHERE + completion_epoch_time > 10000; """ AGENT_ACTIVITY_DURATION_QUERY = """\