diff --git a/postgres/datadog_checks/postgres/config_models/defaults.py b/postgres/datadog_checks/postgres/config_models/defaults.py index 1d1775ed71893..7ff348b983164 100644 --- a/postgres/datadog_checks/postgres/config_models/defaults.py +++ b/postgres/datadog_checks/postgres/config_models/defaults.py @@ -7,200 +7,130 @@ # ddev -x validate config -s # ddev -x validate models -s -from datadog_checks.base.utils.models.fields import get_default_field_value - -def shared_service(field, value): - return get_default_field_value(field, value) - - -def instance_activity_metrics_excluded_aggregations(field, value): +def instance_activity_metrics_excluded_aggregations(): return [] -def instance_application_name(field, value): +def instance_application_name(): return 'datadog-agent' -def instance_aws(field, value): - return get_default_field_value(field, value) - - -def instance_azure(field, value): - return get_default_field_value(field, value) - - -def instance_collect_activity_metrics(field, value): +def instance_collect_activity_metrics(): return False -def instance_collect_bloat_metrics(field, value): +def instance_collect_bloat_metrics(): return False -def instance_collect_count_metrics(field, value): +def instance_collect_count_metrics(): return True -def instance_collect_database_size_metrics(field, value): +def instance_collect_database_size_metrics(): return True -def instance_collect_default_database(field, value): +def instance_collect_default_database(): return True -def instance_collect_function_metrics(field, value): +def instance_collect_function_metrics(): return False -def instance_collect_settings(field, value): - return get_default_field_value(field, value) - - -def instance_collect_wal_metrics(field, value): +def instance_collect_wal_metrics(): return False -def instance_custom_queries(field, value): - return get_default_field_value(field, value) - - -def instance_data_directory(field, value): +def instance_data_directory(): return '/usr/local/pgsql/data' -def instance_database_autodiscovery(field, value): - return get_default_field_value(field, value) - - -def instance_dbm(field, value): +def instance_dbm(): return False -def instance_dbname(field, value): +def instance_dbname(): return 'postgres' -def instance_dbstrict(field, value): +def instance_dbstrict(): return False -def instance_disable_generic_tags(field, value): +def instance_disable_generic_tags(): return False -def instance_empty_default_hostname(field, value): +def instance_empty_default_hostname(): return False -def instance_gcp(field, value): - return get_default_field_value(field, value) - - -def instance_idle_connection_timeout(field, value): +def instance_idle_connection_timeout(): return 60000 -def instance_ignore_databases(field, value): +def instance_ignore_databases(): return ['template%', 'rdsadmin', 'azure_maintenance'] -def instance_log_unobfuscated_plans(field, value): +def instance_log_unobfuscated_plans(): return False -def instance_log_unobfuscated_queries(field, value): +def instance_log_unobfuscated_queries(): return False -def instance_max_connections(field, value): +def instance_max_connections(): return 30 -def instance_max_relations(field, value): +def instance_max_relations(): return 300 -def instance_metric_patterns(field, value): - return get_default_field_value(field, value) - - -def instance_min_collection_interval(field, value): +def instance_min_collection_interval(): return 15 -def instance_obfuscator_options(field, value): - return get_default_field_value(field, value) - - -def instance_password(field, value): - return get_default_field_value(field, value) - - -def instance_pg_stat_statements_view(field, value): +def instance_pg_stat_statements_view(): return 'show_pg_stat_statements()' -def instance_port(field, value): +def instance_port(): return 5432 -def instance_query_activity(field, value): - return get_default_field_value(field, value) - - -def instance_query_metrics(field, value): - return get_default_field_value(field, value) - - -def instance_query_samples(field, value): - return get_default_field_value(field, value) - - -def instance_query_timeout(field, value): +def instance_query_timeout(): return 5000 -def instance_relations(field, value): - return get_default_field_value(field, value) - - -def instance_reported_hostname(field, value): - return get_default_field_value(field, value) - - -def instance_service(field, value): - return get_default_field_value(field, value) - - -def instance_ssl(field, value): +def instance_ssl(): return False -def instance_ssl_cert(field, value): +def instance_ssl_cert(): return False -def instance_ssl_key(field, value): +def instance_ssl_key(): return False -def instance_ssl_password(field, value): +def instance_ssl_password(): return False -def instance_ssl_root_cert(field, value): +def instance_ssl_root_cert(): return False -def instance_table_count_limit(field, value): +def instance_table_count_limit(): return 200 -def instance_tag_replication_role(field, value): +def instance_tag_replication_role(): return False - - -def instance_tags(field, value): - return get_default_field_value(field, value) diff --git a/postgres/datadog_checks/postgres/config_models/instance.py b/postgres/datadog_checks/postgres/config_models/instance.py index ad4686488ebd2..1754f691c3cab 100644 --- a/postgres/datadog_checks/postgres/config_models/instance.py +++ b/postgres/datadog_checks/postgres/config_models/instance.py @@ -9,9 +9,10 @@ from __future__ import annotations -from typing import Any, Mapping, Optional, Sequence, Union +from types import MappingProxyType +from typing import Any, Optional, Union -from pydantic import BaseModel, root_validator, validator +from pydantic import BaseModel, ConfigDict, field_validator, model_validator from datadog_checks.base.utils.functions import identity from datadog_checks.base.utils.models import validation @@ -20,185 +21,195 @@ class Aws(BaseModel): - class Config: - allow_mutation = False - - instance_endpoint: Optional[str] - region: Optional[str] + model_config = ConfigDict( + arbitrary_types_allowed=True, + frozen=True, + ) + instance_endpoint: Optional[str] = None + region: Optional[str] = None class Azure(BaseModel): - class Config: - allow_mutation = False - - deployment_type: Optional[str] - fully_qualified_domain_name: Optional[str] + model_config = ConfigDict( + arbitrary_types_allowed=True, + frozen=True, + ) + deployment_type: Optional[str] = None + fully_qualified_domain_name: Optional[str] = None class CollectSettings(BaseModel): - class Config: - allow_mutation = False - - collection_interval: Optional[float] - enabled: Optional[bool] + model_config = ConfigDict( + arbitrary_types_allowed=True, + frozen=True, + ) + collection_interval: Optional[float] = None + enabled: Optional[bool] = None class DatabaseAutodiscovery(BaseModel): - class Config: - allow_mutation = False - - enabled: Optional[bool] - exclude: Optional[Sequence[str]] - include: Optional[Sequence[str]] - max_databases: Optional[int] - refresh: Optional[int] + model_config = ConfigDict( + arbitrary_types_allowed=True, + frozen=True, + ) + enabled: Optional[bool] = None + exclude: Optional[tuple[str, ...]] = None + include: Optional[tuple[str, ...]] = None + max_databases: Optional[int] = None + refresh: Optional[int] = None class Gcp(BaseModel): - class Config: - allow_mutation = False - - instance_id: Optional[str] - project_id: Optional[str] + model_config = ConfigDict( + arbitrary_types_allowed=True, + frozen=True, + ) + instance_id: Optional[str] = None + project_id: Optional[str] = None class MetricPatterns(BaseModel): - class Config: - allow_mutation = False - - exclude: Optional[Sequence[str]] - include: Optional[Sequence[str]] + model_config = ConfigDict( + arbitrary_types_allowed=True, + frozen=True, + ) + exclude: Optional[tuple[str, ...]] = None + include: Optional[tuple[str, ...]] = None class ObfuscatorOptions(BaseModel): - class Config: - allow_mutation = False - - collect_commands: Optional[bool] - collect_comments: Optional[bool] - collect_metadata: Optional[bool] - collect_tables: Optional[bool] - keep_dollar_quoted_func: Optional[bool] - keep_sql_alias: Optional[bool] - replace_digits: Optional[bool] + model_config = ConfigDict( + arbitrary_types_allowed=True, + frozen=True, + ) + collect_commands: Optional[bool] = None + collect_comments: Optional[bool] = None + collect_metadata: Optional[bool] = None + collect_tables: Optional[bool] = None + keep_dollar_quoted_func: Optional[bool] = None + keep_sql_alias: Optional[bool] = None + replace_digits: Optional[bool] = None class QueryActivity(BaseModel): - class Config: - allow_mutation = False - - collection_interval: Optional[float] - enabled: Optional[bool] - payload_row_limit: Optional[float] + model_config = ConfigDict( + arbitrary_types_allowed=True, + frozen=True, + ) + collection_interval: Optional[float] = None + enabled: Optional[bool] = None + payload_row_limit: Optional[float] = None class QueryMetrics(BaseModel): - class Config: - allow_mutation = False - - collection_interval: Optional[float] - enabled: Optional[bool] - pg_stat_statements_max_warning_threshold: Optional[float] + model_config = ConfigDict( + arbitrary_types_allowed=True, + frozen=True, + ) + collection_interval: Optional[float] = None + enabled: Optional[bool] = None + pg_stat_statements_max_warning_threshold: Optional[float] = None class QuerySamples(BaseModel): - class Config: - allow_mutation = False - - collection_interval: Optional[float] - enabled: Optional[bool] - explain_function: Optional[str] - explain_parameterized_queries: Optional[bool] - explained_queries_cache_maxsize: Optional[int] - explained_queries_per_hour_per_query: Optional[int] - samples_per_hour_per_query: Optional[int] - seen_samples_cache_maxsize: Optional[int] + model_config = ConfigDict( + arbitrary_types_allowed=True, + frozen=True, + ) + collection_interval: Optional[float] = None + enabled: Optional[bool] = None + explain_function: Optional[str] = None + explain_parameterized_queries: Optional[bool] = None + explained_queries_cache_maxsize: Optional[int] = None + explained_queries_per_hour_per_query: Optional[int] = None + samples_per_hour_per_query: Optional[int] = None + seen_samples_cache_maxsize: Optional[int] = None class Relation(BaseModel): - class Config: - allow_mutation = False - - relation_name: Optional[str] - relation_regex: Optional[str] - relation_schema: Optional[str] - relkind: Optional[Sequence[str]] - schemas: Optional[Sequence[str]] + model_config = ConfigDict( + arbitrary_types_allowed=True, + frozen=True, + ) + relation_name: Optional[str] = None + relation_regex: Optional[str] = None + relation_schema: Optional[str] = None + relkind: Optional[tuple[str, ...]] = None + schemas: Optional[tuple[str, ...]] = None class InstanceConfig(BaseModel): - class Config: - allow_mutation = False - - activity_metrics_excluded_aggregations: Optional[Sequence[str]] - application_name: Optional[str] - aws: Optional[Aws] - azure: Optional[Azure] - collect_activity_metrics: Optional[bool] - collect_bloat_metrics: Optional[bool] - collect_count_metrics: Optional[bool] - collect_database_size_metrics: Optional[bool] - collect_default_database: Optional[bool] - collect_function_metrics: Optional[bool] - collect_settings: Optional[CollectSettings] - collect_wal_metrics: Optional[bool] - custom_queries: Optional[Sequence[Mapping[str, Any]]] - data_directory: Optional[str] - database_autodiscovery: Optional[DatabaseAutodiscovery] - dbm: Optional[bool] - dbname: Optional[str] - dbstrict: Optional[bool] - disable_generic_tags: Optional[bool] - empty_default_hostname: Optional[bool] - gcp: Optional[Gcp] + model_config = ConfigDict( + validate_default=True, + arbitrary_types_allowed=True, + frozen=True, + ) + activity_metrics_excluded_aggregations: Optional[tuple[str, ...]] = None + application_name: Optional[str] = None + aws: Optional[Aws] = None + azure: Optional[Azure] = None + collect_activity_metrics: Optional[bool] = None + collect_bloat_metrics: Optional[bool] = None + collect_count_metrics: Optional[bool] = None + collect_database_size_metrics: Optional[bool] = None + collect_default_database: Optional[bool] = None + collect_function_metrics: Optional[bool] = None + collect_settings: Optional[CollectSettings] = None + collect_wal_metrics: Optional[bool] = None + custom_queries: Optional[tuple[MappingProxyType[str, Any], ...]] = None + data_directory: Optional[str] = None + database_autodiscovery: Optional[DatabaseAutodiscovery] = None + dbm: Optional[bool] = None + dbname: Optional[str] = None + dbstrict: Optional[bool] = None + disable_generic_tags: Optional[bool] = None + empty_default_hostname: Optional[bool] = None + gcp: Optional[Gcp] = None host: str - idle_connection_timeout: Optional[int] - ignore_databases: Optional[Sequence[str]] - log_unobfuscated_plans: Optional[bool] - log_unobfuscated_queries: Optional[bool] - max_connections: Optional[int] - max_relations: Optional[int] - metric_patterns: Optional[MetricPatterns] - min_collection_interval: Optional[float] - obfuscator_options: Optional[ObfuscatorOptions] - password: Optional[str] - pg_stat_statements_view: Optional[str] - port: Optional[int] - query_activity: Optional[QueryActivity] - query_metrics: Optional[QueryMetrics] - query_samples: Optional[QuerySamples] - query_timeout: Optional[int] - relations: Optional[Sequence[Union[str, Relation]]] - reported_hostname: Optional[str] - service: Optional[str] - ssl: Optional[str] - ssl_cert: Optional[str] - ssl_key: Optional[str] - ssl_password: Optional[str] - ssl_root_cert: Optional[str] - table_count_limit: Optional[int] - tag_replication_role: Optional[bool] - tags: Optional[Sequence[str]] + idle_connection_timeout: Optional[int] = None + ignore_databases: Optional[tuple[str, ...]] = None + log_unobfuscated_plans: Optional[bool] = None + log_unobfuscated_queries: Optional[bool] = None + max_connections: Optional[int] = None + max_relations: Optional[int] = None + metric_patterns: Optional[MetricPatterns] = None + min_collection_interval: Optional[float] = None + obfuscator_options: Optional[ObfuscatorOptions] = None + password: Optional[str] = None + pg_stat_statements_view: Optional[str] = None + port: Optional[int] = None + query_activity: Optional[QueryActivity] = None + query_metrics: Optional[QueryMetrics] = None + query_samples: Optional[QuerySamples] = None + query_timeout: Optional[int] = None + relations: Optional[tuple[Union[str, Relation], ...]] = None + reported_hostname: Optional[str] = None + service: Optional[str] = None + ssl: Optional[str] = None + ssl_cert: Optional[str] = None + ssl_key: Optional[str] = None + ssl_password: Optional[str] = None + ssl_root_cert: Optional[str] = None + table_count_limit: Optional[int] = None + tag_replication_role: Optional[bool] = None + tags: Optional[tuple[str, ...]] = None username: str - @root_validator(pre=True) + @model_validator(mode='before') def _initial_validation(cls, values): return validation.core.initialize_config(getattr(validators, 'initialize_instance', identity)(values)) - @validator('*', pre=True, always=True) - def _ensure_defaults(cls, v, field): - if v is not None or field.required: - return v - - return getattr(defaults, f'instance_{field.name}')(field, v) - - @validator('*') - def _run_validations(cls, v, field): - if not v: - return v + @field_validator('*', mode='before') + def _validate(cls, value, info): + field = cls.model_fields[info.field_name] + field_name = field.alias or info.field_name + if field_name in info.context['configured_fields']: + value = getattr(validators, f'instance_{info.field_name}', identity)(value, field=field) + else: + value = getattr(defaults, f'instance_{info.field_name}', lambda: value)() - return getattr(validators, f'instance_{field.name}', identity)(v, field=field) + return validation.utils.make_immutable(value) - @root_validator(pre=False) - def _final_validation(cls, values): - return validation.core.finalize_config(getattr(validators, 'finalize_instance', identity)(values)) + @model_validator(mode='after') + def _final_validation(cls, model): + return validation.core.check_model(getattr(validators, 'check_instance', identity)(model)) \ No newline at end of file diff --git a/postgres/datadog_checks/postgres/config_models/shared.py b/postgres/datadog_checks/postgres/config_models/shared.py index a6de26a779888..bf0fc05f5001b 100644 --- a/postgres/datadog_checks/postgres/config_models/shared.py +++ b/postgres/datadog_checks/postgres/config_models/shared.py @@ -11,38 +11,35 @@ from typing import Optional -from pydantic import BaseModel, root_validator, validator +from pydantic import BaseModel, ConfigDict, field_validator, model_validator from datadog_checks.base.utils.functions import identity from datadog_checks.base.utils.models import validation -from . import defaults, validators +from . import validators class SharedConfig(BaseModel): - class Config: - allow_mutation = False - - service: Optional[str] - - @root_validator(pre=True) + model_config = ConfigDict( + validate_default=True, + arbitrary_types_allowed=True, + frozen=True, + ) + service: Optional[str] = None + + @model_validator(mode='before') def _initial_validation(cls, values): return validation.core.initialize_config(getattr(validators, 'initialize_shared', identity)(values)) - @validator('*', pre=True, always=True) - def _ensure_defaults(cls, v, field): - if v is not None or field.required: - return v - - return getattr(defaults, f'shared_{field.name}')(field, v) - - @validator('*') - def _run_validations(cls, v, field): - if not v: - return v + @field_validator('*', mode='before') + def _validate(cls, value, info): + field = cls.model_fields[info.field_name] + field_name = field.alias or info.field_name + if field_name in info.context['configured_fields']: + value = getattr(validators, f'shared_{info.field_name}', identity)(value, field=field) - return getattr(validators, f'shared_{field.name}', identity)(v, field=field) + return validation.utils.make_immutable(value) - @root_validator(pre=False) - def _final_validation(cls, values): - return validation.core.finalize_config(getattr(validators, 'finalize_shared', identity)(values)) + @model_validator(mode='after') + def _final_validation(cls, model): + return validation.core.check_model(getattr(validators, 'check_shared', identity)(model)) \ No newline at end of file diff --git a/postgres/datadog_checks/postgres/connections.py b/postgres/datadog_checks/postgres/connections.py index 9ee9c6cdfbbd0..e85ed8f0a2442 100644 --- a/postgres/datadog_checks/postgres/connections.py +++ b/postgres/datadog_checks/postgres/connections.py @@ -9,6 +9,7 @@ from typing import Callable, Dict import psycopg + from datadog_checks.base import AgentCheck diff --git a/postgres/datadog_checks/postgres/explain_parameterized_queries.py b/postgres/datadog_checks/postgres/explain_parameterized_queries.py index 077c3b690c8c0..454e65e625c35 100644 --- a/postgres/datadog_checks/postgres/explain_parameterized_queries.py +++ b/postgres/datadog_checks/postgres/explain_parameterized_queries.py @@ -4,7 +4,6 @@ import logging -import psycopg from psycopg.rows import dict_row from datadog_checks.base.utils.db.sql import compute_sql_signature diff --git a/postgres/datadog_checks/postgres/postgres.py b/postgres/datadog_checks/postgres/postgres.py index 21f755fb0f025..28fbad4c7ae84 100644 --- a/postgres/datadog_checks/postgres/postgres.py +++ b/postgres/datadog_checks/postgres/postgres.py @@ -1,8 +1,8 @@ # (C) Datadog, Inc. 2019-present # All rights reserved # Licensed under Simplified BSD License (see LICENSE) -import copy import concurrent.futures +import copy import os from time import time @@ -245,12 +245,15 @@ def cancel(self): """ with concurrent.futures.ThreadPoolExecutor() as executor: tasks = [ - executor.submit(thread.cancel) for thread in [ - self.statement_samples, self.statement_metrics, self.metadata_samples - ] + executor.submit(thread.cancel) + for thread in [self.statement_samples, self.statement_metrics, self.metadata_samples] ] - concurrent.futures.wait(tasks) - # close all connections gracefully + + try: + concurrent.futures.wait(tasks, timeout=self._config.min_collection_interval) + except concurrent.futures.TimeoutError: + self.log.warning("timeout reached on cancel, proceeding with unclean shutdown.") + self._close_db_pool() self.check_cancelled = True @@ -863,5 +866,5 @@ def check(self, _): try: # once check finishes on a cancel, shut down main connection gracefully self.db.close() - except Exception as e: + except Exception: self.log.exception("failed to close DB connection for db=%s", self._config.dbname) diff --git a/postgres/datadog_checks/postgres/statement_samples.py b/postgres/datadog_checks/postgres/statement_samples.py index 66cb8f26cb983..01447ff3d3f37 100644 --- a/postgres/datadog_checks/postgres/statement_samples.py +++ b/postgres/datadog_checks/postgres/statement_samples.py @@ -9,8 +9,8 @@ from typing import Dict, Optional, Tuple # noqa: F401 import psycopg -from psycopg.rows import dict_row from cachetools import TTLCache +from psycopg.rows import dict_row from six import PY2 try: diff --git a/postgres/datadog_checks/postgres/statements.py b/postgres/datadog_checks/postgres/statements.py index 2903c7ed4cae8..e7f8e7a72ee86 100644 --- a/postgres/datadog_checks/postgres/statements.py +++ b/postgres/datadog_checks/postgres/statements.py @@ -7,8 +7,8 @@ import time import psycopg -from psycopg.rows import dict_row from cachetools import TTLCache +from psycopg.rows import dict_row from datadog_checks.base import is_affirmative from datadog_checks.base.utils.common import to_native_string diff --git a/postgres/datadog_checks/postgres/util.py b/postgres/datadog_checks/postgres/util.py index 492d646c271e7..76eac029fa6fd 100644 --- a/postgres/datadog_checks/postgres/util.py +++ b/postgres/datadog_checks/postgres/util.py @@ -2,8 +2,6 @@ # All rights reserved # Licensed under Simplified BSD License (see LICENSE) import string -import json -from ipaddress import IPv4Address from enum import Enum from typing import Any, List, Tuple # noqa: F401 diff --git a/postgres/tests/test_connections.py b/postgres/tests/test_connections.py index e82458ed43839..685019944fdd0 100644 --- a/postgres/tests/test_connections.py +++ b/postgres/tests/test_connections.py @@ -8,11 +8,12 @@ import uuid import psycopg -from psycopg.rows import dict_row import pytest +from psycopg.rows import dict_row from datadog_checks.postgres import PostgreSql from datadog_checks.postgres.connections import ConnectionPoolFullError, MultiDatabaseConnectionPool + from .common import HOST, PASSWORD_ADMIN, USER_ADMIN from .utils import WaitGroup @@ -75,7 +76,7 @@ def test_conn_pool_no_leaks_on_close(pg_instance): # Iterate in the test many times to detect flakiness for _ in range(20): - def exec_connection(dbname): + def exec_connection(pool, wg, dbname): db = pool._get_connection_raw(dbname, 10 * 1000) with db.cursor(row_factory=dict_row) as cursor: cursor.execute("select current_database()") @@ -89,7 +90,7 @@ def exec_connection(dbname): pool = MultiDatabaseConnectionPool(check, check._new_connection) wg = WaitGroup() for i in range(conn_count): - thread = threading.Thread(target=exec_connection, args=('dogs_{}'.format(i),)) + thread = threading.Thread(target=exec_connection, args=(pool, wg, 'dogs_{}'.format(i))) threadpool.append(thread) wg.add(1) thread.start() @@ -324,4 +325,4 @@ def get_activity(db_pool, unique_id): " WHERE datname LIKE 'dogs%%' AND application_name = %s", (unique_id,), ) - return cursor.fetchall() \ No newline at end of file + return cursor.fetchall() diff --git a/postgres/tests/test_deadlock.py b/postgres/tests/test_deadlock.py index b66ecb1d299a8..a238464c6021f 100644 --- a/postgres/tests/test_deadlock.py +++ b/postgres/tests/test_deadlock.py @@ -2,14 +2,12 @@ # All rights reserved # Licensed under Simplified BSD License (see LICENSE) -import copy -import select +import threading import time import psycopg -from psycopg import ClientCursor -import threading import pytest +from psycopg import ClientCursor from .common import DB_NAME, HOST, PORT, POSTGRES_VERSION @@ -40,7 +38,7 @@ def test_deadlock(aggregator, dd_run_check, integration_check, pg_instance): def execute_in_thread(q, args): with psycopg.connect( - host=HOST, dbname=DB_NAME, user="bob", password="bob", cursor_factory=ClientCursor + host=HOST, dbname=DB_NAME, user="bob", password="bob", cursor_factory=ClientCursor ) as tconn: with tconn.cursor() as cur: # this will block, and eventually throw when @@ -73,7 +71,9 @@ def execute_in_thread(q, args): {}; {}; commit; -""".format(update_sql, update_sql) +""".format( + update_sql, update_sql + ) # ... now execute the test query in a separate thread lock_task = threading.Thread(target=execute_in_thread, args=(query, args)) lock_task.start() diff --git a/postgres/tests/test_discovery.py b/postgres/tests/test_discovery.py index 37bb5bf593374..a4d4e56e979cb 100644 --- a/postgres/tests/test_discovery.py +++ b/postgres/tests/test_discovery.py @@ -8,8 +8,8 @@ from contextlib import contextmanager import psycopg -from psycopg import sql import pytest +from psycopg import sql from datadog_checks.base import ConfigurationError @@ -130,9 +130,7 @@ def get_postgres_connection(): assert len(databases) == expected_len + 1 finally: # Need to drop the new database to clean up the environment for next tests. - cursor.execute( - sql.SQL("DROP DATABASE {} WITH (FORCE);").format(sql.Identifier(database_to_find)) - ) + cursor.execute(sql.SQL("DROP DATABASE {} WITH (FORCE);").format(sql.Identifier(database_to_find))) @pytest.mark.integration diff --git a/postgres/tests/test_statements.py b/postgres/tests/test_statements.py index fdcdbc330d028..16b1ad777353f 100644 --- a/postgres/tests/test_statements.py +++ b/postgres/tests/test_statements.py @@ -3,16 +3,16 @@ # Licensed under Simplified BSD License (see LICENSE) import datetime import re +import threading import time from collections import Counter from concurrent.futures.thread import ThreadPoolExecutor import mock import psycopg -from psycopg import ClientCursor -import threading import pytest from dateutil import parser +from psycopg import ClientCursor from semver import VersionInfo from six import string_types @@ -27,7 +27,7 @@ from datadog_checks.postgres.statements import PG_STAT_STATEMENTS_METRICS_COLUMNS, PG_STAT_STATEMENTS_TIMING_COLUMNS from .common import DB_NAME, HOST, PORT, PORT_REPLICA2, POSTGRES_VERSION -from .utils import _get_conn, _get_superconn, requires_over_10, run_one_check, WaitGroup +from .utils import WaitGroup, _get_conn, _get_superconn, requires_over_10, run_one_check pytestmark = [pytest.mark.integration, pytest.mark.usefixtures('dd_environment')] @@ -145,12 +145,14 @@ def _run_queries(): run_one_check(check, dbm_instance) if not track_io_timing_enabled: - with mock.patch('datadog_checks.postgres.PostgreSql.get_pg_settings', - return_value={ - 'pg_stat_statements.max': '10000', - 'track_activity_query_size': '1024', - 'track_io_timing': 'off' - }): + with mock.patch( + 'datadog_checks.postgres.PostgreSql.get_pg_settings', + return_value={ + 'pg_stat_statements.max': '10000', + 'track_activity_query_size': '1024', + 'track_io_timing': 'off', + }, + ): _run_queries() run_one_check(check, dbm_instance) _run_queries() @@ -965,6 +967,7 @@ def execute_in_thread(q): with conn.cursor() as cursor: cursor.execute(q) wg.done() + # we are able to see the full query (including the raw parameters) in pg_stat_activity because psycopg uses # the simple query protocol, sending the whole query as a plain string to postgres. # if a client is using the extended query protocol with prepare then the query would appear as diff --git a/postgres/tests/utils.py b/postgres/tests/utils.py index 66d3ae7e0d22c..b36cbf62369be 100644 --- a/postgres/tests/utils.py +++ b/postgres/tests/utils.py @@ -1,11 +1,11 @@ # (C) Datadog, Inc. 2019-present # All rights reserved # Licensed under a 3-clause BSD style license (see LICENSE) +import threading import time import psycopg import pytest -import threading from .common import POSTGRES_VERSION