Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Wrap schema collection attempt in try #18788

Open
wants to merge 3 commits into
base: master
Choose a base branch
from
+78 −70
Open

Wrap schema collection attempt in try #18788

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
25026dd
Try/catch
sethsamuel Oct 7, 2024
b5a6e57
Finally
sethsamuel Oct 7, 2024
ea51111
Changelog
sethsamuel Oct 7, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions postgres/changelog.d/18788.fixed
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
Wrap schema collection attempt in try/except block to catch collection exceptions
147 changes: 77 additions & 70 deletions postgres/datadog_checks/postgres/metadata.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -282,84 +282,91 @@ def report_postgres_metadata(self):
and elapsed_s_schemas >= self.schemas_collection_interval
):
self._is_schemas_collection_in_progress = True
schema_metadata = self._collect_schema_info()
# We emit an event for each batch of tables to reduce total data in memory and keep event size reasonable
base_event = {
"host": self._check.resolved_hostname,
"agent_version": datadog_agent.get_version(),
"dbms": "postgres",
"kind": "pg_databases",
"collection_interval": self.schemas_collection_interval,
"dbms_version": self._payload_pg_version(),
"tags": self._tags_no_db,
"cloud_metadata": self._config.cloud_metadata,
}

# Tuned from experiments on staging, we may want to make this dynamic based on schema size in the future
chunk_size = 50
total_tables = 0
start_time = time.time()

for database in schema_metadata:
dbname = database["name"]
if not self._should_collect_metadata(dbname, "database"):
continue

with self.db_pool.get_connection(dbname, self._config.idle_connection_timeout) as conn:
with conn.cursor(cursor_factory=psycopg2.extras.DictCursor) as cursor:
for schema in database["schemas"]:
if not self._should_collect_metadata(schema["name"], "schema"):
continue

tables = self._query_tables_for_schema(cursor, schema["id"], dbname)
self._log.debug(
"Tables found for schema '{schema}' in database '{database}':"
"{tables}".format(
schema=database["schemas"],
database=dbname,
tables=[table["name"] for table in tables],
status = "success"
try:
schema_metadata = self._collect_schema_info()
# We emit an event for each batch of tables to reduce total data in memory
# and keep event size reasonable
base_event = {
"host": self._check.resolved_hostname,
"agent_version": datadog_agent.get_version(),
"dbms": "postgres",
"kind": "pg_databases",
"collection_interval": self.schemas_collection_interval,
"dbms_version": self._payload_pg_version(),
"tags": self._tags_no_db,
"cloud_metadata": self._config.cloud_metadata,
}

# Tuned from experiments on staging, we may want to make this dynamic based on schema size in the future
chunk_size = 50
total_tables = 0
start_time = time.time()

for database in schema_metadata:
dbname = database["name"]
if not self._should_collect_metadata(dbname, "database"):
continue

with self.db_pool.get_connection(dbname, self._config.idle_connection_timeout) as conn:
with conn.cursor(cursor_factory=psycopg2.extras.DictCursor) as cursor:
for schema in database["schemas"]:
if not self._should_collect_metadata(schema["name"], "schema"):
continue

tables = self._query_tables_for_schema(cursor, schema["id"], dbname)
self._log.debug(
"Tables found for schema '{schema}' in database '{database}':"
"{tables}".format(
schema=database["schemas"],
database=dbname,
tables=[table["name"] for table in tables],
)
)
)
table_chunks = list(get_list_chunks(tables, chunk_size))
table_chunks = list(get_list_chunks(tables, chunk_size))

buffer_column_count = 0
tables_buffer = []
buffer_column_count = 0
tables_buffer = []

for tables in table_chunks:
table_info = self._query_table_information(cursor, schema['name'], tables)
for tables in table_chunks:
table_info = self._query_table_information(cursor, schema['name'], tables)

tables_buffer = [*tables_buffer, *table_info]
for t in table_info:
buffer_column_count += len(t.get("columns", []))
tables_buffer = [*tables_buffer, *table_info]
for t in table_info:
buffer_column_count += len(t.get("columns", []))

if buffer_column_count >= 100_000:
if buffer_column_count >= 100_000:
self._flush_schema(base_event, database, schema, tables_buffer)
total_tables += len(tables_buffer)
tables_buffer = []
buffer_column_count = 0

if len(tables_buffer) > 0:
self._flush_schema(base_event, database, schema, tables_buffer)
total_tables += len(tables_buffer)
tables_buffer = []
buffer_column_count = 0

if len(tables_buffer) > 0:
self._flush_schema(base_event, database, schema, tables_buffer)
total_tables += len(tables_buffer)
elapsed_ms = (time.time() - start_time) * 1000
self._check.histogram(
"dd.postgres.schema.time",
elapsed_ms,
tags=self._check.tags,
hostname=self._check.resolved_hostname,
raw=True,
)
self._check.gauge(
"dd.postgres.schema.tables_count",
total_tables,
tags=self._check.tags,
hostname=self._check.resolved_hostname,
raw=True,
)
datadog_agent.emit_agent_telemetry("postgres", "schema_tables_elapsed_ms", elapsed_ms, "gauge")
datadog_agent.emit_agent_telemetry("postgres", "schema_tables_count", total_tables, "gauge")
except Exception as e:
self._log.error("Error collecting schema metadata: %s", e)
status = "error"
finally:
elapsed_ms = (time.time() - start_time) * 1000
self._check.histogram(
"dd.postgres.schema.time",
elapsed_ms,
tags=self._check.tags + ["status:" + status],
hostname=self._check.resolved_hostname,
raw=True,
)
self._check.gauge(
"dd.postgres.schema.tables_count",
total_tables,
tags=self._check.tags + ["status:" + status],
hostname=self._check.resolved_hostname,
raw=True,
)
datadog_agent.emit_agent_telemetry("postgres", "schema_tables_elapsed_ms", elapsed_ms, "gauge")
datadog_agent.emit_agent_telemetry("postgres", "schema_tables_count", total_tables, "gauge")

self._is_schemas_collection_in_progress = False
self._is_schemas_collection_in_progress = False

def _should_collect_metadata(self, name, metadata_type):
for re_str in self._config.schemas_metadata_config.get(
Expand Down
Loading