Skip to content

Commit

Permalink
Merge #115727
Browse files Browse the repository at this point in the history 
115727: ui,obsservice: update queries r=maryliag a=maryliag

Update queries on the Insights API to use Obs Service or CRDB,
depending on the cluster setting `sql.insights.export.enabled`.

Also updates the parsing when we get the results, depending on
the source of the data.

Fixes CC-26437
Fixes CC-26440

Release note: None

Co-authored-by: maryliag <[email protected]>
  • Loading branch information
@maryliag
craig[bot] and maryliag committed Dec 12, 2023
2 parents ef2b9ca + 3b97eb6 commit 83bcafa
Show file tree
Hide file tree
Showing 14 changed files with 233 additions and 151 deletions.
2 changes: 1 addition & 1 deletion pkg/obsservice/obslib/process/stmt_insights_processor.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,7 @@ func (p *StmtInsightsProcessor) Process(
p.addInsight(stmtInsight)
insightsSize, lastExportTs := p.getInsightsInfo()

if insightsSize >= InsightsBatchMax || lastExportTs > time.Minute {
if insightsSize >= InsightsBatchMax || lastExportTs > time.Second*30 {
insertStmt, args := p.prepareInsightExport()
err := p.exportInsights(ctx, insertStmt, args)
if err != nil {
Expand Down
3 changes: 3 additions & 0 deletions pkg/sql/sqlstats/insights/registry.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -96,6 +96,9 @@ func (r *lockingRegistry) ObserveTransaction(
if !r.enabled() {
return
}
if transaction.ID.String() == "00000000-0000-0000-0000-000000000000" {
return
}
statements, ok := r.statements[sessionID]
if !ok {
return
Expand Down
227 changes: 163 additions & 64 deletions pkg/ui/workspaces/cluster-ui/src/api/stmtInsightsApi.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,7 @@ export type StmtInsightsReq = {
end?: moment.Moment;
stmtExecutionID?: string;
stmtFingerprintId?: string;
csExportInsights?: boolean;
useObsService?: boolean;
};

export type StmtInsightsResponseRow = {
Expand Down Expand Up @@ -70,6 +70,34 @@ export type StmtInsightsResponseRow = {
status: StatementStatus;
};

export type StmtInsightsObsServiceResponseRow = {
session_id: string;
transaction_id: string;
transaction_fingerprint_id: string; // hex string
implicit_txn: boolean;
statement_id: string;
statement_fingerprint_id: string; // hex string
query: string;
start_time: string; // Timestamp
end_time: string; // Timestamp
full_scan: boolean;
user_name: string;
app_name: string;
database_name: string;
priority: string;
retries: number;
exec_node_ids: number[];
contention: string; // interval
last_retry_reason?: string;
causes: string[];
problem: string;
index_recommendations: string[];
plan_gist: string;
cpu_sql_nanos: number;
error_code: string;
status: StatementStatus;
};

const stmtColumns = `
session_id,
txn_id,
Expand Down Expand Up @@ -101,40 +129,73 @@ last_error_redactable,
status
`;

const stmtInsightsOverviewQuery = (filters?: StmtInsightsReq): string => {
if (filters?.stmtExecutionID) {
return `
SELECT ${stmtColumns} FROM crdb_internal.cluster_execution_insights
WHERE stmt_id = '${filters.stmtExecutionID}'`;
// TODO(maryliag): update columns list once we store values for
// rows_read, rows_written and last_error_redactable.
const stmtColumnsObsService = `
session_id,
transaction_id,
transaction_fingerprint_id,
implicit_txn,
statement_id,
statement_fingerprint_id,
query,
start_time,
end_time,
full_scan,
user_name,
app_name,
database_name,
user_priority,
retries,
execution_node_ids,
contention_time,
last_retry_reason,
causes,
problem,
index_recommendations,
plan_gist,
cpu_sql_nanos,
error_code,
status
`;

const stmtInsightsOverviewQuery = (req?: StmtInsightsReq): string => {
const columns = req.useObsService ? stmtColumnsObsService : stmtColumns;
const table = req.useObsService
? "obsservice.statement_execution_insights"
: "crdb_internal.cluster_execution_insights";
const stmtIdColumnName = req.useObsService ? "statement_id" : "stmt_id";
if (req?.stmtExecutionID) {
return `SELECT ${columns} FROM ${table} WHERE ${stmtIdColumnName} = '${req.stmtExecutionID}'`;
}

const txnIdColumnName = req.useObsService ? "transaction_id" : "txn_id";
const stmtFingerprintIDColumnName = req.useObsService
? "statement_fingerprint_id"
: "stmt_fingerprint_id";
let whereClause = `
WHERE app_name NOT LIKE '${INTERNAL_APP_NAME_PREFIX}%'
AND problem != 'None'
AND txn_id != '00000000-0000-0000-0000-000000000000'`;
if (filters?.start) {
AND ${txnIdColumnName} != '00000000-0000-0000-0000-000000000000'`;
if (req?.start) {
whereClause =
whereClause + ` AND start_time >= '${filters.start.toISOString()}'`;
whereClause + ` AND start_time >= '${req.start.toISOString()}'`;
}
if (filters?.end) {
whereClause =
whereClause + ` AND end_time <= '${filters.end.toISOString()}'`;
if (req?.end) {
whereClause = whereClause + ` AND end_time <= '${req.end.toISOString()}'`;
}
if (filters?.stmtFingerprintId) {
if (req?.stmtFingerprintId) {
whereClause =
whereClause +
` AND encode(stmt_fingerprint_id, 'hex') = '${filters.stmtFingerprintId}'`;
` AND encode(${stmtFingerprintIDColumnName}, 'hex') = '${req.stmtFingerprintId}'`;
}

return `
SELECT ${stmtColumns} FROM
return `SELECT ${columns} FROM
(
SELECT DISTINCT ON (stmt_fingerprint_id, problem, causes)
SELECT DISTINCT ON (${stmtFingerprintIDColumnName}, problem, causes)
*
FROM
crdb_internal.cluster_execution_insights
${whereClause}
ORDER BY stmt_fingerprint_id, problem, causes, end_time DESC
FROM ${table} ${whereClause}
ORDER BY ${stmtFingerprintIDColumnName}, problem, causes, end_time DESC
)`;
};

Expand All @@ -156,10 +217,12 @@ export async function getStmtInsightsApi(
execute: true,
max_result_size: LARGE_RESULT_SIZE,
timeout: LONG_TIMEOUT,
use_obs_service: req.csExportInsights,
use_obs_service: req.useObsService,
};

const result = await executeInternalSql<StmtInsightsResponseRow>(request);
const result = await executeInternalSql<
StmtInsightsResponseRow | StmtInsightsObsServiceResponseRow
>(request);

if (sqlResultsAreEmpty(result)) {
return formatApiResult<StmtInsightEvent[]>(
Expand All @@ -168,7 +231,10 @@ export async function getStmtInsightsApi(
"retrieving insights information",
);
}
const stmtInsightEvent = formatStmtInsights(result.execution?.txn_results[0]);
const stmtInsightEvent = formatStmtInsights(
result.execution?.txn_results[0],
req.useObsService,
);
await addStmtContentionInfoApi(stmtInsightEvent);
return formatApiResult<StmtInsightEvent[]>(
stmtInsightEvent,
Expand Down Expand Up @@ -205,49 +271,82 @@ async function addStmtContentionInfoApi(
}

export function formatStmtInsights(
response: SqlTxnResult<StmtInsightsResponseRow>,
response: SqlTxnResult<
StmtInsightsResponseRow | StmtInsightsObsServiceResponseRow
>,
useObsService?: boolean,
): StmtInsightEvent[] {
if (!response?.rows?.length) {
return [];
}

return response.rows.map((row: StmtInsightsResponseRow) => {
const start = moment.utc(row.start_time);
const end = moment.utc(row.end_time);

return {
transactionExecutionID: row.txn_id,
transactionFingerprintID: FixFingerprintHexValue(row.txn_fingerprint_id),
implicitTxn: row.implicit_txn,
databaseName: row.database_name,
application: row.app_name,
username: row.user_name,
sessionID: row.session_id,
priority: row.priority,
retries: row.retries,
lastRetryReason: row.last_retry_reason,
query: row.query,
startTime: start,
endTime: end,
elapsedTimeMillis: end.diff(start, "milliseconds"),
statementExecutionID: row.stmt_id,
statementFingerprintID: FixFingerprintHexValue(row.stmt_fingerprint_id),
isFullScan: row.full_scan,
rowsRead: row.rows_read,
rowsWritten: row.rows_written,
// This is the total stmt contention.
contentionTime: row.contention ? moment.duration(row.contention) : null,
indexRecommendations: row.index_recommendations,
insights: getInsightsFromProblemsAndCauses(
[row.problem],
row.causes,
InsightExecEnum.STATEMENT,
),
planGist: row.plan_gist,
cpuSQLNanos: row.cpu_sql_nanos,
errorCode: row.error_code,
errorMsg: row.last_error_redactable,
status: row.status,
} as StmtInsightEvent;
});
let txnID;
let txnFingerprintID;
let stmtID;
let stmtFingerprintID;
let rowsRead;
let rowsWritten;
let lastErrorRedactable;

return response.rows.map(
(row: StmtInsightsResponseRow | StmtInsightsObsServiceResponseRow) => {
const start = moment.utc(row.start_time);
const end = moment.utc(row.end_time);
if (useObsService) {
const r = row as StmtInsightsObsServiceResponseRow;
txnID = r.transaction_id;
txnFingerprintID = r.transaction_fingerprint_id;
stmtID = r.statement_id;
stmtFingerprintID = r.statement_fingerprint_id;
// TODO(maryliag); collect the values for rows read, rows written and last error redactable.
rowsRead = 0;
rowsWritten = 0;
lastErrorRedactable = "";
} else {
const r = row as StmtInsightsResponseRow;
txnID = r.txn_id;
txnFingerprintID = r.txn_fingerprint_id;
stmtID = r.stmt_id;
stmtFingerprintID = r.stmt_fingerprint_id;
rowsRead = r.rows_read;
rowsWritten = r.rows_written;
lastErrorRedactable = r.last_error_redactable;
}

return {
transactionExecutionID: txnID,
transactionFingerprintID: FixFingerprintHexValue(txnFingerprintID),
implicitTxn: row.implicit_txn,
databaseName: row.database_name,
application: row.app_name,
username: row.user_name,
sessionID: row.session_id,
priority: row.priority,
retries: row.retries,
lastRetryReason: row.last_retry_reason,
query: row.query,
startTime: start,
endTime: end,
elapsedTimeMillis: end.diff(start, "milliseconds"),
statementExecutionID: stmtID,
statementFingerprintID: FixFingerprintHexValue(stmtFingerprintID),
isFullScan: row.full_scan,
rowsRead: rowsRead,
rowsWritten: rowsWritten,
// This is the total stmt contention.
contentionTime: row.contention ? moment.duration(row.contention) : null,
indexRecommendations: row.index_recommendations,
insights: getInsightsFromProblemsAndCauses(
[row.problem],
row.causes,
InsightExecEnum.STATEMENT,
),
planGist: row.plan_gist,
cpuSQLNanos: row.cpu_sql_nanos,
errorCode: row.error_code,
errorMsg: lastErrorRedactable,
status: row.status,
} as StmtInsightEvent;
},
);
}
42 changes: 21 additions & 21 deletions pkg/ui/workspaces/cluster-ui/src/insights/utils.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -396,23 +396,23 @@ export function getStmtInsightRecommendations(
if (!insightDetails) return [];

const execDetails: ExecutionDetails = {
application: insightDetails.application,
statement: insightDetails.query,
fingerprintID: insightDetails.statementFingerprintID,
retries: insightDetails.retries,
indexRecommendations: insightDetails.indexRecommendations,
databaseName: insightDetails.databaseName,
elapsedTimeMillis: insightDetails.elapsedTimeMillis,
contentionTimeMs: insightDetails.contentionTime?.asMilliseconds(),
statementExecutionID: insightDetails.statementExecutionID,
transactionExecutionID: insightDetails.transactionExecutionID,
application: insightDetails?.application,
statement: insightDetails?.query,
fingerprintID: insightDetails?.statementFingerprintID,
retries: insightDetails?.retries,
indexRecommendations: insightDetails?.indexRecommendations,
databaseName: insightDetails?.databaseName,
elapsedTimeMillis: insightDetails?.elapsedTimeMillis,
contentionTimeMs: insightDetails?.contentionTime?.asMilliseconds(),
statementExecutionID: insightDetails?.statementExecutionID,
transactionExecutionID: insightDetails?.transactionExecutionID,
execType: InsightExecEnum.STATEMENT,
errorCode: insightDetails.errorCode,
errorMsg: insightDetails.errorMsg,
status: insightDetails.status,
errorCode: insightDetails?.errorCode,
errorMsg: insightDetails?.errorMsg,
status: insightDetails?.status,
};

const recs: InsightRecommendation[] = insightDetails.insights?.map(insight =>
const recs: InsightRecommendation[] = insightDetails?.insights?.map(insight =>
getRecommendationForExecInsight(insight, execDetails),
);

Expand All @@ -425,14 +425,14 @@ export function getTxnInsightRecommendations(
if (!insightDetails) return [];

const execDetails: ExecutionDetails = {
application: insightDetails.application,
transactionExecutionID: insightDetails.transactionExecutionID,
retries: insightDetails.retries,
contentionTimeMs: insightDetails.contentionTime.asMilliseconds(),
elapsedTimeMillis: insightDetails.elapsedTimeMillis,
application: insightDetails?.application,
transactionExecutionID: insightDetails?.transactionExecutionID,
retries: insightDetails?.retries,
contentionTimeMs: insightDetails?.contentionTime.asMilliseconds(),
elapsedTimeMillis: insightDetails?.elapsedTimeMillis,
execType: InsightExecEnum.TRANSACTION,
errorCode: insightDetails.errorCode,
errorMsg: insightDetails.errorMsg,
errorCode: insightDetails?.errorCode,
errorMsg: insightDetails?.errorMsg,
};
const recs: InsightRecommendation[] = [];

Expand Down
Loading

0 comments on commit 83bcafa

Please sign in to comment.