diff --git a/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/exec/ControllerImpl.java b/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/exec/ControllerImpl.java index 5c491c0780d2..db95c0a6bd90 100644 --- a/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/exec/ControllerImpl.java +++ b/extensions-core/multi-stage-query/src/main/java/org/apache/druid/msq/exec/ControllerImpl.java @@ -86,6 +86,7 @@ import org.apache.druid.java.util.common.granularity.Granularity; import org.apache.druid.java.util.common.io.Closer; import org.apache.druid.java.util.common.logger.Logger; +import org.apache.druid.msq.counters.ChannelCounters; import org.apache.druid.msq.counters.CounterSnapshots; import org.apache.druid.msq.counters.CounterSnapshotsTree; import org.apache.druid.msq.indexing.InputChannelFactory; @@ -162,6 +163,7 @@ import org.apache.druid.msq.querykit.scan.ScanQueryKit; import org.apache.druid.msq.shuffle.input.DurableStorageInputChannelFactory; import org.apache.druid.msq.shuffle.input.WorkerInputChannelFactory; +import org.apache.druid.msq.statistics.ClusterByStatisticsCollector; import org.apache.druid.msq.statistics.PartialKeyStatisticsInformation; import org.apache.druid.msq.util.IntervalUtils; import org.apache.druid.msq.util.MSQFutureUtils; @@ -514,7 +516,7 @@ private MSQTaskReportPayload runInternal(final QueryListener queryListener, fina stagesReport = null; } - return new MSQTaskReportPayload( + final MSQTaskReportPayload msqTaskReportPayload = new MSQTaskReportPayload( makeStatusReport( taskStateForReport, errorForReport, @@ -529,6 +531,32 @@ private MSQTaskReportPayload runInternal(final QueryListener queryListener, fina countersSnapshot, null ); + // Emit summary metrics + emitSummaryMetrics(msqTaskReportPayload, querySpec); + return msqTaskReportPayload; + } + + private void emitSummaryMetrics(final MSQTaskReportPayload msqTaskReportPayload, final MSQSpec querySpec) + { + long totalProcessedBytes = 0; + + if (msqTaskReportPayload.getCounters() != null) { + totalProcessedBytes = msqTaskReportPayload.getCounters() + .copyMap() + .values() + .stream() + .flatMap(counterSnapshotsMap -> counterSnapshotsMap.values().stream()) + .flatMap(counterSnapshots -> counterSnapshots.getMap().entrySet().stream()) + .filter(entry -> entry.getKey().startsWith("input")) + .mapToLong(entry -> { + ChannelCounters.Snapshot snapshot = (ChannelCounters.Snapshot) entry.getValue(); + return snapshot.getBytes() == null ? 0L : Arrays.stream(snapshot.getBytes()).sum(); + }) + .sum(); + } + + log.debug("Processed bytes[%d] for query[%s].", totalProcessedBytes, querySpec.getQuery()); + context.emitMetric("ingest/processed/bytes", totalProcessedBytes); } /** @@ -2418,7 +2446,7 @@ private void startTaskLauncher() } /** - * Enqueues the fetching {@link org.apache.druid.msq.statistics.ClusterByStatisticsCollector} + * Enqueues the fetching {@link ClusterByStatisticsCollector} * from each worker via {@link WorkerSketchFetcher} */ private void fetchStatsFromWorkers() diff --git a/indexing-service/src/main/java/org/apache/druid/indexing/common/task/batch/parallel/ParallelIndexSupervisorTask.java b/indexing-service/src/main/java/org/apache/druid/indexing/common/task/batch/parallel/ParallelIndexSupervisorTask.java index 03d50c43f06d..beeffac4c5dc 100644 --- a/indexing-service/src/main/java/org/apache/druid/indexing/common/task/batch/parallel/ParallelIndexSupervisorTask.java +++ b/indexing-service/src/main/java/org/apache/druid/indexing/common/task/batch/parallel/ParallelIndexSupervisorTask.java @@ -64,6 +64,7 @@ import org.apache.druid.java.util.common.StringUtils; import org.apache.druid.java.util.common.granularity.Granularity; import org.apache.druid.java.util.common.logger.Logger; +import org.apache.druid.java.util.emitter.service.ServiceMetricEvent; import org.apache.druid.rpc.HttpResponseException; import org.apache.druid.rpc.indexing.OverlordClient; import org.apache.druid.segment.SegmentSchemaMapping; @@ -1251,12 +1252,19 @@ private TaskStatus runSequential(TaskToolbox toolbox) throws Exception */ private TaskReport.ReportMap getTaskCompletionReports(TaskStatus taskStatus) { - return buildIngestionStatsAndContextReport( + final var taskCompletionReport = buildIngestionStatsAndContextReport( IngestionState.COMPLETED, taskStatus.getErrorMsg(), segmentsRead, segmentsPublished ); + final var totalProcessedBytes = indexGenerateRowStats.lhs.get("processedBytes"); + // Emit the processed bytes metric + final ServiceMetricEvent.Builder metricBuilder = new ServiceMetricEvent.Builder(); + //IndexTaskUtils.setTaskDimensions(metricBuilder, task); + toolbox.getEmitter().emit( + metricBuilder.setMetric("ingest/processed/bytes", (Number) totalProcessedBytes)); + return taskCompletionReport; } @Override @@ -1633,6 +1641,9 @@ private Pair, Map> doGetRowStatsAndUnparseab ); buildSegmentsRowStats.addRowIngestionMetersTotals(rowStatsForRunningTasks); + // Emit the processed bytes metric + emitMetric(toolbox.getEmitter(), "ingest/processed/bytes", rowStatsForRunningTasks.getProcessedBytes()); + return createStatsAndErrorsReport(buildSegmentsRowStats.getTotals(), unparseableEvents); }