[improvement](statistics)Skip auto analyze when table row count is no…

…t fully reported. (return -1) (apache#42209) (apache#42407) backport: apache#42209
eldenmoon · Oct 24, 2024 · dba6296 · dba6296
1 parent 2f8d63f
commit dba6296
Show file tree

Hide file tree

Showing 3 changed files with 43 additions and 2 deletions.
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
@@ -130,6 +130,8 @@ public enum OlapTableState {
         WAITING_STABLE
     }
 
+    public static long ROW_COUNT_BEFORE_REPORT = -1;
+
     @SerializedName(value = "state")
     private volatile OlapTableState state;
 
@@ -1516,10 +1518,10 @@ public long getRowCountForIndex(long indexId, boolean strict) {
             if (index == null) {
                 LOG.warn("Index {} not exist in partition {}, table {}, {}",
                         indexId, entry.getValue().getName(), id, name);
-                return -1;
+                return ROW_COUNT_BEFORE_REPORT;
             }
             if (strict && !index.getRowCountReported()) {
-                return -1;
+                return ROW_COUNT_BEFORE_REPORT;
             }
             rowCount += index.getRowCount() == -1 ? 0 : index.getRowCount();
         }

diff --git a/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsAutoCollector.java b/fe/fe-core/src/main/java/org/apache/doris/statistics/StatisticsAutoCollector.java
@@ -179,6 +179,13 @@ protected void createAnalyzeJobForTbl(DatabaseIf<? extends TableIf> db,
             List<AnalysisInfo> analysisInfos, TableIf table) {
         AnalysisMethod analysisMethod = table.getDataSize(true) >= StatisticsUtil.getHugeTableLowerBoundSizeInBytes()
                 ? AnalysisMethod.SAMPLE : AnalysisMethod.FULL;
+        if (table instanceof OlapTable && analysisMethod.equals(AnalysisMethod.SAMPLE)) {
+            OlapTable ot = (OlapTable) table;
+            if (ot.getRowCountForIndex(ot.getBaseIndexId(), true) == OlapTable.ROW_COUNT_BEFORE_REPORT) {
+                LOG.info("Table {} row count is not fully reported, skip auto analyzing this time.", ot.getName());
+                return;
+            }
+        }
         long rowCount = StatisticsUtil.isEmptyTable(table, analysisMethod) ? 0 : table.getRowCount();
         AnalysisInfo jobInfo = new AnalysisInfoBuilder()
                 .setJobId(Env.getCurrentEnv().getNextId())

diff --git a/fe/fe-core/src/test/java/org/apache/doris/statistics/StatisticsAutoCollectorTest.java b/fe/fe-core/src/test/java/org/apache/doris/statistics/StatisticsAutoCollectorTest.java
@@ -470,4 +470,36 @@ protected boolean canCollect() {
 
         sac.analyzeAll();
     }
+
+    @Test
+    public void testCreateAnalyzeJobForTbl() {
+        StatisticsAutoCollector collector = new StatisticsAutoCollector();
+        OlapTable table = new OlapTable();
+        new MockUp<OlapTable>() {
+            @Mock
+            public long getDataSize(boolean singleReplica) {
+                return 100;
+            }
+
+            @Mock
+            public long getRowCountForIndex(long indexId, boolean strict) {
+                return -1;
+            }
+
+            @Mock
+            public boolean isPartitionedTable() {
+                return false;
+            }
+        };
+        List<AnalysisInfo> infos = Lists.newArrayList();
+        collector.createAnalyzeJobForTbl(null, infos, table);
+        Assertions.assertEquals(0, infos.size());
+        new MockUp<OlapTable>() {
+            @Mock
+            public long getRowCountForIndex(long indexId, boolean strict) {
+                return 100;
+            }
+        };
+        Assertions.assertThrows(NullPointerException.class, () -> collector.createAnalyzeJobForTbl(null, infos, table));
+    }
 }