From d6313143857bb5209c9930b6f70d5ecb101ee62b Mon Sep 17 00:00:00 2001
From: Devan <devandbenz@gmail.com>
Date: Mon, 16 Dec 2024 17:30:35 -0600
Subject: [PATCH 01/31] feat: Modify optimized compaction to cover edge cases
 This PR changes the algorithm for compaction to account for the following
 cases that were not previously accounted for:

- Many generations with a groupsize over 2 GB
- Single generation with many files and a groupsize under 2 GB
Where groupsize is the total size of the TSM files in said shard directory.

closes https://github.com/influxdata/influxdb/issues/25666
---
 tsdb/engine/tsm1/compact.go      |  66 +++++----
 tsdb/engine/tsm1/compact_test.go | 227 ++++++++++++++++++++++++++++++-
 tsdb/engine/tsm1/engine.go       |  14 +-
 tsdb/engine/tsm1/engine_test.go  |   2 +-
 4 files changed, 278 insertions(+), 31 deletions(-)

diff --git a/tsdb/engine/tsm1/compact.go b/tsdb/engine/tsm1/compact.go
index c4be1156766..63eee35f693 100644
--- a/tsdb/engine/tsm1/compact.go
+++ b/tsdb/engine/tsm1/compact.go
@@ -96,7 +96,7 @@ type CompactionGroup []string
 type CompactionPlanner interface {
 	Plan(lastWrite time.Time) ([]CompactionGroup, int64)
 	PlanLevel(level int) ([]CompactionGroup, int64)
-	PlanOptimize() ([]CompactionGroup, int64)
+	PlanOptimize() ([]CompactionGroup, int64, int64)
 	Release(group []CompactionGroup)
 	FullyCompacted() (bool, string)
 
@@ -225,6 +225,10 @@ func (c *DefaultPlanner) FullyCompacted() (bool, string) {
 	} else if gens.hasTombstones() {
 		return false, "not fully compacted and not idle because of tombstones"
 	} else {
+		// Safety: check for first index so we don't accidentally do out of bounds access
+		if len(gens) == 1 && len(gens[0].files) > 1 && gens[0].size() < uint64(maxTSMFileSize) {
+			return false, "not fully compacted and not idle because group size under 2 GB and more then single file"
+		}
 		return true, ""
 	}
 }
@@ -335,13 +339,13 @@ func (c *DefaultPlanner) PlanLevel(level int) ([]CompactionGroup, int64) {
 // PlanOptimize returns all TSM files if they are in different generations in order
 // to optimize the index across TSM files.  Each returned compaction group can be
 // compacted concurrently.
-func (c *DefaultPlanner) PlanOptimize() ([]CompactionGroup, int64) {
+func (c *DefaultPlanner) PlanOptimize() ([]CompactionGroup, int64, int64) {
 	// If a full plan has been requested, don't plan any levels which will prevent
 	// the full plan from acquiring them.
 	c.mu.RLock()
 	if c.forceFull {
 		c.mu.RUnlock()
-		return nil, 0
+		return nil, 0, 0
 	}
 	c.mu.RUnlock()
 
@@ -350,10 +354,20 @@ func (c *DefaultPlanner) PlanOptimize() ([]CompactionGroup, int64) {
 	// split across several files in sequence.
 	generations := c.findGenerations(true)
 
-	// If there is only one generation and no tombstones, then there's nothing to
-	// do.
-	if len(generations) <= 1 && !generations.hasTombstones() {
-		return nil, 0
+	// Safety check for potential 0 generations.
+	if len(generations) < 1 {
+		return nil, 0, 0
+	}
+
+	// Return if there's only a single file and single generation.
+	if len(generations) == 1 && len(generations[0].files) == 1 {
+		return nil, 0, 0
+	}
+
+	// If there is a single generation, no tombstones, and the entire group size (all TSM files in the generation)
+	// is over or equal to 2 GB (Max TSM file size) there is nothing to do and we will return.
+	if len(generations) == 1 && generations[0].size() >= uint64(maxTSMFileSize) && !generations.hasTombstones() {
+		return nil, 0, 0
 	}
 
 	// Group each generation by level such that two adjacent generations in the same
@@ -363,11 +377,6 @@ func (c *DefaultPlanner) PlanOptimize() ([]CompactionGroup, int64) {
 	for i := 0; i < len(generations); i++ {
 		cur := generations[i]
 
-		// Skip the file if it's over the max size and contains a full block and it does not have any tombstones
-		if cur.count() > 2 && cur.size() > uint64(maxTSMFileSize) && c.FileStore.BlockCount(cur.files[0].Path, 1) == tsdb.DefaultMaxPointsPerBlock && !cur.hasTombstones() {
-			continue
-		}
-
 		// See if this generation is orphan'd which would prevent it from being further
 		// compacted until a final full compactin runs.
 		if i < len(generations)-1 {
@@ -392,21 +401,21 @@ func (c *DefaultPlanner) PlanOptimize() ([]CompactionGroup, int64) {
 	}
 
 	// Only optimize level 4 files since using lower-levels will collide
-	// with the level planners
+	// with the level planners. If this is a single generation optimization
+	// do not skip any levels.
 	var levelGroups []tsmGenerations
-	for _, cur := range groups {
-		if cur.level() == 4 {
-			levelGroups = append(levelGroups, cur)
+	if len(generations) == 1 {
+		levelGroups = append(levelGroups, groups...)
+	} else {
+		for _, cur := range groups {
+			if cur.level() == 4 {
+				levelGroups = append(levelGroups, cur)
+			}
 		}
 	}
 
 	var cGroups []CompactionGroup
 	for _, group := range levelGroups {
-		// Skip the group if it's not worthwhile to optimize it
-		if len(group) < 4 && !group.hasTombstones() {
-			continue
-		}
-
 		var cGroup CompactionGroup
 		for _, gen := range group {
 			for _, file := range gen.files {
@@ -417,11 +426,18 @@ func (c *DefaultPlanner) PlanOptimize() ([]CompactionGroup, int64) {
 		cGroups = append(cGroups, cGroup)
 	}
 
+	// The third return value is the generation length.
+	// Need to use this to decide how many points per block to use during compaction.
+	// This value is mostly ignored in normal compaction code paths, but,
+	// for the edge case where there is a single generation with many
+	// files and a group size under 2 GB we need to know that there is a single
+	// generation so the block size can be adjusted to allow for more optimal
+	// compaction.
 	if !c.acquire(cGroups) {
-		return nil, int64(len(cGroups))
+		return nil, int64(len(cGroups)), int64(len(generations))
 	}
 
-	return cGroups, int64(len(cGroups))
+	return cGroups, int64(len(cGroups)), int64(len(generations))
 }
 
 // Plan returns a set of TSM files to rewrite for level 4 or higher.  The planning returns
@@ -905,6 +921,10 @@ func (c *Compactor) WriteSnapshot(cache *Cache, logger *zap.Logger) ([]string, e
 
 // compact writes multiple smaller TSM files into 1 or more larger files.
 func (c *Compactor) compact(fast bool, tsmFiles []string, logger *zap.Logger) ([]string, error) {
+	// Sets the points per block size. The larger this value is set
+	// the more points there will be a single index. Under normal
+	// conditions this should always be 1000 but there is an edge case
+	// where this is increased.
 	size := c.Size
 	if size <= 0 {
 		size = tsdb.DefaultMaxPointsPerBlock
diff --git a/tsdb/engine/tsm1/compact_test.go b/tsdb/engine/tsm1/compact_test.go
index c90beb6b1cf..2464cee2bc8 100644
--- a/tsdb/engine/tsm1/compact_test.go
+++ b/tsdb/engine/tsm1/compact_test.go
@@ -12,6 +12,7 @@ import (
 
 	"github.com/influxdata/influxdb/tsdb"
 	"github.com/influxdata/influxdb/tsdb/engine/tsm1"
+	"github.com/stretchr/testify/require"
 	"go.uber.org/zap"
 )
 
@@ -2200,11 +2201,13 @@ func TestDefaultPlanner_PlanOptimize_NoLevel4(t *testing.T) {
 	)
 
 	expFiles := []tsm1.FileStat{}
-	tsm, pLen := cp.PlanOptimize()
+	tsm, pLen, gLen := cp.PlanOptimize()
 	if exp, got := len(expFiles), len(tsm); got != exp {
 		t.Fatalf("tsm file length mismatch: got %v, exp %v", got, exp)
 	} else if pLen != int64(len(tsm)) {
 		t.Fatalf("tsm file plan length mismatch: got %v, exp %v", pLen, exp)
+	} else if gLen != int64(3) {
+		t.Fatalf("generation len plan mismatch: got %v, exp %v", gLen, 3)
 	}
 }
 
@@ -2249,7 +2252,7 @@ func TestDefaultPlanner_PlanOptimize_Level4(t *testing.T) {
 	)
 
 	expFiles1 := []tsm1.FileStat{data[0], data[1], data[2], data[3], data[4], data[5]}
-	tsm, pLen := cp.PlanOptimize()
+	tsm, pLen, _ := cp.PlanOptimize()
 	if exp, got := 1, len(tsm); exp != got {
 		t.Fatalf("group length mismatch: got %v, exp %v", got, exp)
 	} else if pLen != int64(len(tsm)) {
@@ -2267,6 +2270,218 @@ func TestDefaultPlanner_PlanOptimize_Level4(t *testing.T) {
 	}
 }
 
+// This test is added to acount for many TSM files within a group being over 2 GB
+// we want to ensure that the shard will be planned.
+func TestDefaultPlanner_PlanOptimize_LargeMultiGeneration(t *testing.T) {
+	data := []tsm1.FileStat{
+		{
+			Path: "01-05.tsm1",
+			Size: 2048 * 1024 * 1024,
+		},
+		{
+			Path: "01-06.tsm1",
+			Size: 2048 * 1024 * 1024,
+		},
+		{
+			Path: "01-07.tsm1",
+			Size: 2048 * 1024 * 1024,
+		},
+		{
+			Path: "01-08.tsm1",
+			Size: 1048 * 1024 * 1024,
+		},
+		{
+			Path: "02-05.tsm1",
+			Size: 2048 * 1024 * 1024,
+		},
+		{
+			Path: "02-06.tsm1",
+			Size: 2048 * 1024 * 1024,
+		},
+		{
+			Path: "02-07.tsm1",
+			Size: 2048 * 1024 * 1024,
+		},
+		{
+			Path: "02-08.tsm1",
+			Size: 1048 * 1024 * 1024,
+		},
+		{
+			Path: "03-04.tsm1",
+			Size: 2048 * 1024 * 1024,
+		},
+		{
+			Path: "03-05.tsm1",
+			Size: 512 * 1024 * 1024,
+		},
+	}
+
+	cp := tsm1.NewDefaultPlanner(
+		&fakeFileStore{
+			PathsFn: func() []tsm1.FileStat {
+				return data
+			},
+		}, tsdb.DefaultCompactFullWriteColdDuration,
+	)
+
+	expFiles := make([]tsm1.FileStat, 0)
+	for _, file := range data {
+		expFiles = append(expFiles, file)
+	}
+
+	tsm, pLen, _ := cp.PlanOptimize()
+	require.Equal(t, 1, len(tsm), "group length mismatch: got %d, exp %d", len(tsm), 1)
+	require.Equal(t, int64(len(tsm)), pLen, "tsm file plan length mismatch: got %d, exp %d", pLen, int64(len(tsm)))
+	require.Equal(t, len(expFiles), len(tsm[0]), "tsm file length mismatch: got %d, exp %d", len(tsm[0]), len(expFiles))
+}
+
+// This test is added to account for a single generation that has a group size
+// under 2 GB so it should be further compacted to a single file.
+func TestDefaultPlanner_PlanOptimize_SmallSingleGeneration(t *testing.T) {
+	// ~650 MB total group size
+	data := []tsm1.FileStat{
+		{
+			Path: "01-05.tsm1",
+			Size: 300 * 1024 * 1024,
+		},
+		{
+			Path: "01-06.tsm1",
+			Size: 200 * 1024 * 1024,
+		},
+		{
+			Path: "01-07.tsm1",
+			Size: 100 * 1024 * 1024,
+		},
+		{
+			Path: "01-08.tsm1",
+			Size: 50 * 1024 * 1024,
+		},
+	}
+
+	cp := tsm1.NewDefaultPlanner(
+		&fakeFileStore{
+			PathsFn: func() []tsm1.FileStat {
+				return data
+			},
+		}, tsdb.DefaultCompactFullWriteColdDuration,
+	)
+
+	expFiles := make([]tsm1.FileStat, 0)
+	for _, file := range data {
+		expFiles = append(expFiles, file)
+	}
+
+	tsm, pLen, gLen := cp.PlanOptimize()
+	require.Equal(t, 1, len(tsm), "group length mismatch: got %d, exp %d", len(tsm), 1)
+	require.Equal(t, int64(len(tsm)), pLen, "tsm file plan length mismatch: got %d, exp %d", pLen, int64(len(expFiles)))
+	require.Equal(t, int64(1), gLen, "generation length mismatch: got %d, exp %d", gLen, 1)
+	require.Equal(t, len(expFiles), len(tsm[0]), "tsm file length mismatch: got %d, exp %d", len(tsm[0]), expFiles)
+}
+
+// This test is added to account for a single generation that has a group size
+// under 2 GB and has less then level 4 files it should be further compacted to a single file.
+// FullyCompacted should NOT skip over opening this shard.
+func TestDefaultPlanner_PlanOptimize_SmallSingleGenerationUnderLevel4(t *testing.T) {
+	// ~650 MB total group size
+	data := []tsm1.FileStat{
+		{
+			Path: "01-02.tsm1",
+			Size: 300 * 1024 * 1024,
+		},
+		{
+			Path: "01-03.tsm1",
+			Size: 200 * 1024 * 1024,
+		},
+		{
+			Path: "01-04.tsm1",
+			Size: 100 * 1024 * 1024,
+		},
+	}
+
+	cp := tsm1.NewDefaultPlanner(
+		&fakeFileStore{
+			PathsFn: func() []tsm1.FileStat {
+				return data
+			},
+		}, tsdb.DefaultCompactFullWriteColdDuration,
+	)
+
+	expFiles := make([]tsm1.FileStat, 0)
+	for _, file := range data {
+		expFiles = append(expFiles, file)
+	}
+
+	tsm, pLen, gLen := cp.PlanOptimize()
+	require.Equal(t, 1, len(tsm), "group length mismatch: got %d, exp %d", len(tsm), 1)
+	require.Equal(t, int64(len(tsm)), pLen, "tsm file plan length mismatch: got %d, exp %d", pLen, int64(len(expFiles)))
+	require.Equal(t, int64(1), gLen, "generation length mismatch: got %d, exp %d", gLen, 1)
+	require.Equal(t, len(expFiles), len(tsm[0]), "tsm file length mismatch: got %d, exp %d", len(tsm[0]), expFiles)
+}
+
+// This test is added to account for a single generation that has a group size
+// under 2 GB so it should be further compacted to a single file.
+// FullyCompacted should NOT skip over opening this shard.
+func TestDefaultPlanner_FullyCompacted_SmallSingleGeneration(t *testing.T) {
+	// ~650 MB total group size
+	data := []tsm1.FileStat{
+		{
+			Path: "01-05.tsm1",
+			Size: 300 * 1024 * 1024,
+		},
+		{
+			Path: "01-06.tsm1",
+			Size: 200 * 1024 * 1024,
+		},
+		{
+			Path: "01-07.tsm1",
+			Size: 100 * 1024 * 1024,
+		},
+		{
+			Path: "01-08.tsm1",
+			Size: 50 * 1024 * 1024,
+		},
+	}
+
+	cp := tsm1.NewDefaultPlanner(
+		&fakeFileStore{
+			PathsFn: func() []tsm1.FileStat {
+				return data
+			},
+		}, tsdb.DefaultCompactFullWriteColdDuration,
+	)
+
+	compacted, reason := cp.FullyCompacted()
+	reasonExp := "not fully compacted and not idle because group size under 2 GB and more then single file"
+	require.Equal(t, reason, reasonExp)
+	require.Equal(t, false, compacted)
+}
+
+// This test is added to account for halting state after
+// TestDefaultPlanner_FullyCompacted_SmallSingleGeneration
+// will need to ensure that once we have single TSM file under 2 GB we stop
+func TestDefaultPlanner_FullyCompacted_SmallSingleGeneration_Halt(t *testing.T) {
+	// ~650 MB total group size
+	data := []tsm1.FileStat{
+		{
+			Path: "01-09.tsm1",
+			Size: 650 * 1024 * 1024,
+		},
+	}
+
+	cp := tsm1.NewDefaultPlanner(
+		&fakeFileStore{
+			PathsFn: func() []tsm1.FileStat {
+				return data
+			},
+		}, tsdb.DefaultCompactFullWriteColdDuration,
+	)
+
+	compacted, reason := cp.FullyCompacted()
+	reasonExp := ""
+	require.Equal(t, reason, reasonExp)
+	require.Equal(t, true, compacted)
+}
+
 func TestDefaultPlanner_PlanOptimize_Multiple(t *testing.T) {
 	data := []tsm1.FileStat{
 		{
@@ -2322,7 +2537,7 @@ func TestDefaultPlanner_PlanOptimize_Multiple(t *testing.T) {
 	expFiles1 := []tsm1.FileStat{data[0], data[1], data[2], data[3]}
 	expFiles2 := []tsm1.FileStat{data[6], data[7], data[8], data[9]}
 
-	tsm, pLen := cp.PlanOptimize()
+	tsm, pLen, _ := cp.PlanOptimize()
 	if exp, got := 2, len(tsm); exp != got {
 		t.Fatalf("group length mismatch: got %v, exp %v", got, exp)
 	} else if pLen != int64(len(tsm)) {
@@ -2375,7 +2590,7 @@ func TestDefaultPlanner_PlanOptimize_Optimized(t *testing.T) {
 	)
 
 	expFiles := []tsm1.FileStat{}
-	tsm, pLen := cp.PlanOptimize()
+	tsm, pLen, _ := cp.PlanOptimize()
 	if exp, got := len(expFiles), len(tsm); got != exp {
 		t.Fatalf("tsm file length mismatch: got %v, exp %v", got, exp)
 	} else if pLen != int64(len(tsm)) {
@@ -2409,7 +2624,7 @@ func TestDefaultPlanner_PlanOptimize_Tombstones(t *testing.T) {
 	)
 
 	expFiles := []tsm1.FileStat{data[0], data[1], data[2]}
-	tsm, pLen := cp.PlanOptimize()
+	tsm, pLen, _ := cp.PlanOptimize()
 	if exp, got := len(expFiles), len(tsm[0]); got != exp {
 		t.Fatalf("tsm file length mismatch: got %v, exp %v", got, exp)
 	} else if pLen != int64(len(tsm)) {
@@ -2586,7 +2801,7 @@ func TestDefaultPlanner_Plan_SkipPlanningAfterFull(t *testing.T) {
 	}
 	cp.Release(plan)
 
-	plan, pLen = cp.PlanOptimize()
+	plan, pLen, _ = cp.PlanOptimize()
 	// ensure the optimize planner would pick this up
 	if exp, got := 1, len(plan); got != exp {
 		t.Fatalf("tsm file length mismatch: got %v, exp %v", got, exp)
diff --git a/tsdb/engine/tsm1/engine.go b/tsdb/engine/tsm1/engine.go
index a27e41f8254..4e1d7defeb7 100644
--- a/tsdb/engine/tsm1/engine.go
+++ b/tsdb/engine/tsm1/engine.go
@@ -2130,8 +2130,9 @@ func (e *Engine) compact(wg *sync.WaitGroup) {
 			atomic.StoreInt64(&e.stats.TSMOptimizeCompactionsQueue, len4)
 
 			// If no full compactions are need, see if an optimize is needed
+			var genLen int64
 			if len(level4Groups) == 0 {
-				level4Groups, len4 = e.CompactionPlan.PlanOptimize()
+				level4Groups, len4, genLen = e.CompactionPlan.PlanOptimize()
 				atomic.StoreInt64(&e.stats.TSMOptimizeCompactionsQueue, len4)
 			}
 
@@ -2166,6 +2167,17 @@ func (e *Engine) compact(wg *sync.WaitGroup) {
 						level3Groups = level3Groups[1:]
 					}
 				case 4:
+					// This is a heuristic. 100_000 points per block is suitable for when we have a
+					// single generation with multiple files at max block size under 2 GB.
+					if genLen == 1 {
+						// Log TSM files that will have an increased points per block count.
+						for _, f := range level4Groups[0] {
+							e.logger.Info("TSM optimized compaction on single generation running, increasing total points per block to 100_000.", zap.String("path", f))
+						}
+						e.Compactor.Size = tsdb.DefaultMaxPointsPerBlock * 100
+					} else {
+						e.Compactor.Size = tsdb.DefaultMaxPointsPerBlock
+					}
 					if e.compactFull(level4Groups[0], wg) {
 						level4Groups = level4Groups[1:]
 					}
diff --git a/tsdb/engine/tsm1/engine_test.go b/tsdb/engine/tsm1/engine_test.go
index 947f4f65842..ae7cc19a330 100644
--- a/tsdb/engine/tsm1/engine_test.go
+++ b/tsdb/engine/tsm1/engine_test.go
@@ -2889,7 +2889,7 @@ type mockPlanner struct{}
 
 func (m *mockPlanner) Plan(lastWrite time.Time) ([]tsm1.CompactionGroup, int64) { return nil, 0 }
 func (m *mockPlanner) PlanLevel(level int) ([]tsm1.CompactionGroup, int64)      { return nil, 0 }
-func (m *mockPlanner) PlanOptimize() ([]tsm1.CompactionGroup, int64)            { return nil, 0 }
+func (m *mockPlanner) PlanOptimize() ([]tsm1.CompactionGroup, int64, int64)     { return nil, 0, 0 }
 func (m *mockPlanner) Release(groups []tsm1.CompactionGroup)                    {}
 func (m *mockPlanner) FullyCompacted() (bool, string)                           { return false, "not compacted" }
 func (m *mockPlanner) ForceFull()                                               {}

From 67849aee729c729890017a5fc410e580859afdfe Mon Sep 17 00:00:00 2001
From: Devan <devandbenz@gmail.com>
Date: Tue, 17 Dec 2024 15:03:32 -0600
Subject: [PATCH 02/31] feat: Modify the PR to include optimized compaction for
 shards that may have over a 2 GB group size but many fragmented files (under
 2 GB and under aggressive point per block count)

---
 tsdb/config.go                   |   4 +
 tsdb/engine/tsm1/compact.go      |  50 ++--
 tsdb/engine/tsm1/compact_test.go | 399 +++++++++++++++++++++++++------
 tsdb/engine/tsm1/engine.go       |   2 +-
 4 files changed, 368 insertions(+), 87 deletions(-)

diff --git a/tsdb/config.go b/tsdb/config.go
index feb4927783a..a58e1680528 100644
--- a/tsdb/config.go
+++ b/tsdb/config.go
@@ -52,6 +52,10 @@ const (
 	// block in a TSM file
 	DefaultMaxPointsPerBlock = 1000
 
+	// AggressiveMaxPointsPerBlock is used when we want to further compact blocks
+	// it is 100 times the default amount of points we use per block
+	AggressiveMaxPointsPerBlock = 100000
+
 	// DefaultMaxSeriesPerDatabase is the maximum number of series a node can hold per database.
 	// This limit only applies to the "inmem" index.
 	DefaultMaxSeriesPerDatabase = 1000000
diff --git a/tsdb/engine/tsm1/compact.go b/tsdb/engine/tsm1/compact.go
index 63eee35f693..7c544e6a0f9 100644
--- a/tsdb/engine/tsm1/compact.go
+++ b/tsdb/engine/tsm1/compact.go
@@ -96,7 +96,9 @@ type CompactionGroup []string
 type CompactionPlanner interface {
 	Plan(lastWrite time.Time) ([]CompactionGroup, int64)
 	PlanLevel(level int) ([]CompactionGroup, int64)
-	PlanOptimize() ([]CompactionGroup, int64, int64)
+	// PlanOptimize will return the groups for compaction, the compaction group length,
+	// and the amount of generations within the compaction group.
+	PlanOptimize() (compactGroup []CompactionGroup, compactionGroupLen int64, generationCount int64)
 	Release(group []CompactionGroup)
 	FullyCompacted() (bool, string)
 
@@ -220,14 +222,32 @@ func (c *DefaultPlanner) ParseFileName(path string) (int, int, error) {
 // FullyCompacted returns true if the shard is fully compacted.
 func (c *DefaultPlanner) FullyCompacted() (bool, string) {
 	gens := c.findGenerations(false)
+
 	if len(gens) > 1 {
 		return false, "not fully compacted and not idle because of more than one generation"
 	} else if gens.hasTombstones() {
 		return false, "not fully compacted and not idle because of tombstones"
 	} else {
-		// Safety: check for first index so we don't accidentally do out of bounds access
-		if len(gens) == 1 && len(gens[0].files) > 1 && gens[0].size() < uint64(maxTSMFileSize) {
-			return false, "not fully compacted and not idle because group size under 2 GB and more then single file"
+		// For planning we want to ensure that if there is a single generation
+		// shard, but it has many files that are under 2 GB and many files that are
+		// not at the aggressive compaction points per block count (100,000) we further
+		// compact the shard. It is okay to stop compaction if there are many
+		// files that are under 2 GB but at the aggressive points per block count.
+		if len(gens) == 1 && len(gens[0].files) > 1 {
+			aggressivePointsPerBlockCount := 0
+			filesUnderMaxTsmSizeCount := 0
+			for _, tsmFile := range gens[0].files {
+				if c.FileStore.BlockCount(tsmFile.Path, 0) == tsdb.AggressiveMaxPointsPerBlock {
+					aggressivePointsPerBlockCount++
+				}
+				if tsmFile.Size < maxTSMFileSize {
+					filesUnderMaxTsmSizeCount++
+				}
+			}
+
+			if filesUnderMaxTsmSizeCount > 1 && aggressivePointsPerBlockCount < len(gens[0].files) {
+				return false, "not fully compacted and not idle because single generation with many files under 2 GB and many files under aggressive compaction points per block count (100,000 points)"
+			}
 		}
 		return true, ""
 	}
@@ -353,20 +373,9 @@ func (c *DefaultPlanner) PlanOptimize() ([]CompactionGroup, int64, int64) {
 	// a generation conceptually as a single file even though it may be
 	// split across several files in sequence.
 	generations := c.findGenerations(true)
+	fullyCompacted, _ := c.FullyCompacted()
 
-	// Safety check for potential 0 generations.
-	if len(generations) < 1 {
-		return nil, 0, 0
-	}
-
-	// Return if there's only a single file and single generation.
-	if len(generations) == 1 && len(generations[0].files) == 1 {
-		return nil, 0, 0
-	}
-
-	// If there is a single generation, no tombstones, and the entire group size (all TSM files in the generation)
-	// is over or equal to 2 GB (Max TSM file size) there is nothing to do and we will return.
-	if len(generations) == 1 && generations[0].size() >= uint64(maxTSMFileSize) && !generations.hasTombstones() {
+	if fullyCompacted {
 		return nil, 0, 0
 	}
 
@@ -464,8 +473,11 @@ func (c *DefaultPlanner) Plan(lastWrite time.Time) ([]CompactionGroup, int64) {
 		for i, group := range generations {
 			var skip bool
 
+			gs := group.size()
+			bc := c.FileStore.BlockCount(group.files[0].Path, 1)
+			gl := len(generations)
 			// Skip the file if it's over the max size and contains a full block and it does not have any tombstones
-			if len(generations) > 2 && group.size() > uint64(maxTSMFileSize) && c.FileStore.BlockCount(group.files[0].Path, 1) == tsdb.DefaultMaxPointsPerBlock && !group.hasTombstones() {
+			if gl > 2 && gs >= uint64(maxTSMFileSize) && bc == tsdb.DefaultMaxPointsPerBlock && !group.hasTombstones() {
 				skip = true
 			}
 
@@ -922,7 +934,7 @@ func (c *Compactor) WriteSnapshot(cache *Cache, logger *zap.Logger) ([]string, e
 // compact writes multiple smaller TSM files into 1 or more larger files.
 func (c *Compactor) compact(fast bool, tsmFiles []string, logger *zap.Logger) ([]string, error) {
 	// Sets the points per block size. The larger this value is set
-	// the more points there will be a single index. Under normal
+	// the more points there will be in a single index. Under normal
 	// conditions this should always be 1000 but there is an edge case
 	// where this is increased.
 	size := c.Size
diff --git a/tsdb/engine/tsm1/compact_test.go b/tsdb/engine/tsm1/compact_test.go
index 2464cee2bc8..a7722f254e2 100644
--- a/tsdb/engine/tsm1/compact_test.go
+++ b/tsdb/engine/tsm1/compact_test.go
@@ -1,6 +1,7 @@
 package tsm1_test
 
 import (
+	"errors"
 	"fmt"
 	"math"
 	"os"
@@ -2330,9 +2331,9 @@ func TestDefaultPlanner_PlanOptimize_LargeMultiGeneration(t *testing.T) {
 	}
 
 	tsm, pLen, _ := cp.PlanOptimize()
-	require.Equal(t, 1, len(tsm), "group length mismatch: got %d, exp %d", len(tsm), 1)
-	require.Equal(t, int64(len(tsm)), pLen, "tsm file plan length mismatch: got %d, exp %d", pLen, int64(len(tsm)))
-	require.Equal(t, len(expFiles), len(tsm[0]), "tsm file length mismatch: got %d, exp %d", len(tsm[0]), len(expFiles))
+	require.Equal(t, 1, len(tsm))
+	require.Equal(t, int64(len(tsm)), pLen)
+	require.Equal(t, len(expFiles), len(tsm[0]))
 }
 
 // This test is added to account for a single generation that has a group size
@@ -2372,15 +2373,14 @@ func TestDefaultPlanner_PlanOptimize_SmallSingleGeneration(t *testing.T) {
 	}
 
 	tsm, pLen, gLen := cp.PlanOptimize()
-	require.Equal(t, 1, len(tsm), "group length mismatch: got %d, exp %d", len(tsm), 1)
-	require.Equal(t, int64(len(tsm)), pLen, "tsm file plan length mismatch: got %d, exp %d", pLen, int64(len(expFiles)))
-	require.Equal(t, int64(1), gLen, "generation length mismatch: got %d, exp %d", gLen, 1)
-	require.Equal(t, len(expFiles), len(tsm[0]), "tsm file length mismatch: got %d, exp %d", len(tsm[0]), expFiles)
+	require.Equal(t, 1, len(tsm))
+	require.Equal(t, int64(len(tsm)), pLen)
+	require.Equal(t, int64(1), gLen)
+	require.Equal(t, len(expFiles), len(tsm[0]))
 }
 
 // This test is added to account for a single generation that has a group size
 // under 2 GB and has less then level 4 files it should be further compacted to a single file.
-// FullyCompacted should NOT skip over opening this shard.
 func TestDefaultPlanner_PlanOptimize_SmallSingleGenerationUnderLevel4(t *testing.T) {
 	// ~650 MB total group size
 	data := []tsm1.FileStat{
@@ -2412,10 +2412,10 @@ func TestDefaultPlanner_PlanOptimize_SmallSingleGenerationUnderLevel4(t *testing
 	}
 
 	tsm, pLen, gLen := cp.PlanOptimize()
-	require.Equal(t, 1, len(tsm), "group length mismatch: got %d, exp %d", len(tsm), 1)
-	require.Equal(t, int64(len(tsm)), pLen, "tsm file plan length mismatch: got %d, exp %d", pLen, int64(len(expFiles)))
-	require.Equal(t, int64(1), gLen, "generation length mismatch: got %d, exp %d", gLen, 1)
-	require.Equal(t, len(expFiles), len(tsm[0]), "tsm file length mismatch: got %d, exp %d", len(tsm[0]), expFiles)
+	require.Equal(t, 1, len(tsm))
+	require.Equal(t, int64(len(tsm)), pLen)
+	require.Equal(t, int64(1), gLen)
+	require.Equal(t, len(expFiles), len(tsm[0]))
 }
 
 // This test is added to account for a single generation that has a group size
@@ -2442,16 +2442,19 @@ func TestDefaultPlanner_FullyCompacted_SmallSingleGeneration(t *testing.T) {
 		},
 	}
 
-	cp := tsm1.NewDefaultPlanner(
-		&fakeFileStore{
-			PathsFn: func() []tsm1.FileStat {
-				return data
-			},
-		}, tsdb.DefaultCompactFullWriteColdDuration,
-	)
+	fs := &fakeFileStore{
+		PathsFn: func() []tsm1.FileStat {
+			return data
+		},
+	}
+
+	err := fs.SetBlockCounts([]int{tsdb.DefaultMaxPointsPerBlock, tsdb.DefaultMaxPointsPerBlock, tsdb.DefaultMaxPointsPerBlock, tsdb.DefaultMaxPointsPerBlock})
+	require.NoError(t, err)
+
+	cp := tsm1.NewDefaultPlanner(fs, tsdb.DefaultCompactFullWriteColdDuration)
 
 	compacted, reason := cp.FullyCompacted()
-	reasonExp := "not fully compacted and not idle because group size under 2 GB and more then single file"
+	reasonExp := "not fully compacted and not idle because single generation with many files under 2 GB and many files under aggressive compaction points per block count (100,000 points)"
 	require.Equal(t, reason, reasonExp)
 	require.Equal(t, false, compacted)
 }
@@ -2482,6 +2485,246 @@ func TestDefaultPlanner_FullyCompacted_SmallSingleGeneration_Halt(t *testing.T)
 	require.Equal(t, true, compacted)
 }
 
+// This test is added to account for a single generation that has a group size
+// under 2 GB so it should be further compacted to a single file.
+// FullyCompacted should NOT skip over opening this shard.
+func TestDefaultPlanner_FullyCompacted_LargeSingleGenerationUnderAggressiveBlocks(t *testing.T) {
+	// > 2 GB total group size
+	// 50% of files are at aggressive max block size
+	data := []tsm1.FileStat{
+		{
+			Path: "01-05.tsm1",
+			Size: 700 * 1024 * 1024,
+		},
+		{
+			Path: "01-06.tsm1",
+			Size: 500 * 1024 * 1024,
+		},
+		{
+			Path: "01-07.tsm1",
+			Size: 400 * 1024 * 1024,
+		},
+		{
+			Path: "01-08.tsm1",
+			Size: 300 * 1024 * 1024,
+		},
+		{
+			Path: "01-09.tsm1",
+			Size: 200 * 1024 * 1024,
+		},
+		{
+			Path: "01-10.tsm1",
+			Size: 100 * 1024 * 1024,
+		},
+		{
+			Path: "01-11.tsm1",
+			Size: 50 * 1024 * 1024,
+		},
+		{
+			Path: "01-12.tsm1",
+			Size: 25 * 1024 * 1024,
+		},
+	}
+
+	fs := &fakeFileStore{
+		PathsFn: func() []tsm1.FileStat {
+			return data
+		},
+	}
+	blocks := []int{
+		tsdb.AggressiveMaxPointsPerBlock,
+		tsdb.AggressiveMaxPointsPerBlock,
+		tsdb.AggressiveMaxPointsPerBlock,
+		tsdb.AggressiveMaxPointsPerBlock,
+		tsdb.DefaultMaxPointsPerBlock,
+		tsdb.DefaultMaxPointsPerBlock,
+		tsdb.DefaultMaxPointsPerBlock,
+		tsdb.DefaultMaxPointsPerBlock,
+	}
+	err := fs.SetBlockCounts(blocks)
+	require.NoError(t, err)
+
+	cp := tsm1.NewDefaultPlanner(fs, tsdb.DefaultCompactFullWriteColdDuration)
+
+	compacted, reason := cp.FullyCompacted()
+	reasonExp := "not fully compacted and not idle because single generation with many files under 2 GB and many files under aggressive compaction points per block count (100,000 points)"
+	require.Equal(t, reason, reasonExp)
+	require.Equal(t, false, compacted)
+}
+
+// This test is added to account for a single generation that has a group size
+// over 2 GB with 1 file under 2 GB all at max points per block with aggressive compaction.
+func TestDefaultPlanner_FullyCompacted_LargeSingleGenerationMaxAggressiveBlocks(t *testing.T) {
+	// > 2 GB total group size
+	// 100% of files are at aggressive max block size
+	data := []tsm1.FileStat{
+		{
+			Path: "01-13.tsm1",
+			Size: 2048 * 1024 * 1024,
+		},
+		{
+			Path: "01-14.tsm1",
+			Size: 691 * 1024 * 1024,
+		},
+	}
+
+	fs := &fakeFileStore{
+		PathsFn: func() []tsm1.FileStat {
+			return data
+		},
+	}
+	blocks := []int{
+		tsdb.AggressiveMaxPointsPerBlock,
+		tsdb.AggressiveMaxPointsPerBlock,
+	}
+	err := fs.SetBlockCounts(blocks)
+	require.NoError(t, err)
+
+	cp := tsm1.NewDefaultPlanner(fs, tsdb.DefaultCompactFullWriteColdDuration)
+
+	compacted, reason := cp.FullyCompacted()
+	reasonExp := ""
+	require.Equal(t, reason, reasonExp)
+	require.Equal(t, true, compacted)
+
+	cgroup, cgLen, genLen := cp.PlanOptimize()
+	require.Equal(t, []tsm1.CompactionGroup(nil), cgroup)
+	require.Equal(t, int64(0), cgLen)
+	require.Equal(t, int64(0), genLen)
+}
+
+// This test is added to account for a single generation that has a group size
+// over 2 GB with 1 file under 2 GB all at max points per block with aggressive compaction.
+func TestDefaultPlanner_FullyCompacted_LargeSingleGenerationNoMaxAggrBlocks(t *testing.T) {
+	// > 2 GB total group size
+	// 100% of files are at aggressive max block size
+	data := []tsm1.FileStat{
+		{
+			Path: "01-13.tsm1",
+			Size: 2048 * 1024 * 1024,
+		},
+		{
+			Path: "01-14.tsm1",
+			Size: 691 * 1024 * 1024,
+		},
+	}
+
+	fs := &fakeFileStore{
+		PathsFn: func() []tsm1.FileStat {
+			return data
+		},
+	}
+	blocks := []int{
+		tsdb.AggressiveMaxPointsPerBlock,
+		tsdb.DefaultMaxPointsPerBlock,
+	}
+	err := fs.SetBlockCounts(blocks)
+	require.NoError(t, err)
+
+	cp := tsm1.NewDefaultPlanner(fs, tsdb.DefaultCompactFullWriteColdDuration)
+
+	compacted, reason := cp.FullyCompacted()
+	reasonExp := ""
+	require.Equal(t, reason, reasonExp)
+	require.Equal(t, true, compacted)
+
+	cgroup, cgLen, genLen := cp.PlanOptimize()
+	require.Equal(t, []tsm1.CompactionGroup(nil), cgroup)
+	require.Equal(t, int64(0), cgLen)
+	require.Equal(t, int64(0), genLen)
+}
+
+// This test is added to account for a single generation that has a group size
+// over 2 GB and multiple files under 2 GB all at max points per block for aggressive compaction.
+func TestDefaultPlanner_FullyCompacted_ManySingleGenLessThen2GBMaxAggrBlocks(t *testing.T) {
+	// > 2 GB total group size
+	// 100% of files are at aggressive max block size
+	data := []tsm1.FileStat{
+		{
+			Path: "01-13.tsm1",
+			Size: 2048 * 1024 * 1024,
+		},
+		{
+			Path: "01-14.tsm1",
+			Size: 650 * 1024 * 1024,
+		},
+		{
+			Path: "01-15.tsm1",
+			Size: 450 * 1024 * 1024,
+		},
+	}
+
+	fs := &fakeFileStore{
+		PathsFn: func() []tsm1.FileStat {
+			return data
+		},
+	}
+	blocks := []int{
+		tsdb.AggressiveMaxPointsPerBlock,
+		tsdb.AggressiveMaxPointsPerBlock,
+		tsdb.AggressiveMaxPointsPerBlock,
+	}
+	err := fs.SetBlockCounts(blocks)
+	require.NoError(t, err)
+
+	cp := tsm1.NewDefaultPlanner(fs, tsdb.DefaultCompactFullWriteColdDuration)
+
+	compacted, reason := cp.FullyCompacted()
+	reasonExp := ""
+	require.Equal(t, reason, reasonExp)
+	require.Equal(t, true, compacted)
+
+	cgroup, cgLen, genLen := cp.PlanOptimize()
+	require.Equal(t, []tsm1.CompactionGroup(nil), cgroup)
+	require.Equal(t, int64(0), cgLen)
+	require.Equal(t, int64(0), genLen)
+}
+
+// This test is added to account for a single generation that has a group size
+// over 2 GB and multiple files under 2 GB all at max points per block for aggressive compaction.
+func TestDefaultPlanner_FullyCompacted_ManySingleGenLessThen2GBNotMaxAggrBlocks(t *testing.T) {
+	// > 2 GB total group size
+	// 100% of files are at aggressive max block size
+	data := []tsm1.FileStat{
+		{
+			Path: "01-13.tsm1",
+			Size: 2048 * 1024 * 1024,
+		},
+		{
+			Path: "01-14.tsm1",
+			Size: 650 * 1024 * 1024,
+		},
+		{
+			Path: "01-15.tsm1",
+			Size: 450 * 1024 * 1024,
+		},
+	}
+
+	fs := &fakeFileStore{
+		PathsFn: func() []tsm1.FileStat {
+			return data
+		},
+	}
+	blocks := []int{
+		tsdb.AggressiveMaxPointsPerBlock,
+		tsdb.DefaultMaxPointsPerBlock,
+		tsdb.DefaultMaxPointsPerBlock,
+	}
+	err := fs.SetBlockCounts(blocks)
+	require.NoError(t, err)
+
+	cp := tsm1.NewDefaultPlanner(fs, tsdb.DefaultCompactFullWriteColdDuration)
+
+	compacted, reason := cp.FullyCompacted()
+	reasonExp := "not fully compacted and not idle because single generation with many files under 2 GB and many files under aggressive compaction points per block count (100,000 points)"
+	require.Equal(t, reason, reasonExp)
+	require.Equal(t, false, compacted)
+
+	_, cgLen, genLen := cp.PlanOptimize()
+	require.Equal(t, int64(1), cgLen)
+	require.Equal(t, int64(1), genLen)
+}
+
 func TestDefaultPlanner_PlanOptimize_Multiple(t *testing.T) {
 	data := []tsm1.FileStat{
 		{
@@ -2565,39 +2808,6 @@ func TestDefaultPlanner_PlanOptimize_Multiple(t *testing.T) {
 	}
 }
 
-func TestDefaultPlanner_PlanOptimize_Optimized(t *testing.T) {
-	data := []tsm1.FileStat{
-		{
-			Path: "01-03.tsm1",
-			Size: 251 * 1024 * 1024,
-		},
-		{
-			Path: "01-04.tsm1",
-			Size: 1 * 1024 * 1024,
-		},
-		{
-			Path: "01-05.tsm1",
-			Size: 2 * 1024 * 1024 * 1024,
-		},
-	}
-
-	cp := tsm1.NewDefaultPlanner(
-		&fakeFileStore{
-			PathsFn: func() []tsm1.FileStat {
-				return data
-			},
-		}, tsdb.DefaultCompactFullWriteColdDuration,
-	)
-
-	expFiles := []tsm1.FileStat{}
-	tsm, pLen, _ := cp.PlanOptimize()
-	if exp, got := len(expFiles), len(tsm); got != exp {
-		t.Fatalf("tsm file length mismatch: got %v, exp %v", got, exp)
-	} else if pLen != int64(len(tsm)) {
-		t.Fatalf("tsm file plan length mismatch: got %v, exp %v", pLen, exp)
-	}
-}
-
 func TestDefaultPlanner_PlanOptimize_Tombstones(t *testing.T) {
 	data := []tsm1.FileStat{
 		{
@@ -2748,9 +2958,11 @@ func TestDefaultPlanner_Plan_SkipPlanningAfterFull(t *testing.T) {
 		PathsFn: func() []tsm1.FileStat {
 			return testSet
 		},
-		blockCount: 1000,
 	}
 
+	err := fs.SetBlockCounts([]int{tsdb.DefaultMaxPointsPerBlock, tsdb.DefaultMaxPointsPerBlock, tsdb.DefaultMaxPointsPerBlock, tsdb.DefaultMaxPointsPerBlock})
+	require.NoError(t, err)
+
 	cp := tsm1.NewDefaultPlanner(fs, time.Nanosecond)
 	plan, pLen := cp.Plan(time.Now().Add(-time.Second))
 	// first verify that our test set would return files
@@ -2789,9 +3001,17 @@ func TestDefaultPlanner_Plan_SkipPlanningAfterFull(t *testing.T) {
 		PathsFn: func() []tsm1.FileStat {
 			return over
 		},
-		blockCount: 1000,
 	}
 
+	err = overFs.SetBlockCounts([]int{
+		tsdb.DefaultMaxPointsPerBlock,
+		tsdb.DefaultMaxPointsPerBlock,
+		tsdb.DefaultMaxPointsPerBlock,
+		tsdb.DefaultMaxPointsPerBlock,
+		tsdb.DefaultMaxPointsPerBlock,
+	})
+	require.NoError(t, err)
+
 	cp.FileStore = overFs
 	plan, pLen = cp.Plan(time.Now().Add(-time.Second))
 	if exp, got := 0, len(plan); got != exp {
@@ -2869,14 +3089,21 @@ func TestDefaultPlanner_Plan_TwoGenLevel3(t *testing.T) {
 		},
 	}
 
-	cp := tsm1.NewDefaultPlanner(
-		&fakeFileStore{
-			blockCount: 1000,
-			PathsFn: func() []tsm1.FileStat {
-				return data
-			},
+	fs := &fakeFileStore{
+		PathsFn: func() []tsm1.FileStat {
+			return data
 		},
-		time.Hour)
+	}
+
+	var bcs []int
+	for range data {
+		bcs = append(bcs, tsdb.DefaultMaxPointsPerBlock)
+	}
+
+	err := fs.SetBlockCounts(bcs)
+	require.NoError(t, err)
+
+	cp := tsm1.NewDefaultPlanner(fs, time.Hour)
 
 	tsm, pLen := cp.Plan(time.Now().Add(-24 * time.Hour))
 	if exp, got := 1, len(tsm); got != exp {
@@ -2912,9 +3139,16 @@ func TestDefaultPlanner_Plan_NotFullOverMaxsize(t *testing.T) {
 		PathsFn: func() []tsm1.FileStat {
 			return testSet
 		},
-		blockCount: 100,
 	}
 
+	var bcs []int
+	for range testSet {
+		bcs = append(bcs, 100)
+	}
+
+	err := fs.SetBlockCounts(bcs)
+	require.NoError(t, err)
+
 	cp := tsm1.NewDefaultPlanner(
 		fs,
 		time.Nanosecond,
@@ -2945,9 +3179,16 @@ func TestDefaultPlanner_Plan_NotFullOverMaxsize(t *testing.T) {
 		PathsFn: func() []tsm1.FileStat {
 			return over
 		},
-		blockCount: 100,
 	}
 
+	bcs = make([]int, 0)
+	for range over {
+		bcs = append(bcs, 100)
+	}
+
+	err = overFs.SetBlockCounts(bcs)
+	require.NoError(t, err)
+
 	cp.FileStore = overFs
 	cGroups, pLen := cp.Plan(time.Now().Add(-time.Second))
 	if exp, got := 1, len(cGroups); got != exp {
@@ -3256,8 +3497,10 @@ func MustOpenTSMReader(name string) *tsm1.TSMReader {
 type fakeFileStore struct {
 	PathsFn      func() []tsm1.FileStat
 	lastModified time.Time
-	blockCount   int
-	readers      []*tsm1.TSMReader
+	// fakeFileStore blockCount holds a map of file paths from
+	// PathsFn.FileStat to a mock block count as an integer.
+	blockCount map[string]int
+	readers    []*tsm1.TSMReader
 }
 
 func (w *fakeFileStore) Stats() []tsm1.FileStat {
@@ -3272,8 +3515,30 @@ func (w *fakeFileStore) LastModified() time.Time {
 	return w.lastModified
 }
 
+// Utility function to set mock block counts on a TSM file basis
+// If the number of inputs supplied is less then the amount of TSM
+// files in a given PathsFn it will error.
+func (w *fakeFileStore) SetBlockCounts(inputs []int) error {
+	if len(inputs) != len(w.PathsFn()) {
+		return errors.New("inputs []int length does not equal length of PathsFn()")
+	}
+
+	bc := make(map[string]int)
+	for i, f := range w.PathsFn() {
+		bc[f.Path] = inputs[i]
+	}
+
+	w.blockCount = bc
+	return nil
+}
+
 func (w *fakeFileStore) BlockCount(path string, idx int) int {
-	return w.blockCount
+	for _, f := range w.PathsFn() {
+		if f.Path == path {
+			return w.blockCount[path]
+		}
+	}
+	return 0
 }
 
 func (w *fakeFileStore) TSMReader(path string) (*tsm1.TSMReader, error) {
diff --git a/tsdb/engine/tsm1/engine.go b/tsdb/engine/tsm1/engine.go
index 4e1d7defeb7..7c50be10894 100644
--- a/tsdb/engine/tsm1/engine.go
+++ b/tsdb/engine/tsm1/engine.go
@@ -2174,7 +2174,7 @@ func (e *Engine) compact(wg *sync.WaitGroup) {
 						for _, f := range level4Groups[0] {
 							e.logger.Info("TSM optimized compaction on single generation running, increasing total points per block to 100_000.", zap.String("path", f))
 						}
-						e.Compactor.Size = tsdb.DefaultMaxPointsPerBlock * 100
+						e.Compactor.Size = tsdb.AggressiveMaxPointsPerBlock
 					} else {
 						e.Compactor.Size = tsdb.DefaultMaxPointsPerBlock
 					}

From 827e85962da381cfd9bab38cfa88ced4fb03d8d0 Mon Sep 17 00:00:00 2001
From: Devan <devandbenz@gmail.com>
Date: Tue, 17 Dec 2024 15:09:22 -0600
Subject: [PATCH 03/31] feat: Use named variables for PlanOptimize

---
 tsdb/engine/tsm1/compact.go | 13 +++++--------
 1 file changed, 5 insertions(+), 8 deletions(-)

diff --git a/tsdb/engine/tsm1/compact.go b/tsdb/engine/tsm1/compact.go
index 7c544e6a0f9..9c91ea8a9bb 100644
--- a/tsdb/engine/tsm1/compact.go
+++ b/tsdb/engine/tsm1/compact.go
@@ -98,6 +98,10 @@ type CompactionPlanner interface {
 	PlanLevel(level int) ([]CompactionGroup, int64)
 	// PlanOptimize will return the groups for compaction, the compaction group length,
 	// and the amount of generations within the compaction group.
+	// generationCount needs to be set to decide how many points per block during compaction.
+	// This value is mostly ignored in normal compaction code paths, but,
+	// for the edge case where there is a single generation with many
+	// files under 2 GB this value is an important indicator.
 	PlanOptimize() (compactGroup []CompactionGroup, compactionGroupLen int64, generationCount int64)
 	Release(group []CompactionGroup)
 	FullyCompacted() (bool, string)
@@ -359,7 +363,7 @@ func (c *DefaultPlanner) PlanLevel(level int) ([]CompactionGroup, int64) {
 // PlanOptimize returns all TSM files if they are in different generations in order
 // to optimize the index across TSM files.  Each returned compaction group can be
 // compacted concurrently.
-func (c *DefaultPlanner) PlanOptimize() ([]CompactionGroup, int64, int64) {
+func (c *DefaultPlanner) PlanOptimize() (compactGroup []CompactionGroup, compactionGroupLen int64, generationCount int64) {
 	// If a full plan has been requested, don't plan any levels which will prevent
 	// the full plan from acquiring them.
 	c.mu.RLock()
@@ -435,13 +439,6 @@ func (c *DefaultPlanner) PlanOptimize() ([]CompactionGroup, int64, int64) {
 		cGroups = append(cGroups, cGroup)
 	}
 
-	// The third return value is the generation length.
-	// Need to use this to decide how many points per block to use during compaction.
-	// This value is mostly ignored in normal compaction code paths, but,
-	// for the edge case where there is a single generation with many
-	// files and a group size under 2 GB we need to know that there is a single
-	// generation so the block size can be adjusted to allow for more optimal
-	// compaction.
 	if !c.acquire(cGroups) {
 		return nil, int64(len(cGroups)), int64(len(generations))
 	}

From 5387ca383725db487e1f7677f7f9117c6ae1aedd Mon Sep 17 00:00:00 2001
From: Devan <devandbenz@gmail.com>
Date: Tue, 17 Dec 2024 15:16:46 -0600
Subject: [PATCH 04/31] feat: adjust test comments

---
 tsdb/engine/tsm1/compact_test.go | 16 ++++++++++------
 1 file changed, 10 insertions(+), 6 deletions(-)

diff --git a/tsdb/engine/tsm1/compact_test.go b/tsdb/engine/tsm1/compact_test.go
index a7722f254e2..c6fb0a5f4e3 100644
--- a/tsdb/engine/tsm1/compact_test.go
+++ b/tsdb/engine/tsm1/compact_test.go
@@ -2419,8 +2419,8 @@ func TestDefaultPlanner_PlanOptimize_SmallSingleGenerationUnderLevel4(t *testing
 }
 
 // This test is added to account for a single generation that has a group size
-// under 2 GB so it should be further compacted to a single file.
-// FullyCompacted should NOT skip over opening this shard.
+// under 2 GB and all files at max default points per block of 1000.
+// This should be planned for compaction at a more aggressive points per block.
 func TestDefaultPlanner_FullyCompacted_SmallSingleGeneration(t *testing.T) {
 	// ~650 MB total group size
 	data := []tsm1.FileStat{
@@ -2486,8 +2486,8 @@ func TestDefaultPlanner_FullyCompacted_SmallSingleGeneration_Halt(t *testing.T)
 }
 
 // This test is added to account for a single generation that has a group size
-// under 2 GB so it should be further compacted to a single file.
-// FullyCompacted should NOT skip over opening this shard.
+// under 2 GB and a mix of aggressive max blocks and default max blocks
+// it should be further compacted.
 func TestDefaultPlanner_FullyCompacted_LargeSingleGenerationUnderAggressiveBlocks(t *testing.T) {
 	// > 2 GB total group size
 	// 50% of files are at aggressive max block size
@@ -2554,6 +2554,7 @@ func TestDefaultPlanner_FullyCompacted_LargeSingleGenerationUnderAggressiveBlock
 
 // This test is added to account for a single generation that has a group size
 // over 2 GB with 1 file under 2 GB all at max points per block with aggressive compaction.
+// It should not compact any further.
 func TestDefaultPlanner_FullyCompacted_LargeSingleGenerationMaxAggressiveBlocks(t *testing.T) {
 	// > 2 GB total group size
 	// 100% of files are at aggressive max block size
@@ -2594,7 +2595,9 @@ func TestDefaultPlanner_FullyCompacted_LargeSingleGenerationMaxAggressiveBlocks(
 }
 
 // This test is added to account for a single generation that has a group size
-// over 2 GB with 1 file under 2 GB all at max points per block with aggressive compaction.
+// over 2 GB at max points per block with aggressive compaction, and, 1 file
+// under 2 GB at default max points per block.
+// It should not compact any further.
 func TestDefaultPlanner_FullyCompacted_LargeSingleGenerationNoMaxAggrBlocks(t *testing.T) {
 	// > 2 GB total group size
 	// 100% of files are at aggressive max block size
@@ -2681,7 +2684,8 @@ func TestDefaultPlanner_FullyCompacted_ManySingleGenLessThen2GBMaxAggrBlocks(t *
 }
 
 // This test is added to account for a single generation that has a group size
-// over 2 GB and multiple files under 2 GB all at max points per block for aggressive compaction.
+// over 2 GB and multiple files under 2 GB with multiple files under aggressive
+// max points per block. This should further compact.
 func TestDefaultPlanner_FullyCompacted_ManySingleGenLessThen2GBNotMaxAggrBlocks(t *testing.T) {
 	// > 2 GB total group size
 	// 100% of files are at aggressive max block size

From 31535963e18c32dc6181031816ecb5a687a5241d Mon Sep 17 00:00:00 2001
From: Devan <devandbenz@gmail.com>
Date: Tue, 17 Dec 2024 15:18:39 -0600
Subject: [PATCH 05/31] feat: code removal from debugging

---
 tsdb/engine/tsm1/compact.go | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/tsdb/engine/tsm1/compact.go b/tsdb/engine/tsm1/compact.go
index 9c91ea8a9bb..1ccde947e50 100644
--- a/tsdb/engine/tsm1/compact.go
+++ b/tsdb/engine/tsm1/compact.go
@@ -470,11 +470,8 @@ func (c *DefaultPlanner) Plan(lastWrite time.Time) ([]CompactionGroup, int64) {
 		for i, group := range generations {
 			var skip bool
 
-			gs := group.size()
-			bc := c.FileStore.BlockCount(group.files[0].Path, 1)
-			gl := len(generations)
 			// Skip the file if it's over the max size and contains a full block and it does not have any tombstones
-			if gl > 2 && gs >= uint64(maxTSMFileSize) && bc == tsdb.DefaultMaxPointsPerBlock && !group.hasTombstones() {
+			if len(generations) > 2 && group.size() > uint64(maxTSMFileSize) && c.FileStore.BlockCount(group.files[0].Path, 1) == tsdb.DefaultMaxPointsPerBlock && !group.hasTombstones() {
 				skip = true
 			}
 

From 83d28ec079af7f73afd6509b4e9545a07410c166 Mon Sep 17 00:00:00 2001
From: Devan <devandbenz@gmail.com>
Date: Tue, 17 Dec 2024 15:29:05 -0600
Subject: [PATCH 06/31] feat: setting BlockCount idx value to 1

---
 tsdb/engine/tsm1/compact.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tsdb/engine/tsm1/compact.go b/tsdb/engine/tsm1/compact.go
index 1ccde947e50..9e0e4fe9804 100644
--- a/tsdb/engine/tsm1/compact.go
+++ b/tsdb/engine/tsm1/compact.go
@@ -241,7 +241,7 @@ func (c *DefaultPlanner) FullyCompacted() (bool, string) {
 			aggressivePointsPerBlockCount := 0
 			filesUnderMaxTsmSizeCount := 0
 			for _, tsmFile := range gens[0].files {
-				if c.FileStore.BlockCount(tsmFile.Path, 0) == tsdb.AggressiveMaxPointsPerBlock {
+				if c.FileStore.BlockCount(tsmFile.Path, 1) == tsdb.AggressiveMaxPointsPerBlock {
 					aggressivePointsPerBlockCount++
 				}
 				if tsmFile.Size < maxTSMFileSize {

From f896a01ec09abd0060e160b36341e644ba6acf61 Mon Sep 17 00:00:00 2001
From: Devan <devandbenz@gmail.com>
Date: Wed, 18 Dec 2024 09:41:18 -0600
Subject: [PATCH 07/31] feat: Adjust testing and add sprintf for magic vars

---
 tsdb/config.go                   |   5 +-
 tsdb/engine/tsm1/compact.go      |  17 ++---
 tsdb/engine/tsm1/compact_test.go | 110 +++++++++++++++++--------------
 3 files changed, 73 insertions(+), 59 deletions(-)

diff --git a/tsdb/config.go b/tsdb/config.go
index a58e1680528..0dbb0e4228a 100644
--- a/tsdb/config.go
+++ b/tsdb/config.go
@@ -54,7 +54,7 @@ const (
 
 	// AggressiveMaxPointsPerBlock is used when we want to further compact blocks
 	// it is 100 times the default amount of points we use per block
-	AggressiveMaxPointsPerBlock = 100000
+	AggressiveMaxPointsPerBlock = DefaultMaxPointsPerBlock * 100
 
 	// DefaultMaxSeriesPerDatabase is the maximum number of series a node can hold per database.
 	// This limit only applies to the "inmem" index.
@@ -81,6 +81,9 @@ const (
 	// partition snapshot compactions that can run at one time.
 	// A value of 0 results in runtime.GOMAXPROCS(0).
 	DefaultSeriesFileMaxConcurrentSnapshotCompactions = 0
+
+	// MaxTSMFileSize is the maximum size of TSM files.
+	MaxTSMFileSize = uint32(2048 * 1024 * 1024) // 2GB
 )
 
 // Config holds the configuration for the tsbd package.
diff --git a/tsdb/engine/tsm1/compact.go b/tsdb/engine/tsm1/compact.go
index 9e0e4fe9804..730b2d4ba17 100644
--- a/tsdb/engine/tsm1/compact.go
+++ b/tsdb/engine/tsm1/compact.go
@@ -32,7 +32,6 @@ import (
 	"go.uber.org/zap"
 )
 
-const maxTSMFileSize = uint32(2048 * 1024 * 1024) // 2GB
 const logEvery = 2 * DefaultSegmentSize
 
 const (
@@ -226,6 +225,8 @@ func (c *DefaultPlanner) ParseFileName(path string) (int, int, error) {
 // FullyCompacted returns true if the shard is fully compacted.
 func (c *DefaultPlanner) FullyCompacted() (bool, string) {
 	gens := c.findGenerations(false)
+	// 1048576000 is a magic number for bytes per gigabyte
+	singleGenReason := fmt.Sprintf("not fully compacted and not idle because single generation with many files under %d GB and many files under aggressive compaction points per block count (%d points)", int(tsdb.MaxTSMFileSize/1048576000), tsdb.AggressiveMaxPointsPerBlock)
 
 	if len(gens) > 1 {
 		return false, "not fully compacted and not idle because of more than one generation"
@@ -241,16 +242,16 @@ func (c *DefaultPlanner) FullyCompacted() (bool, string) {
 			aggressivePointsPerBlockCount := 0
 			filesUnderMaxTsmSizeCount := 0
 			for _, tsmFile := range gens[0].files {
-				if c.FileStore.BlockCount(tsmFile.Path, 1) == tsdb.AggressiveMaxPointsPerBlock {
+				if c.FileStore.BlockCount(tsmFile.Path, 1) >= tsdb.AggressiveMaxPointsPerBlock {
 					aggressivePointsPerBlockCount++
 				}
-				if tsmFile.Size < maxTSMFileSize {
+				if tsmFile.Size < tsdb.MaxTSMFileSize {
 					filesUnderMaxTsmSizeCount++
 				}
 			}
 
 			if filesUnderMaxTsmSizeCount > 1 && aggressivePointsPerBlockCount < len(gens[0].files) {
-				return false, "not fully compacted and not idle because single generation with many files under 2 GB and many files under aggressive compaction points per block count (100,000 points)"
+				return false, singleGenReason
 			}
 		}
 		return true, ""
@@ -471,7 +472,7 @@ func (c *DefaultPlanner) Plan(lastWrite time.Time) ([]CompactionGroup, int64) {
 			var skip bool
 
 			// Skip the file if it's over the max size and contains a full block and it does not have any tombstones
-			if len(generations) > 2 && group.size() > uint64(maxTSMFileSize) && c.FileStore.BlockCount(group.files[0].Path, 1) == tsdb.DefaultMaxPointsPerBlock && !group.hasTombstones() {
+			if len(generations) > 2 && group.size() > uint64(tsdb.MaxTSMFileSize) && c.FileStore.BlockCount(group.files[0].Path, 1) == tsdb.DefaultMaxPointsPerBlock && !group.hasTombstones() {
 				skip = true
 			}
 
@@ -547,7 +548,7 @@ func (c *DefaultPlanner) Plan(lastWrite time.Time) ([]CompactionGroup, int64) {
 		// Skip the file if it's over the max size and contains a full block or the generation is split
 		// over multiple files.  In the latter case, that would mean the data in the file spilled over
 		// the 2GB limit.
-		if g.size() > uint64(maxTSMFileSize) && c.FileStore.BlockCount(g.files[0].Path, 1) == tsdb.DefaultMaxPointsPerBlock {
+		if g.size() > uint64(tsdb.MaxTSMFileSize) && c.FileStore.BlockCount(g.files[0].Path, 1) == tsdb.DefaultMaxPointsPerBlock {
 			start = i + 1
 		}
 
@@ -591,7 +592,7 @@ func (c *DefaultPlanner) Plan(lastWrite time.Time) ([]CompactionGroup, int64) {
 			}
 
 			// Skip the file if it's over the max size and it contains a full block
-			if gen.size() >= uint64(maxTSMFileSize) && c.FileStore.BlockCount(gen.files[0].Path, 1) == tsdb.DefaultMaxPointsPerBlock && !gen.hasTombstones() {
+			if gen.size() >= uint64(tsdb.MaxTSMFileSize) && c.FileStore.BlockCount(gen.files[0].Path, 1) == tsdb.DefaultMaxPointsPerBlock && !gen.hasTombstones() {
 				startIndex++
 				continue
 			}
@@ -1216,7 +1217,7 @@ func (c *Compactor) write(path string, iter KeyIterator, throttle bool, logger *
 
 		// If we have a max file size configured and we're over it, close out the file
 		// and return the error.
-		if w.Size() > maxTSMFileSize {
+		if w.Size() > tsdb.MaxTSMFileSize {
 			if err := w.WriteIndex(); err != nil {
 				return err
 			}
diff --git a/tsdb/engine/tsm1/compact_test.go b/tsdb/engine/tsm1/compact_test.go
index c6fb0a5f4e3..5e9d6c7e438 100644
--- a/tsdb/engine/tsm1/compact_test.go
+++ b/tsdb/engine/tsm1/compact_test.go
@@ -2331,9 +2331,9 @@ func TestDefaultPlanner_PlanOptimize_LargeMultiGeneration(t *testing.T) {
 	}
 
 	tsm, pLen, _ := cp.PlanOptimize()
-	require.Equal(t, 1, len(tsm))
-	require.Equal(t, int64(len(tsm)), pLen)
-	require.Equal(t, len(expFiles), len(tsm[0]))
+	require.Equal(t, 1, len(tsm), "compaction group")
+	require.Equal(t, int64(len(tsm)), pLen, "compaction group length")
+	require.Equal(t, len(expFiles), len(tsm[0]), "expected TSM files")
 }
 
 // This test is added to account for a single generation that has a group size
@@ -2373,10 +2373,10 @@ func TestDefaultPlanner_PlanOptimize_SmallSingleGeneration(t *testing.T) {
 	}
 
 	tsm, pLen, gLen := cp.PlanOptimize()
-	require.Equal(t, 1, len(tsm))
-	require.Equal(t, int64(len(tsm)), pLen)
-	require.Equal(t, int64(1), gLen)
-	require.Equal(t, len(expFiles), len(tsm[0]))
+	require.Equal(t, 1, len(tsm), "compaction group")
+	require.Equal(t, int64(len(tsm)), pLen, "compaction group length")
+	require.Equal(t, int64(1), gLen, "generation of TSM files")
+	require.Equal(t, len(expFiles), len(tsm[0]), "expected TSM files")
 }
 
 // This test is added to account for a single generation that has a group size
@@ -2410,12 +2410,15 @@ func TestDefaultPlanner_PlanOptimize_SmallSingleGenerationUnderLevel4(t *testing
 	for _, file := range data {
 		expFiles = append(expFiles, file)
 	}
+	tsmP, pLenP := cp.Plan(time.Now().Add(-time.Second))
+	require.Equal(t, 0, len(tsmP), "compaction group; Plan()")
+	require.Equal(t, 0, pLenP, "compaction group length; Plan()")
 
 	tsm, pLen, gLen := cp.PlanOptimize()
-	require.Equal(t, 1, len(tsm))
-	require.Equal(t, int64(len(tsm)), pLen)
-	require.Equal(t, int64(1), gLen)
-	require.Equal(t, len(expFiles), len(tsm[0]))
+	require.Equal(t, 1, len(tsm), "compaction group")
+	require.Equal(t, int64(len(tsm)), pLen, "compaction group length")
+	require.Equal(t, int64(1), gLen, "generation of TSM files")
+	require.Equal(t, len(expFiles), len(tsm[0]), "expected TSM files")
 }
 
 // This test is added to account for a single generation that has a group size
@@ -2449,14 +2452,16 @@ func TestDefaultPlanner_FullyCompacted_SmallSingleGeneration(t *testing.T) {
 	}
 
 	err := fs.SetBlockCounts([]int{tsdb.DefaultMaxPointsPerBlock, tsdb.DefaultMaxPointsPerBlock, tsdb.DefaultMaxPointsPerBlock, tsdb.DefaultMaxPointsPerBlock})
-	require.NoError(t, err)
+	require.NoError(t, err, "SetBlockCounts")
 
 	cp := tsm1.NewDefaultPlanner(fs, tsdb.DefaultCompactFullWriteColdDuration)
 
+	// 1048576000 is a magic number for bytes per gigabyte
+	reasonExp := fmt.Sprintf("not fully compacted and not idle because single generation with many files under %d GB and many files under aggressive compaction points per block count (%d points)", int(tsdb.MaxTSMFileSize/1048576000), tsdb.AggressiveMaxPointsPerBlock)
+
 	compacted, reason := cp.FullyCompacted()
-	reasonExp := "not fully compacted and not idle because single generation with many files under 2 GB and many files under aggressive compaction points per block count (100,000 points)"
-	require.Equal(t, reason, reasonExp)
-	require.Equal(t, false, compacted)
+	require.Equal(t, reason, reasonExp, "fullyCompacted reason")
+	require.False(t, compacted, "is fully compacted")
 }
 
 // This test is added to account for halting state after
@@ -2481,8 +2486,8 @@ func TestDefaultPlanner_FullyCompacted_SmallSingleGeneration_Halt(t *testing.T)
 
 	compacted, reason := cp.FullyCompacted()
 	reasonExp := ""
-	require.Equal(t, reason, reasonExp)
-	require.Equal(t, true, compacted)
+	require.Equal(t, reason, reasonExp, "fullyCompacted reason")
+	require.True(t, compacted, "is fully compacted")
 }
 
 // This test is added to account for a single generation that has a group size
@@ -2542,14 +2547,17 @@ func TestDefaultPlanner_FullyCompacted_LargeSingleGenerationUnderAggressiveBlock
 		tsdb.DefaultMaxPointsPerBlock,
 	}
 	err := fs.SetBlockCounts(blocks)
-	require.NoError(t, err)
+	require.NoError(t, err, "SetBlockCounts")
 
 	cp := tsm1.NewDefaultPlanner(fs, tsdb.DefaultCompactFullWriteColdDuration)
 
+	// 1048576000 is a magic number for bytes per gigabyte
+	reasonExp := fmt.Sprintf("not fully compacted and not idle because single generation with many files under %d GB and many files under aggressive compaction points per block count (%d points)", int(tsdb.MaxTSMFileSize/1048576000), tsdb.AggressiveMaxPointsPerBlock)
+
 	compacted, reason := cp.FullyCompacted()
-	reasonExp := "not fully compacted and not idle because single generation with many files under 2 GB and many files under aggressive compaction points per block count (100,000 points)"
-	require.Equal(t, reason, reasonExp)
-	require.Equal(t, false, compacted)
+	require.Equal(t, reason, reasonExp, "fullyCompacted reason")
+	require.False(t, compacted, "is fully compacted")
+
 }
 
 // This test is added to account for a single generation that has a group size
@@ -2579,19 +2587,19 @@ func TestDefaultPlanner_FullyCompacted_LargeSingleGenerationMaxAggressiveBlocks(
 		tsdb.AggressiveMaxPointsPerBlock,
 	}
 	err := fs.SetBlockCounts(blocks)
-	require.NoError(t, err)
+	require.NoError(t, err, "SetBlockCounts")
 
 	cp := tsm1.NewDefaultPlanner(fs, tsdb.DefaultCompactFullWriteColdDuration)
 
 	compacted, reason := cp.FullyCompacted()
 	reasonExp := ""
-	require.Equal(t, reason, reasonExp)
-	require.Equal(t, true, compacted)
+	require.Equal(t, reason, reasonExp, "fullyCompacted reason")
+	require.True(t, compacted, "is fully compacted")
 
 	cgroup, cgLen, genLen := cp.PlanOptimize()
-	require.Equal(t, []tsm1.CompactionGroup(nil), cgroup)
-	require.Equal(t, int64(0), cgLen)
-	require.Equal(t, int64(0), genLen)
+	require.Equal(t, []tsm1.CompactionGroup(nil), cgroup, "compaction group")
+	require.Equal(t, int64(0), cgLen, "compaction group length")
+	require.Equal(t, int64(0), genLen, "generation count")
 }
 
 // This test is added to account for a single generation that has a group size
@@ -2622,19 +2630,19 @@ func TestDefaultPlanner_FullyCompacted_LargeSingleGenerationNoMaxAggrBlocks(t *t
 		tsdb.DefaultMaxPointsPerBlock,
 	}
 	err := fs.SetBlockCounts(blocks)
-	require.NoError(t, err)
+	require.NoError(t, err, "SetBlockCounts")
 
 	cp := tsm1.NewDefaultPlanner(fs, tsdb.DefaultCompactFullWriteColdDuration)
 
 	compacted, reason := cp.FullyCompacted()
 	reasonExp := ""
-	require.Equal(t, reason, reasonExp)
-	require.Equal(t, true, compacted)
+	require.Equal(t, reason, reasonExp, "fullyCompacted reason")
+	require.True(t, compacted, "is fully compacted")
 
 	cgroup, cgLen, genLen := cp.PlanOptimize()
-	require.Equal(t, []tsm1.CompactionGroup(nil), cgroup)
-	require.Equal(t, int64(0), cgLen)
-	require.Equal(t, int64(0), genLen)
+	require.Equal(t, []tsm1.CompactionGroup(nil), cgroup, "compaction group")
+	require.Equal(t, int64(0), cgLen, "compaction group length")
+	require.Equal(t, int64(0), genLen, "generation count")
 }
 
 // This test is added to account for a single generation that has a group size
@@ -2668,19 +2676,19 @@ func TestDefaultPlanner_FullyCompacted_ManySingleGenLessThen2GBMaxAggrBlocks(t *
 		tsdb.AggressiveMaxPointsPerBlock,
 	}
 	err := fs.SetBlockCounts(blocks)
-	require.NoError(t, err)
+	require.NoError(t, err, "SetBlockCounts")
 
 	cp := tsm1.NewDefaultPlanner(fs, tsdb.DefaultCompactFullWriteColdDuration)
 
 	compacted, reason := cp.FullyCompacted()
 	reasonExp := ""
-	require.Equal(t, reason, reasonExp)
-	require.Equal(t, true, compacted)
+	require.Equal(t, reason, reasonExp, "fullyCompacted reason")
+	require.True(t, compacted, "is fully compacted")
 
 	cgroup, cgLen, genLen := cp.PlanOptimize()
-	require.Equal(t, []tsm1.CompactionGroup(nil), cgroup)
-	require.Equal(t, int64(0), cgLen)
-	require.Equal(t, int64(0), genLen)
+	require.Equal(t, []tsm1.CompactionGroup(nil), cgroup, "compaction group")
+	require.Equal(t, int64(0), cgLen, "compaction group length")
+	require.Equal(t, int64(0), genLen, "generation count")
 }
 
 // This test is added to account for a single generation that has a group size
@@ -2715,18 +2723,20 @@ func TestDefaultPlanner_FullyCompacted_ManySingleGenLessThen2GBNotMaxAggrBlocks(
 		tsdb.DefaultMaxPointsPerBlock,
 	}
 	err := fs.SetBlockCounts(blocks)
-	require.NoError(t, err)
+	require.NoError(t, err, "SetBlockCounts")
 
 	cp := tsm1.NewDefaultPlanner(fs, tsdb.DefaultCompactFullWriteColdDuration)
 
+	// 1048576000 is a magic number for bytes per gigabyte
+	reasonExp := fmt.Sprintf("not fully compacted and not idle because single generation with many files under %d GB and many files under aggressive compaction points per block count (%d points)", int(tsdb.MaxTSMFileSize/1048576000), tsdb.AggressiveMaxPointsPerBlock)
+
 	compacted, reason := cp.FullyCompacted()
-	reasonExp := "not fully compacted and not idle because single generation with many files under 2 GB and many files under aggressive compaction points per block count (100,000 points)"
-	require.Equal(t, reason, reasonExp)
-	require.Equal(t, false, compacted)
+	require.Equal(t, reason, reasonExp, "fullyCompacted reason")
+	require.False(t, compacted, "is fully compacted")
 
 	_, cgLen, genLen := cp.PlanOptimize()
-	require.Equal(t, int64(1), cgLen)
-	require.Equal(t, int64(1), genLen)
+	require.Equal(t, int64(1), cgLen, "compaction group length")
+	require.Equal(t, int64(1), genLen, "generation count")
 }
 
 func TestDefaultPlanner_PlanOptimize_Multiple(t *testing.T) {
@@ -2965,7 +2975,7 @@ func TestDefaultPlanner_Plan_SkipPlanningAfterFull(t *testing.T) {
 	}
 
 	err := fs.SetBlockCounts([]int{tsdb.DefaultMaxPointsPerBlock, tsdb.DefaultMaxPointsPerBlock, tsdb.DefaultMaxPointsPerBlock, tsdb.DefaultMaxPointsPerBlock})
-	require.NoError(t, err)
+	require.NoError(t, err, "SetBlockCounts")
 
 	cp := tsm1.NewDefaultPlanner(fs, time.Nanosecond)
 	plan, pLen := cp.Plan(time.Now().Add(-time.Second))
@@ -3014,7 +3024,7 @@ func TestDefaultPlanner_Plan_SkipPlanningAfterFull(t *testing.T) {
 		tsdb.DefaultMaxPointsPerBlock,
 		tsdb.DefaultMaxPointsPerBlock,
 	})
-	require.NoError(t, err)
+	require.NoError(t, err, "SetBlockCounts")
 
 	cp.FileStore = overFs
 	plan, pLen = cp.Plan(time.Now().Add(-time.Second))
@@ -3105,7 +3115,7 @@ func TestDefaultPlanner_Plan_TwoGenLevel3(t *testing.T) {
 	}
 
 	err := fs.SetBlockCounts(bcs)
-	require.NoError(t, err)
+	require.NoError(t, err, "SetBlockCounts")
 
 	cp := tsm1.NewDefaultPlanner(fs, time.Hour)
 
@@ -3151,7 +3161,7 @@ func TestDefaultPlanner_Plan_NotFullOverMaxsize(t *testing.T) {
 	}
 
 	err := fs.SetBlockCounts(bcs)
-	require.NoError(t, err)
+	require.NoError(t, err, "SetBlockCounts")
 
 	cp := tsm1.NewDefaultPlanner(
 		fs,
@@ -3191,7 +3201,7 @@ func TestDefaultPlanner_Plan_NotFullOverMaxsize(t *testing.T) {
 	}
 
 	err = overFs.SetBlockCounts(bcs)
-	require.NoError(t, err)
+	require.NoError(t, err, "SetBlockCounts")
 
 	cp.FileStore = overFs
 	cGroups, pLen := cp.Plan(time.Now().Add(-time.Second))

From f15d9be4157c765c323b937aecfd7111063c22e5 Mon Sep 17 00:00:00 2001
From: Devan <devandbenz@gmail.com>
Date: Wed, 18 Dec 2024 09:49:03 -0600
Subject: [PATCH 08/31] feat: need to use int64 instead of int

---
 tsdb/engine/tsm1/compact_test.go | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tsdb/engine/tsm1/compact_test.go b/tsdb/engine/tsm1/compact_test.go
index 5e9d6c7e438..9cc2b4f89d2 100644
--- a/tsdb/engine/tsm1/compact_test.go
+++ b/tsdb/engine/tsm1/compact_test.go
@@ -2411,8 +2411,8 @@ func TestDefaultPlanner_PlanOptimize_SmallSingleGenerationUnderLevel4(t *testing
 		expFiles = append(expFiles, file)
 	}
 	tsmP, pLenP := cp.Plan(time.Now().Add(-time.Second))
-	require.Equal(t, 0, len(tsmP), "compaction group; Plan()")
-	require.Equal(t, 0, pLenP, "compaction group length; Plan()")
+	require.Equal(t, int64(0), len(tsmP), "compaction group; Plan()")
+	require.Equal(t, int64(0), pLenP, "compaction group length; Plan()")
 
 	tsm, pLen, gLen := cp.PlanOptimize()
 	require.Equal(t, 1, len(tsm), "compaction group")

From 54c8e1c446102ae45c2597e6aafd488e4dea4495 Mon Sep 17 00:00:00 2001
From: Devan <devandbenz@gmail.com>
Date: Wed, 18 Dec 2024 09:56:00 -0600
Subject: [PATCH 09/31] feat: touch

---
 tsdb/engine/tsm1/compact_test.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tsdb/engine/tsm1/compact_test.go b/tsdb/engine/tsm1/compact_test.go
index 9cc2b4f89d2..11e38fcab6d 100644
--- a/tsdb/engine/tsm1/compact_test.go
+++ b/tsdb/engine/tsm1/compact_test.go
@@ -2411,7 +2411,7 @@ func TestDefaultPlanner_PlanOptimize_SmallSingleGenerationUnderLevel4(t *testing
 		expFiles = append(expFiles, file)
 	}
 	tsmP, pLenP := cp.Plan(time.Now().Add(-time.Second))
-	require.Equal(t, int64(0), len(tsmP), "compaction group; Plan()")
+	require.Equal(t, 0, len(tsmP), "compaction group; Plan()")
 	require.Equal(t, int64(0), pLenP, "compaction group length; Plan()")
 
 	tsm, pLen, gLen := cp.PlanOptimize()

From 403d888020cc5fcba1e0c5751117b0e327dbd06f Mon Sep 17 00:00:00 2001
From: Devan <devandbenz@gmail.com>
Date: Wed, 18 Dec 2024 14:25:06 -0600
Subject: [PATCH 10/31] feat: Adjust tests to include lower level planning
 function calls

---
 tsdb/config.go                   |   8 ++
 tsdb/engine/tsm1/compact.go      |   5 +-
 tsdb/engine/tsm1/compact_test.go | 125 ++++++++++++++++++++++++++++---
 3 files changed, 125 insertions(+), 13 deletions(-)

diff --git a/tsdb/config.go b/tsdb/config.go
index 0dbb0e4228a..a9252b2cc10 100644
--- a/tsdb/config.go
+++ b/tsdb/config.go
@@ -84,8 +84,16 @@ const (
 
 	// MaxTSMFileSize is the maximum size of TSM files.
 	MaxTSMFileSize = uint32(2048 * 1024 * 1024) // 2GB
+
 )
 
+// SingleGenerationReason outputs a log message for our single generation compaction
+// when checked for full compaction.
+// 1048576000 is a magic number for bytes per gigabyte.
+func SingleGenerationReason() string {
+	return fmt.Sprintf("not fully compacted and not idle because single generation with many files under %d GB and many files under aggressive compaction points per block count (%d points)", int(MaxTSMFileSize/1048576000), AggressiveMaxPointsPerBlock)
+}
+
 // Config holds the configuration for the tsbd package.
 type Config struct {
 	Dir    string `toml:"dir"`
diff --git a/tsdb/engine/tsm1/compact.go b/tsdb/engine/tsm1/compact.go
index 730b2d4ba17..945cc650a40 100644
--- a/tsdb/engine/tsm1/compact.go
+++ b/tsdb/engine/tsm1/compact.go
@@ -225,9 +225,6 @@ func (c *DefaultPlanner) ParseFileName(path string) (int, int, error) {
 // FullyCompacted returns true if the shard is fully compacted.
 func (c *DefaultPlanner) FullyCompacted() (bool, string) {
 	gens := c.findGenerations(false)
-	// 1048576000 is a magic number for bytes per gigabyte
-	singleGenReason := fmt.Sprintf("not fully compacted and not idle because single generation with many files under %d GB and many files under aggressive compaction points per block count (%d points)", int(tsdb.MaxTSMFileSize/1048576000), tsdb.AggressiveMaxPointsPerBlock)
-
 	if len(gens) > 1 {
 		return false, "not fully compacted and not idle because of more than one generation"
 	} else if gens.hasTombstones() {
@@ -251,7 +248,7 @@ func (c *DefaultPlanner) FullyCompacted() (bool, string) {
 			}
 
 			if filesUnderMaxTsmSizeCount > 1 && aggressivePointsPerBlockCount < len(gens[0].files) {
-				return false, singleGenReason
+				return false, tsdb.SingleGenerationReason()
 			}
 		}
 		return true, ""
diff --git a/tsdb/engine/tsm1/compact_test.go b/tsdb/engine/tsm1/compact_test.go
index 11e38fcab6d..3c283d24dae 100644
--- a/tsdb/engine/tsm1/compact_test.go
+++ b/tsdb/engine/tsm1/compact_test.go
@@ -2330,6 +2330,17 @@ func TestDefaultPlanner_PlanOptimize_LargeMultiGeneration(t *testing.T) {
 		expFiles = append(expFiles, file)
 	}
 
+	_, cgLen := cp.PlanLevel(1)
+	require.Equal(t, int64(0), cgLen, "compaction group length; PlanLevel(1)")
+	_, cgLen = cp.PlanLevel(2)
+	require.Equal(t, int64(0), cgLen, "compaction group length; PlanLevel(2)")
+	_, cgLen = cp.PlanLevel(3)
+	require.Equal(t, int64(0), cgLen, "compaction group length; PlanLevel(3)")
+
+	tsmP, pLenP := cp.Plan(time.Now().Add(-time.Second))
+	require.Equal(t, 0, len(tsmP), "compaction group; Plan()")
+	require.Equal(t, int64(0), pLenP, "compaction group length; Plan()")
+
 	tsm, pLen, _ := cp.PlanOptimize()
 	require.Equal(t, 1, len(tsm), "compaction group")
 	require.Equal(t, int64(len(tsm)), pLen, "compaction group length")
@@ -2372,6 +2383,17 @@ func TestDefaultPlanner_PlanOptimize_SmallSingleGeneration(t *testing.T) {
 		expFiles = append(expFiles, file)
 	}
 
+	_, cgLen := cp.PlanLevel(1)
+	require.Equal(t, int64(0), cgLen, "compaction group length; PlanLevel(1)")
+	_, cgLen = cp.PlanLevel(2)
+	require.Equal(t, int64(0), cgLen, "compaction group length; PlanLevel(2)")
+	_, cgLen = cp.PlanLevel(3)
+	require.Equal(t, int64(0), cgLen, "compaction group length; PlanLevel(3)")
+
+	tsmP, pLenP := cp.Plan(time.Now().Add(-time.Second))
+	require.Equal(t, 0, len(tsmP), "compaction group; Plan()")
+	require.Equal(t, int64(0), pLenP, "compaction group length; Plan()")
+
 	tsm, pLen, gLen := cp.PlanOptimize()
 	require.Equal(t, 1, len(tsm), "compaction group")
 	require.Equal(t, int64(len(tsm)), pLen, "compaction group length")
@@ -2410,6 +2432,14 @@ func TestDefaultPlanner_PlanOptimize_SmallSingleGenerationUnderLevel4(t *testing
 	for _, file := range data {
 		expFiles = append(expFiles, file)
 	}
+
+	_, cgLen := cp.PlanLevel(1)
+	require.Equal(t, int64(0), cgLen, "compaction group length; PlanLevel(1)")
+	_, cgLen = cp.PlanLevel(2)
+	require.Equal(t, int64(0), cgLen, "compaction group length; PlanLevel(2)")
+	_, cgLen = cp.PlanLevel(3)
+	require.Equal(t, int64(0), cgLen, "compaction group length; PlanLevel(3)")
+
 	tsmP, pLenP := cp.Plan(time.Now().Add(-time.Second))
 	require.Equal(t, 0, len(tsmP), "compaction group; Plan()")
 	require.Equal(t, int64(0), pLenP, "compaction group length; Plan()")
@@ -2456,12 +2486,24 @@ func TestDefaultPlanner_FullyCompacted_SmallSingleGeneration(t *testing.T) {
 
 	cp := tsm1.NewDefaultPlanner(fs, tsdb.DefaultCompactFullWriteColdDuration)
 
-	// 1048576000 is a magic number for bytes per gigabyte
-	reasonExp := fmt.Sprintf("not fully compacted and not idle because single generation with many files under %d GB and many files under aggressive compaction points per block count (%d points)", int(tsdb.MaxTSMFileSize/1048576000), tsdb.AggressiveMaxPointsPerBlock)
-
 	compacted, reason := cp.FullyCompacted()
-	require.Equal(t, reason, reasonExp, "fullyCompacted reason")
+	require.Equal(t, reason, tsdb.SingleGenerationReason(), "fullyCompacted reason")
 	require.False(t, compacted, "is fully compacted")
+
+	_, cgLen := cp.PlanLevel(1)
+	require.Equal(t, int64(0), cgLen, "compaction group length; PlanLevel(1)")
+	_, cgLen = cp.PlanLevel(2)
+	require.Equal(t, int64(0), cgLen, "compaction group length; PlanLevel(2)")
+	_, cgLen = cp.PlanLevel(3)
+	require.Equal(t, int64(0), cgLen, "compaction group length; PlanLevel(3)")
+
+	_, cgLen = cp.Plan(time.Now().Add(-1))
+	require.Equal(t, int64(0), cgLen, "compaction group length; Plan()")
+
+	cgroup, cgLen, genLen := cp.PlanOptimize()
+	require.Equal(t, []tsm1.CompactionGroup(nil), cgroup, "compaction group")
+	require.Equal(t, int64(0), cgLen, "compaction group length")
+	require.Equal(t, int64(0), genLen, "generation count")
 }
 
 // This test is added to account for halting state after
@@ -2488,6 +2530,21 @@ func TestDefaultPlanner_FullyCompacted_SmallSingleGeneration_Halt(t *testing.T)
 	reasonExp := ""
 	require.Equal(t, reason, reasonExp, "fullyCompacted reason")
 	require.True(t, compacted, "is fully compacted")
+
+	_, cgLen := cp.PlanLevel(1)
+	require.Equal(t, int64(0), cgLen, "compaction group length; PlanLevel(1)")
+	_, cgLen = cp.PlanLevel(2)
+	require.Equal(t, int64(0), cgLen, "compaction group length; PlanLevel(2)")
+	_, cgLen = cp.PlanLevel(3)
+	require.Equal(t, int64(0), cgLen, "compaction group length; PlanLevel(3)")
+
+	_, cgLen = cp.Plan(time.Now().Add(-1))
+	require.Equal(t, int64(0), cgLen, "compaction group length; Plan()")
+
+	cgroup, cgLen, genLen := cp.PlanOptimize()
+	require.Equal(t, []tsm1.CompactionGroup(nil), cgroup, "compaction group")
+	require.Equal(t, int64(0), cgLen, "compaction group length")
+	require.Equal(t, int64(0), genLen, "generation count")
 }
 
 // This test is added to account for a single generation that has a group size
@@ -2550,14 +2607,24 @@ func TestDefaultPlanner_FullyCompacted_LargeSingleGenerationUnderAggressiveBlock
 	require.NoError(t, err, "SetBlockCounts")
 
 	cp := tsm1.NewDefaultPlanner(fs, tsdb.DefaultCompactFullWriteColdDuration)
-
-	// 1048576000 is a magic number for bytes per gigabyte
-	reasonExp := fmt.Sprintf("not fully compacted and not idle because single generation with many files under %d GB and many files under aggressive compaction points per block count (%d points)", int(tsdb.MaxTSMFileSize/1048576000), tsdb.AggressiveMaxPointsPerBlock)
-
 	compacted, reason := cp.FullyCompacted()
-	require.Equal(t, reason, reasonExp, "fullyCompacted reason")
+	require.Equal(t, reason, tsdb.SingleGenerationReason(), "fullyCompacted reason")
 	require.False(t, compacted, "is fully compacted")
 
+	_, cgLen := cp.PlanLevel(1)
+	require.Equal(t, int64(0), cgLen, "compaction group length; PlanLevel(1)")
+	_, cgLen = cp.PlanLevel(2)
+	require.Equal(t, int64(0), cgLen, "compaction group length; PlanLevel(2)")
+	_, cgLen = cp.PlanLevel(3)
+	require.Equal(t, int64(0), cgLen, "compaction group length; PlanLevel(3)")
+
+	_, cgLen = cp.Plan(time.Now().Add(-1))
+	require.Equal(t, int64(0), cgLen, "compaction group length; Plan()")
+
+	cgroup, cgLen, genLen := cp.PlanOptimize()
+	require.Equal(t, []tsm1.CompactionGroup(nil), cgroup, "compaction group")
+	require.Equal(t, int64(0), cgLen, "compaction group length")
+	require.Equal(t, int64(0), genLen, "generation count")
 }
 
 // This test is added to account for a single generation that has a group size
@@ -2596,6 +2663,16 @@ func TestDefaultPlanner_FullyCompacted_LargeSingleGenerationMaxAggressiveBlocks(
 	require.Equal(t, reason, reasonExp, "fullyCompacted reason")
 	require.True(t, compacted, "is fully compacted")
 
+	_, cgLen := cp.PlanLevel(1)
+	require.Equal(t, int64(0), cgLen, "compaction group length; PlanLevel(1)")
+	_, cgLen = cp.PlanLevel(2)
+	require.Equal(t, int64(0), cgLen, "compaction group length; PlanLevel(2)")
+	_, cgLen = cp.PlanLevel(3)
+	require.Equal(t, int64(0), cgLen, "compaction group length; PlanLevel(3)")
+
+	_, cgLen = cp.Plan(time.Now().Add(-1))
+	require.Equal(t, int64(0), cgLen, "compaction group length; Plan()")
+
 	cgroup, cgLen, genLen := cp.PlanOptimize()
 	require.Equal(t, []tsm1.CompactionGroup(nil), cgroup, "compaction group")
 	require.Equal(t, int64(0), cgLen, "compaction group length")
@@ -2639,6 +2716,16 @@ func TestDefaultPlanner_FullyCompacted_LargeSingleGenerationNoMaxAggrBlocks(t *t
 	require.Equal(t, reason, reasonExp, "fullyCompacted reason")
 	require.True(t, compacted, "is fully compacted")
 
+	_, cgLen := cp.PlanLevel(1)
+	require.Equal(t, int64(0), cgLen, "compaction group length; PlanLevel(1)")
+	_, cgLen = cp.PlanLevel(2)
+	require.Equal(t, int64(0), cgLen, "compaction group length; PlanLevel(2)")
+	_, cgLen = cp.PlanLevel(3)
+	require.Equal(t, int64(0), cgLen, "compaction group length; PlanLevel(3)")
+
+	_, cgLen = cp.Plan(time.Now().Add(-1))
+	require.Equal(t, int64(0), cgLen, "compaction group length; Plan()")
+
 	cgroup, cgLen, genLen := cp.PlanOptimize()
 	require.Equal(t, []tsm1.CompactionGroup(nil), cgroup, "compaction group")
 	require.Equal(t, int64(0), cgLen, "compaction group length")
@@ -2685,6 +2772,16 @@ func TestDefaultPlanner_FullyCompacted_ManySingleGenLessThen2GBMaxAggrBlocks(t *
 	require.Equal(t, reason, reasonExp, "fullyCompacted reason")
 	require.True(t, compacted, "is fully compacted")
 
+	_, cgLen := cp.PlanLevel(1)
+	require.Equal(t, int64(0), cgLen, "compaction group length; PlanLevel(1)")
+	_, cgLen = cp.PlanLevel(2)
+	require.Equal(t, int64(0), cgLen, "compaction group length; PlanLevel(2)")
+	_, cgLen = cp.PlanLevel(3)
+	require.Equal(t, int64(0), cgLen, "compaction group length; PlanLevel(3)")
+
+	_, cgLen = cp.Plan(time.Now().Add(-1))
+	require.Equal(t, int64(0), cgLen, "compaction group length; Plan()")
+
 	cgroup, cgLen, genLen := cp.PlanOptimize()
 	require.Equal(t, []tsm1.CompactionGroup(nil), cgroup, "compaction group")
 	require.Equal(t, int64(0), cgLen, "compaction group length")
@@ -2734,6 +2831,16 @@ func TestDefaultPlanner_FullyCompacted_ManySingleGenLessThen2GBNotMaxAggrBlocks(
 	require.Equal(t, reason, reasonExp, "fullyCompacted reason")
 	require.False(t, compacted, "is fully compacted")
 
+	_, cgLen := cp.PlanLevel(1)
+	require.Equal(t, int64(0), cgLen, "compaction group length; PlanLevel(1)")
+	_, cgLen = cp.PlanLevel(2)
+	require.Equal(t, int64(0), cgLen, "compaction group length; PlanLevel(2)")
+	_, cgLen = cp.PlanLevel(3)
+	require.Equal(t, int64(0), cgLen, "compaction group length; PlanLevel(3)")
+
+	_, cgLen = cp.Plan(time.Now().Add(-1))
+	require.Equal(t, int64(0), cgLen, "compaction group length; Plan()")
+
 	_, cgLen, genLen := cp.PlanOptimize()
 	require.Equal(t, int64(1), cgLen, "compaction group length")
 	require.Equal(t, int64(1), genLen, "generation count")

From 23d12e104662367c68605f6fcf6278b520e2d372 Mon Sep 17 00:00:00 2001
From: Devan <devandbenz@gmail.com>
Date: Wed, 18 Dec 2024 14:34:55 -0600
Subject: [PATCH 11/31] feat: Fix up some tests that I forgot to adjust

---
 tsdb/engine/tsm1/compact_test.go | 13 ++++++-------
 1 file changed, 6 insertions(+), 7 deletions(-)

diff --git a/tsdb/engine/tsm1/compact_test.go b/tsdb/engine/tsm1/compact_test.go
index 3c283d24dae..07732050b6b 100644
--- a/tsdb/engine/tsm1/compact_test.go
+++ b/tsdb/engine/tsm1/compact_test.go
@@ -2501,9 +2501,9 @@ func TestDefaultPlanner_FullyCompacted_SmallSingleGeneration(t *testing.T) {
 	require.Equal(t, int64(0), cgLen, "compaction group length; Plan()")
 
 	cgroup, cgLen, genLen := cp.PlanOptimize()
-	require.Equal(t, []tsm1.CompactionGroup(nil), cgroup, "compaction group")
-	require.Equal(t, int64(0), cgLen, "compaction group length")
-	require.Equal(t, int64(0), genLen, "generation count")
+	require.Equal(t, len(data), cgroup, "compaction group")
+	require.Equal(t, int64(4), cgLen, "compaction group length")
+	require.Equal(t, int64(1), genLen, "generation count")
 }
 
 // This test is added to account for halting state after
@@ -2621,10 +2621,9 @@ func TestDefaultPlanner_FullyCompacted_LargeSingleGenerationUnderAggressiveBlock
 	_, cgLen = cp.Plan(time.Now().Add(-1))
 	require.Equal(t, int64(0), cgLen, "compaction group length; Plan()")
 
-	cgroup, cgLen, genLen := cp.PlanOptimize()
-	require.Equal(t, []tsm1.CompactionGroup(nil), cgroup, "compaction group")
-	require.Equal(t, int64(0), cgLen, "compaction group length")
-	require.Equal(t, int64(0), genLen, "generation count")
+	_, cgLen, genLen := cp.PlanOptimize()
+	require.Equal(t, int64(8), cgLen, "compaction group length")
+	require.Equal(t, int64(1), genLen, "generation count")
 }
 
 // This test is added to account for a single generation that has a group size

From d3afb030bd40bee93ed3648bb6b9af27a67a1917 Mon Sep 17 00:00:00 2001
From: Devan <devandbenz@gmail.com>
Date: Wed, 18 Dec 2024 14:43:35 -0600
Subject: [PATCH 12/31] feat: fix typo

---
 tsdb/engine/tsm1/compact_test.go | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/tsdb/engine/tsm1/compact_test.go b/tsdb/engine/tsm1/compact_test.go
index 07732050b6b..5f00fedcedc 100644
--- a/tsdb/engine/tsm1/compact_test.go
+++ b/tsdb/engine/tsm1/compact_test.go
@@ -2501,8 +2501,8 @@ func TestDefaultPlanner_FullyCompacted_SmallSingleGeneration(t *testing.T) {
 	require.Equal(t, int64(0), cgLen, "compaction group length; Plan()")
 
 	cgroup, cgLen, genLen := cp.PlanOptimize()
-	require.Equal(t, len(data), cgroup, "compaction group")
-	require.Equal(t, int64(4), cgLen, "compaction group length")
+	require.Equal(t, len(data), len(cgroup), "compaction group")
+	require.Equal(t, int64(1), cgLen, "compaction group length")
 	require.Equal(t, int64(1), genLen, "generation count")
 }
 
@@ -2622,7 +2622,7 @@ func TestDefaultPlanner_FullyCompacted_LargeSingleGenerationUnderAggressiveBlock
 	require.Equal(t, int64(0), cgLen, "compaction group length; Plan()")
 
 	_, cgLen, genLen := cp.PlanOptimize()
-	require.Equal(t, int64(8), cgLen, "compaction group length")
+	require.Equal(t, int64(1), cgLen, "compaction group length")
 	require.Equal(t, int64(1), genLen, "generation count")
 }
 

From cf657a8cad12d70425c34aeeeeebdc7dc1067646 Mon Sep 17 00:00:00 2001
From: Devan <devandbenz@gmail.com>
Date: Wed, 18 Dec 2024 14:51:04 -0600
Subject: [PATCH 13/31] feat: touch

---
 tsdb/engine/tsm1/compact_test.go | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/tsdb/engine/tsm1/compact_test.go b/tsdb/engine/tsm1/compact_test.go
index 5f00fedcedc..83eb187b0e2 100644
--- a/tsdb/engine/tsm1/compact_test.go
+++ b/tsdb/engine/tsm1/compact_test.go
@@ -2500,8 +2500,7 @@ func TestDefaultPlanner_FullyCompacted_SmallSingleGeneration(t *testing.T) {
 	_, cgLen = cp.Plan(time.Now().Add(-1))
 	require.Equal(t, int64(0), cgLen, "compaction group length; Plan()")
 
-	cgroup, cgLen, genLen := cp.PlanOptimize()
-	require.Equal(t, len(data), len(cgroup), "compaction group")
+	_, cgLen, genLen := cp.PlanOptimize()
 	require.Equal(t, int64(1), cgLen, "compaction group length")
 	require.Equal(t, int64(1), genLen, "generation count")
 }

From fc6ca13ea772376a3f12da5e4218775af8e73bf1 Mon Sep 17 00:00:00 2001
From: Devan <devandbenz@gmail.com>
Date: Thu, 19 Dec 2024 12:04:45 -0600
Subject: [PATCH 14/31] feat: Call SingleGenerationReason() once by
 initializing a var called SingleGenerationReasonText var

---
 tsdb/config.go                   | 3 ++-
 tsdb/engine/tsm1/compact.go      | 2 +-
 tsdb/engine/tsm1/compact_test.go | 4 ++--
 3 files changed, 5 insertions(+), 4 deletions(-)

diff --git a/tsdb/config.go b/tsdb/config.go
index a9252b2cc10..2d5bf2508f5 100644
--- a/tsdb/config.go
+++ b/tsdb/config.go
@@ -84,9 +84,10 @@ const (
 
 	// MaxTSMFileSize is the maximum size of TSM files.
 	MaxTSMFileSize = uint32(2048 * 1024 * 1024) // 2GB
-
 )
 
+var SingleGenerationReasonText string = SingleGenerationReason()
+
 // SingleGenerationReason outputs a log message for our single generation compaction
 // when checked for full compaction.
 // 1048576000 is a magic number for bytes per gigabyte.
diff --git a/tsdb/engine/tsm1/compact.go b/tsdb/engine/tsm1/compact.go
index 945cc650a40..a8cfd413c5f 100644
--- a/tsdb/engine/tsm1/compact.go
+++ b/tsdb/engine/tsm1/compact.go
@@ -248,7 +248,7 @@ func (c *DefaultPlanner) FullyCompacted() (bool, string) {
 			}
 
 			if filesUnderMaxTsmSizeCount > 1 && aggressivePointsPerBlockCount < len(gens[0].files) {
-				return false, tsdb.SingleGenerationReason()
+				return false, tsdb.SingleGenerationReasonText
 			}
 		}
 		return true, ""
diff --git a/tsdb/engine/tsm1/compact_test.go b/tsdb/engine/tsm1/compact_test.go
index 83eb187b0e2..1a2a24a0c75 100644
--- a/tsdb/engine/tsm1/compact_test.go
+++ b/tsdb/engine/tsm1/compact_test.go
@@ -2487,7 +2487,7 @@ func TestDefaultPlanner_FullyCompacted_SmallSingleGeneration(t *testing.T) {
 	cp := tsm1.NewDefaultPlanner(fs, tsdb.DefaultCompactFullWriteColdDuration)
 
 	compacted, reason := cp.FullyCompacted()
-	require.Equal(t, reason, tsdb.SingleGenerationReason(), "fullyCompacted reason")
+	require.Equal(t, reason, tsdb.SingleGenerationReasonText, "fullyCompacted reason")
 	require.False(t, compacted, "is fully compacted")
 
 	_, cgLen := cp.PlanLevel(1)
@@ -2607,7 +2607,7 @@ func TestDefaultPlanner_FullyCompacted_LargeSingleGenerationUnderAggressiveBlock
 
 	cp := tsm1.NewDefaultPlanner(fs, tsdb.DefaultCompactFullWriteColdDuration)
 	compacted, reason := cp.FullyCompacted()
-	require.Equal(t, reason, tsdb.SingleGenerationReason(), "fullyCompacted reason")
+	require.Equal(t, reason, tsdb.SingleGenerationReasonText, "fullyCompacted reason")
 	require.False(t, compacted, "is fully compacted")
 
 	_, cgLen := cp.PlanLevel(1)

From 4fc4d5546d7bbd82877ff303c8725d98485eb6ab Mon Sep 17 00:00:00 2001
From: Devan <devandbenz@gmail.com>
Date: Thu, 19 Dec 2024 12:15:49 -0600
Subject: [PATCH 15/31] feat: clarify file counts for reason we are not fully
 compacted

---
 tsdb/config.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tsdb/config.go b/tsdb/config.go
index 2d5bf2508f5..3d7a85a82fc 100644
--- a/tsdb/config.go
+++ b/tsdb/config.go
@@ -92,7 +92,7 @@ var SingleGenerationReasonText string = SingleGenerationReason()
 // when checked for full compaction.
 // 1048576000 is a magic number for bytes per gigabyte.
 func SingleGenerationReason() string {
-	return fmt.Sprintf("not fully compacted and not idle because single generation with many files under %d GB and many files under aggressive compaction points per block count (%d points)", int(MaxTSMFileSize/1048576000), AggressiveMaxPointsPerBlock)
+	return fmt.Sprintf("not fully compacted and not idle because single generation with more then 2 files under %d GB and more then 1 file(s) under aggressive compaction points per block count (%d points)", int(MaxTSMFileSize/1048576000), AggressiveMaxPointsPerBlock)
 }
 
 // Config holds the configuration for the tsbd package.

From c93bdfbc550a2814da15f13fce0af03e0d35927a Mon Sep 17 00:00:00 2001
From: Devan <devandbenz@gmail.com>
Date: Thu, 19 Dec 2024 12:16:48 -0600
Subject: [PATCH 16/31] feat: grammar typo

---
 tsdb/config.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tsdb/config.go b/tsdb/config.go
index 3d7a85a82fc..0490934f1bb 100644
--- a/tsdb/config.go
+++ b/tsdb/config.go
@@ -92,7 +92,7 @@ var SingleGenerationReasonText string = SingleGenerationReason()
 // when checked for full compaction.
 // 1048576000 is a magic number for bytes per gigabyte.
 func SingleGenerationReason() string {
-	return fmt.Sprintf("not fully compacted and not idle because single generation with more then 2 files under %d GB and more then 1 file(s) under aggressive compaction points per block count (%d points)", int(MaxTSMFileSize/1048576000), AggressiveMaxPointsPerBlock)
+	return fmt.Sprintf("not fully compacted and not idle because single generation with more than 2 files under %d GB and more than 1 file(s) under aggressive compaction points per block count (%d points)", int(MaxTSMFileSize/1048576000), AggressiveMaxPointsPerBlock)
 }
 
 // Config holds the configuration for the tsbd package.

From 2dd5ef40ed2fc68fd4b20d1be68cd02b260c28e3 Mon Sep 17 00:00:00 2001
From: Devan <devandbenz@gmail.com>
Date: Thu, 19 Dec 2024 12:41:33 -0600
Subject: [PATCH 17/31] feat: missed a test when updating the variable! whoops!

---
 tsdb/engine/tsm1/compact_test.go | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/tsdb/engine/tsm1/compact_test.go b/tsdb/engine/tsm1/compact_test.go
index 1a2a24a0c75..c7399e17d67 100644
--- a/tsdb/engine/tsm1/compact_test.go
+++ b/tsdb/engine/tsm1/compact_test.go
@@ -2822,11 +2822,8 @@ func TestDefaultPlanner_FullyCompacted_ManySingleGenLessThen2GBNotMaxAggrBlocks(
 
 	cp := tsm1.NewDefaultPlanner(fs, tsdb.DefaultCompactFullWriteColdDuration)
 
-	// 1048576000 is a magic number for bytes per gigabyte
-	reasonExp := fmt.Sprintf("not fully compacted and not idle because single generation with many files under %d GB and many files under aggressive compaction points per block count (%d points)", int(tsdb.MaxTSMFileSize/1048576000), tsdb.AggressiveMaxPointsPerBlock)
-
 	compacted, reason := cp.FullyCompacted()
-	require.Equal(t, reason, reasonExp, "fullyCompacted reason")
+	require.Equal(t, reason, tsdb.SingleGenerationReasonText, "fullyCompacted reason")
 	require.False(t, compacted, "is fully compacted")
 
 	_, cgLen := cp.PlanLevel(1)

From 479de96f9b583c83f31edfa6183aaf2d8492df51 Mon Sep 17 00:00:00 2001
From: Devan <devandbenz@gmail.com>
Date: Thu, 19 Dec 2024 19:03:23 -0600
Subject: [PATCH 18/31] feat: Add test for another edge case found; Many tsm
 generations over level 4 compaction single tsm generation under level 4
 compaction all in same shard. Group size is over 2 GB for each generation.

---
 tsdb/engine/tsm1/compact_test.go | 86 ++++++++++++++++++++++++++++++++
 1 file changed, 86 insertions(+)

diff --git a/tsdb/engine/tsm1/compact_test.go b/tsdb/engine/tsm1/compact_test.go
index c7399e17d67..4dbd23c40bf 100644
--- a/tsdb/engine/tsm1/compact_test.go
+++ b/tsdb/engine/tsm1/compact_test.go
@@ -2841,6 +2841,92 @@ func TestDefaultPlanner_FullyCompacted_ManySingleGenLessThen2GBNotMaxAggrBlocks(
 	require.Equal(t, int64(1), genLen, "generation count")
 }
 
+// This test is added to account for multiple generations over level 4
+// compaction and over 2 GB group size, with a level 3 start generation
+// over 2 GB group size.
+func TestDefaultPlanner_FullyCompacted_ManySingleGen2GBLastLevel2(t *testing.T) {
+	// > 2 GB total group size
+	// 100% of files are at aggressive max block size
+	data := []tsm1.FileStat{
+		{
+			Path: "01-05.tsm1",
+			Size: 2048 * 1024 * 1024,
+		},
+		{
+			Path: "01-06.tsm1",
+			Size: 2048 * 1024 * 1024,
+		},
+		{
+			Path: "01-07.tsm1",
+			Size: 2048 * 1024 * 1024,
+		},
+		{
+			Path: "01-08.tsm1",
+			Size: 1048 * 1024 * 1024,
+		},
+		{
+			Path: "02-05.tsm1",
+			Size: 2048 * 1024 * 1024,
+		},
+		{
+			Path: "02-06.tsm1",
+			Size: 2048 * 1024 * 1024,
+		},
+		{
+			Path: "02-07.tsm1",
+			Size: 2048 * 1024 * 1024,
+		},
+		{
+			Path: "02-08.tsm1",
+			Size: 1048 * 1024 * 1024,
+		},
+		{
+			Path: "03-02.tsm1",
+			Size: 2048 * 1024 * 1024,
+		},
+		{
+			Path: "03-03.tsm1",
+			Size: 2048 * 1024 * 1024,
+		},
+		{
+			Path: "03-04.tsm1",
+			Size: 600 * 1024 * 1024,
+		},
+		{
+			Path: "03-05.tsm1",
+			Size: 500 * 1024 * 1024,
+		},
+	}
+
+	fs := &fakeFileStore{
+		PathsFn: func() []tsm1.FileStat {
+			return data
+		},
+	}
+
+	cp := tsm1.NewDefaultPlanner(fs, tsdb.DefaultCompactFullWriteColdDuration)
+
+	expFiles := make([]tsm1.FileStat, 0)
+	for _, file := range data {
+		expFiles = append(expFiles, file)
+	}
+
+	_, cgLen := cp.PlanLevel(1)
+	require.Equal(t, int64(0), cgLen, "compaction group length; PlanLevel(1)")
+	_, cgLen = cp.PlanLevel(2)
+	require.Equal(t, int64(0), cgLen, "compaction group length; PlanLevel(2)")
+	_, cgLen = cp.PlanLevel(3)
+	require.Equal(t, int64(0), cgLen, "compaction group length; PlanLevel(3)")
+
+	_, cgLen = cp.Plan(time.Now().Add(-1))
+	require.Equal(t, int64(0), cgLen, "compaction group length; Plan()")
+
+	tsm, cgLen, genLen := cp.PlanOptimize()
+	require.Equal(t, int64(1), cgLen, "compaction group length")
+	require.Equal(t, int64(3), genLen, "generation count")
+	require.Equal(t, len(expFiles), len(tsm[0]), "tsm files in compaction group")
+}
+
 func TestDefaultPlanner_PlanOptimize_Multiple(t *testing.T) {
 	data := []tsm1.FileStat{
 		{

From c3929069692cb49e1319cb6f40b0e4759daf49ce Mon Sep 17 00:00:00 2001
From: Devan <devandbenz@gmail.com>
Date: Fri, 20 Dec 2024 15:12:52 -0600
Subject: [PATCH 19/31] feat: Remove some overlapping tests Add a check to
 ensure that "orphaned" levels are compacted further with the rest of the
 shard.

---
 tsdb/engine/tsm1/compact.go      |   2 +-
 tsdb/engine/tsm1/compact_test.go | 148 +------------------------------
 2 files changed, 4 insertions(+), 146 deletions(-)

diff --git a/tsdb/engine/tsm1/compact.go b/tsdb/engine/tsm1/compact.go
index a8cfd413c5f..fd94f81f036 100644
--- a/tsdb/engine/tsm1/compact.go
+++ b/tsdb/engine/tsm1/compact.go
@@ -397,7 +397,7 @@ func (c *DefaultPlanner) PlanOptimize() (compactGroup []CompactionGroup, compact
 			}
 		}
 
-		if len(currentGen) == 0 || currentGen.level() == cur.level() {
+		if len(currentGen) == 0 || currentGen.level() >= cur.level() {
 			currentGen = append(currentGen, cur)
 			continue
 		}
diff --git a/tsdb/engine/tsm1/compact_test.go b/tsdb/engine/tsm1/compact_test.go
index 4dbd23c40bf..71e61d53789 100644
--- a/tsdb/engine/tsm1/compact_test.go
+++ b/tsdb/engine/tsm1/compact_test.go
@@ -2212,65 +2212,6 @@ func TestDefaultPlanner_PlanOptimize_NoLevel4(t *testing.T) {
 	}
 }
 
-func TestDefaultPlanner_PlanOptimize_Level4(t *testing.T) {
-	data := []tsm1.FileStat{
-		{
-			Path: "01-04.tsm1",
-			Size: 251 * 1024 * 1024,
-		},
-		{
-			Path: "02-04.tsm1",
-			Size: 1 * 1024 * 1024,
-		},
-		{
-			Path: "03-04.tsm1",
-			Size: 1 * 1024 * 1024,
-		},
-		{
-			Path: "04-04.tsm1",
-			Size: 1 * 1024 * 1024,
-		},
-		{
-			Path: "05-03.tsm1",
-			Size: 2 * 1024 * 1024 * 1024,
-		},
-		{
-			Path: "06-04.tsm1",
-			Size: 2 * 1024 * 1024 * 1024,
-		},
-		{
-			Path: "07-03.tsm1",
-			Size: 2 * 1024 * 1024 * 1024,
-		},
-	}
-
-	cp := tsm1.NewDefaultPlanner(
-		&fakeFileStore{
-			PathsFn: func() []tsm1.FileStat {
-				return data
-			},
-		}, tsdb.DefaultCompactFullWriteColdDuration,
-	)
-
-	expFiles1 := []tsm1.FileStat{data[0], data[1], data[2], data[3], data[4], data[5]}
-	tsm, pLen, _ := cp.PlanOptimize()
-	if exp, got := 1, len(tsm); exp != got {
-		t.Fatalf("group length mismatch: got %v, exp %v", got, exp)
-	} else if pLen != int64(len(tsm)) {
-		t.Fatalf("tsm file plan length mismatch: got %v, exp %v", pLen, exp)
-	}
-
-	if exp, got := len(expFiles1), len(tsm[0]); got != exp {
-		t.Fatalf("tsm file length mismatch: got %v, exp %v", got, exp)
-	}
-
-	for i, p := range expFiles1 {
-		if got, exp := tsm[0][i], p.Path; got != exp {
-			t.Fatalf("tsm file mismatch: got %v, exp %v", got, exp)
-		}
-	}
-}
-
 // This test is added to acount for many TSM files within a group being over 2 GB
 // we want to ensure that the shard will be planned.
 func TestDefaultPlanner_PlanOptimize_LargeMultiGeneration(t *testing.T) {
@@ -2881,11 +2822,11 @@ func TestDefaultPlanner_FullyCompacted_ManySingleGen2GBLastLevel2(t *testing.T)
 			Size: 1048 * 1024 * 1024,
 		},
 		{
-			Path: "03-02.tsm1",
+			Path: "03-03.tsm1",
 			Size: 2048 * 1024 * 1024,
 		},
 		{
-			Path: "03-03.tsm1",
+			Path: "03-04.tsm1",
 			Size: 2048 * 1024 * 1024,
 		},
 		{
@@ -2893,7 +2834,7 @@ func TestDefaultPlanner_FullyCompacted_ManySingleGen2GBLastLevel2(t *testing.T)
 			Size: 600 * 1024 * 1024,
 		},
 		{
-			Path: "03-05.tsm1",
+			Path: "03-06.tsm1",
 			Size: 500 * 1024 * 1024,
 		},
 	}
@@ -2927,89 +2868,6 @@ func TestDefaultPlanner_FullyCompacted_ManySingleGen2GBLastLevel2(t *testing.T)
 	require.Equal(t, len(expFiles), len(tsm[0]), "tsm files in compaction group")
 }
 
-func TestDefaultPlanner_PlanOptimize_Multiple(t *testing.T) {
-	data := []tsm1.FileStat{
-		{
-			Path: "01-04.tsm1",
-			Size: 251 * 1024 * 1024,
-		},
-		{
-			Path: "02-04.tsm1",
-			Size: 1 * 1024 * 1024,
-		},
-		{
-			Path: "03-04.tsm1",
-			Size: 1 * 1024 * 1024,
-		},
-		{
-			Path: "04-04.tsm1",
-			Size: 1 * 1024 * 1024,
-		},
-		{
-			Path: "05-03.tsm1",
-			Size: 2 * 1024 * 1024 * 1024,
-		},
-		{
-			Path: "06-03.tsm1",
-			Size: 2 * 1024 * 1024 * 1024,
-		},
-		{
-			Path: "07-04.tsm1",
-			Size: 2 * 1024 * 1024 * 1024,
-		},
-		{
-			Path: "08-04.tsm1",
-			Size: 2 * 1024 * 1024 * 1024,
-		},
-		{
-			Path: "09-04.tsm1",
-			Size: 2 * 1024 * 1024 * 1024,
-		},
-		{
-			Path: "10-04.tsm1",
-			Size: 2 * 1024 * 1024 * 1024,
-		},
-	}
-
-	cp := tsm1.NewDefaultPlanner(
-		&fakeFileStore{
-			PathsFn: func() []tsm1.FileStat {
-				return data
-			},
-		}, tsdb.DefaultCompactFullWriteColdDuration,
-	)
-
-	expFiles1 := []tsm1.FileStat{data[0], data[1], data[2], data[3]}
-	expFiles2 := []tsm1.FileStat{data[6], data[7], data[8], data[9]}
-
-	tsm, pLen, _ := cp.PlanOptimize()
-	if exp, got := 2, len(tsm); exp != got {
-		t.Fatalf("group length mismatch: got %v, exp %v", got, exp)
-	} else if pLen != int64(len(tsm)) {
-		t.Fatalf("tsm file plan length mismatch: got %v, exp %v", pLen, exp)
-	}
-
-	if exp, got := len(expFiles1), len(tsm[0]); got != exp {
-		t.Fatalf("tsm file length mismatch: got %v, exp %v", got, exp)
-	}
-
-	for i, p := range expFiles1 {
-		if got, exp := tsm[0][i], p.Path; got != exp {
-			t.Fatalf("tsm file mismatch: got %v, exp %v", got, exp)
-		}
-	}
-
-	if exp, got := len(expFiles2), len(tsm[1]); got != exp {
-		t.Fatalf("tsm file length mismatch: got %v, exp %v", got, exp)
-	}
-
-	for i, p := range expFiles2 {
-		if got, exp := tsm[1][i], p.Path; got != exp {
-			t.Fatalf("tsm file mismatch: got %v, exp %v", got, exp)
-		}
-	}
-}
-
 func TestDefaultPlanner_PlanOptimize_Tombstones(t *testing.T) {
 	data := []tsm1.FileStat{
 		{

From f444518abbd45caf867a3a14e9ca5199f709e35b Mon Sep 17 00:00:00 2001
From: Devan <devandbenz@gmail.com>
Date: Thu, 26 Dec 2024 12:04:42 -0600
Subject: [PATCH 20/31] feat: Adds check for block counts and adjusts tests to
 use require.Zero()

---
 tsdb/engine/tsm1/compact.go      |   6 +-
 tsdb/engine/tsm1/compact_test.go | 126 ++++++++++++++++---------------
 2 files changed, 70 insertions(+), 62 deletions(-)

diff --git a/tsdb/engine/tsm1/compact.go b/tsdb/engine/tsm1/compact.go
index fd94f81f036..8f49be13e20 100644
--- a/tsdb/engine/tsm1/compact.go
+++ b/tsdb/engine/tsm1/compact.go
@@ -469,7 +469,7 @@ func (c *DefaultPlanner) Plan(lastWrite time.Time) ([]CompactionGroup, int64) {
 			var skip bool
 
 			// Skip the file if it's over the max size and contains a full block and it does not have any tombstones
-			if len(generations) > 2 && group.size() > uint64(tsdb.MaxTSMFileSize) && c.FileStore.BlockCount(group.files[0].Path, 1) == tsdb.DefaultMaxPointsPerBlock && !group.hasTombstones() {
+			if len(generations) > 2 && group.size() > uint64(tsdb.MaxTSMFileSize) && c.FileStore.BlockCount(group.files[0].Path, 1) >= tsdb.DefaultMaxPointsPerBlock && !group.hasTombstones() {
 				skip = true
 			}
 
@@ -545,7 +545,7 @@ func (c *DefaultPlanner) Plan(lastWrite time.Time) ([]CompactionGroup, int64) {
 		// Skip the file if it's over the max size and contains a full block or the generation is split
 		// over multiple files.  In the latter case, that would mean the data in the file spilled over
 		// the 2GB limit.
-		if g.size() > uint64(tsdb.MaxTSMFileSize) && c.FileStore.BlockCount(g.files[0].Path, 1) == tsdb.DefaultMaxPointsPerBlock {
+		if g.size() > uint64(tsdb.MaxTSMFileSize) && c.FileStore.BlockCount(g.files[0].Path, 1) >= tsdb.DefaultMaxPointsPerBlock {
 			start = i + 1
 		}
 
@@ -589,7 +589,7 @@ func (c *DefaultPlanner) Plan(lastWrite time.Time) ([]CompactionGroup, int64) {
 			}
 
 			// Skip the file if it's over the max size and it contains a full block
-			if gen.size() >= uint64(tsdb.MaxTSMFileSize) && c.FileStore.BlockCount(gen.files[0].Path, 1) == tsdb.DefaultMaxPointsPerBlock && !gen.hasTombstones() {
+			if gen.size() >= uint64(tsdb.MaxTSMFileSize) && c.FileStore.BlockCount(gen.files[0].Path, 1) >= tsdb.DefaultMaxPointsPerBlock && !gen.hasTombstones() {
 				startIndex++
 				continue
 			}
diff --git a/tsdb/engine/tsm1/compact_test.go b/tsdb/engine/tsm1/compact_test.go
index 71e61d53789..32258104010 100644
--- a/tsdb/engine/tsm1/compact_test.go
+++ b/tsdb/engine/tsm1/compact_test.go
@@ -2272,15 +2272,15 @@ func TestDefaultPlanner_PlanOptimize_LargeMultiGeneration(t *testing.T) {
 	}
 
 	_, cgLen := cp.PlanLevel(1)
-	require.Equal(t, int64(0), cgLen, "compaction group length; PlanLevel(1)")
+	require.Zero(t, cgLen, "compaction group length; PlanLevel(1)")
 	_, cgLen = cp.PlanLevel(2)
-	require.Equal(t, int64(0), cgLen, "compaction group length; PlanLevel(2)")
+	require.Zero(t, cgLen, "compaction group length; PlanLevel(2)")
 	_, cgLen = cp.PlanLevel(3)
-	require.Equal(t, int64(0), cgLen, "compaction group length; PlanLevel(3)")
+	require.Zero(t, cgLen, "compaction group length; PlanLevel(3)")
 
 	tsmP, pLenP := cp.Plan(time.Now().Add(-time.Second))
-	require.Equal(t, 0, len(tsmP), "compaction group; Plan()")
-	require.Equal(t, int64(0), pLenP, "compaction group length; Plan()")
+	require.Zero(t, len(tsmP), "compaction group; Plan()")
+	require.Zero(t, pLenP, "compaction group length; Plan()")
 
 	tsm, pLen, _ := cp.PlanOptimize()
 	require.Equal(t, 1, len(tsm), "compaction group")
@@ -2325,15 +2325,15 @@ func TestDefaultPlanner_PlanOptimize_SmallSingleGeneration(t *testing.T) {
 	}
 
 	_, cgLen := cp.PlanLevel(1)
-	require.Equal(t, int64(0), cgLen, "compaction group length; PlanLevel(1)")
+	require.Zero(t, cgLen, "compaction group length; PlanLevel(1)")
 	_, cgLen = cp.PlanLevel(2)
-	require.Equal(t, int64(0), cgLen, "compaction group length; PlanLevel(2)")
+	require.Zero(t, cgLen, "compaction group length; PlanLevel(2)")
 	_, cgLen = cp.PlanLevel(3)
-	require.Equal(t, int64(0), cgLen, "compaction group length; PlanLevel(3)")
+	require.Zero(t, cgLen, "compaction group length; PlanLevel(3)")
 
 	tsmP, pLenP := cp.Plan(time.Now().Add(-time.Second))
-	require.Equal(t, 0, len(tsmP), "compaction group; Plan()")
-	require.Equal(t, int64(0), pLenP, "compaction group length; Plan()")
+	require.Zero(t, len(tsmP), "compaction group; Plan()")
+	require.Zero(t, pLenP, "compaction group length; Plan()")
 
 	tsm, pLen, gLen := cp.PlanOptimize()
 	require.Equal(t, 1, len(tsm), "compaction group")
@@ -2375,15 +2375,15 @@ func TestDefaultPlanner_PlanOptimize_SmallSingleGenerationUnderLevel4(t *testing
 	}
 
 	_, cgLen := cp.PlanLevel(1)
-	require.Equal(t, int64(0), cgLen, "compaction group length; PlanLevel(1)")
+	require.Zero(t, cgLen, "compaction group length; PlanLevel(1)")
 	_, cgLen = cp.PlanLevel(2)
-	require.Equal(t, int64(0), cgLen, "compaction group length; PlanLevel(2)")
+	require.Zero(t, cgLen, "compaction group length; PlanLevel(2)")
 	_, cgLen = cp.PlanLevel(3)
-	require.Equal(t, int64(0), cgLen, "compaction group length; PlanLevel(3)")
+	require.Zero(t, cgLen, "compaction group length; PlanLevel(3)")
 
 	tsmP, pLenP := cp.Plan(time.Now().Add(-time.Second))
-	require.Equal(t, 0, len(tsmP), "compaction group; Plan()")
-	require.Equal(t, int64(0), pLenP, "compaction group length; Plan()")
+	require.Zero(t, len(tsmP), "compaction group; Plan()")
+	require.Zero(t, pLenP, "compaction group length; Plan()")
 
 	tsm, pLen, gLen := cp.PlanOptimize()
 	require.Equal(t, 1, len(tsm), "compaction group")
@@ -2432,14 +2432,15 @@ func TestDefaultPlanner_FullyCompacted_SmallSingleGeneration(t *testing.T) {
 	require.False(t, compacted, "is fully compacted")
 
 	_, cgLen := cp.PlanLevel(1)
-	require.Equal(t, int64(0), cgLen, "compaction group length; PlanLevel(1)")
+	require.Zero(t, cgLen, "compaction group length; PlanLevel(1)")
 	_, cgLen = cp.PlanLevel(2)
-	require.Equal(t, int64(0), cgLen, "compaction group length; PlanLevel(2)")
+	require.Zero(t, cgLen, "compaction group length; PlanLevel(2)")
 	_, cgLen = cp.PlanLevel(3)
-	require.Equal(t, int64(0), cgLen, "compaction group length; PlanLevel(3)")
+	require.Zero(t, cgLen, "compaction group length; PlanLevel(3)")
 
-	_, cgLen = cp.Plan(time.Now().Add(-1))
-	require.Equal(t, int64(0), cgLen, "compaction group length; Plan()")
+	tsmP, pLenP := cp.Plan(time.Now().Add(-time.Second))
+	require.Zero(t, len(tsmP), "compaction group; Plan()")
+	require.Zero(t, pLenP, "compaction group length; Plan()")
 
 	_, cgLen, genLen := cp.PlanOptimize()
 	require.Equal(t, int64(1), cgLen, "compaction group length")
@@ -2472,19 +2473,20 @@ func TestDefaultPlanner_FullyCompacted_SmallSingleGeneration_Halt(t *testing.T)
 	require.True(t, compacted, "is fully compacted")
 
 	_, cgLen := cp.PlanLevel(1)
-	require.Equal(t, int64(0), cgLen, "compaction group length; PlanLevel(1)")
+	require.Zero(t, cgLen, "compaction group length; PlanLevel(1)")
 	_, cgLen = cp.PlanLevel(2)
-	require.Equal(t, int64(0), cgLen, "compaction group length; PlanLevel(2)")
+	require.Zero(t, cgLen, "compaction group length; PlanLevel(2)")
 	_, cgLen = cp.PlanLevel(3)
-	require.Equal(t, int64(0), cgLen, "compaction group length; PlanLevel(3)")
+	require.Zero(t, cgLen, "compaction group length; PlanLevel(3)")
 
-	_, cgLen = cp.Plan(time.Now().Add(-1))
-	require.Equal(t, int64(0), cgLen, "compaction group length; Plan()")
+	tsmP, pLenP := cp.Plan(time.Now().Add(-time.Second))
+	require.Zero(t, len(tsmP), "compaction group; Plan()")
+	require.Zero(t, pLenP, "compaction group length; Plan()")
 
 	cgroup, cgLen, genLen := cp.PlanOptimize()
 	require.Equal(t, []tsm1.CompactionGroup(nil), cgroup, "compaction group")
-	require.Equal(t, int64(0), cgLen, "compaction group length")
-	require.Equal(t, int64(0), genLen, "generation count")
+	require.Zero(t, cgLen, "compaction group length")
+	require.Zero(t, genLen, "generation count")
 }
 
 // This test is added to account for a single generation that has a group size
@@ -2552,14 +2554,15 @@ func TestDefaultPlanner_FullyCompacted_LargeSingleGenerationUnderAggressiveBlock
 	require.False(t, compacted, "is fully compacted")
 
 	_, cgLen := cp.PlanLevel(1)
-	require.Equal(t, int64(0), cgLen, "compaction group length; PlanLevel(1)")
+	require.Zero(t, cgLen, "compaction group length; PlanLevel(1)")
 	_, cgLen = cp.PlanLevel(2)
-	require.Equal(t, int64(0), cgLen, "compaction group length; PlanLevel(2)")
+	require.Zero(t, cgLen, "compaction group length; PlanLevel(2)")
 	_, cgLen = cp.PlanLevel(3)
-	require.Equal(t, int64(0), cgLen, "compaction group length; PlanLevel(3)")
+	require.Zero(t, cgLen, "compaction group length; PlanLevel(3)")
 
-	_, cgLen = cp.Plan(time.Now().Add(-1))
-	require.Equal(t, int64(0), cgLen, "compaction group length; Plan()")
+	tsmP, pLenP := cp.Plan(time.Now().Add(-time.Second))
+	require.Zero(t, len(tsmP), "compaction group; Plan()")
+	require.Zero(t, pLenP, "compaction group length; Plan()")
 
 	_, cgLen, genLen := cp.PlanOptimize()
 	require.Equal(t, int64(1), cgLen, "compaction group length")
@@ -2603,14 +2606,15 @@ func TestDefaultPlanner_FullyCompacted_LargeSingleGenerationMaxAggressiveBlocks(
 	require.True(t, compacted, "is fully compacted")
 
 	_, cgLen := cp.PlanLevel(1)
-	require.Equal(t, int64(0), cgLen, "compaction group length; PlanLevel(1)")
+	require.Zero(t, cgLen, "compaction group length; PlanLevel(1)")
 	_, cgLen = cp.PlanLevel(2)
-	require.Equal(t, int64(0), cgLen, "compaction group length; PlanLevel(2)")
+	require.Zero(t, cgLen, "compaction group length; PlanLevel(2)")
 	_, cgLen = cp.PlanLevel(3)
-	require.Equal(t, int64(0), cgLen, "compaction group length; PlanLevel(3)")
+	require.Zero(t, cgLen, "compaction group length; PlanLevel(3)")
 
-	_, cgLen = cp.Plan(time.Now().Add(-1))
-	require.Equal(t, int64(0), cgLen, "compaction group length; Plan()")
+	tsmP, pLenP := cp.Plan(time.Now().Add(-time.Second))
+	require.Zero(t, len(tsmP), "compaction group; Plan()")
+	require.Zero(t, pLenP, "compaction group length; Plan()")
 
 	cgroup, cgLen, genLen := cp.PlanOptimize()
 	require.Equal(t, []tsm1.CompactionGroup(nil), cgroup, "compaction group")
@@ -2656,14 +2660,15 @@ func TestDefaultPlanner_FullyCompacted_LargeSingleGenerationNoMaxAggrBlocks(t *t
 	require.True(t, compacted, "is fully compacted")
 
 	_, cgLen := cp.PlanLevel(1)
-	require.Equal(t, int64(0), cgLen, "compaction group length; PlanLevel(1)")
+	require.Zero(t, cgLen, "compaction group length; PlanLevel(1)")
 	_, cgLen = cp.PlanLevel(2)
-	require.Equal(t, int64(0), cgLen, "compaction group length; PlanLevel(2)")
+	require.Zero(t, cgLen, "compaction group length; PlanLevel(2)")
 	_, cgLen = cp.PlanLevel(3)
-	require.Equal(t, int64(0), cgLen, "compaction group length; PlanLevel(3)")
+	require.Zero(t, cgLen, "compaction group length; PlanLevel(3)")
 
-	_, cgLen = cp.Plan(time.Now().Add(-1))
-	require.Equal(t, int64(0), cgLen, "compaction group length; Plan()")
+	tsmP, pLenP := cp.Plan(time.Now().Add(-time.Second))
+	require.Zero(t, len(tsmP), "compaction group; Plan()")
+	require.Zero(t, pLenP, "compaction group length; Plan()")
 
 	cgroup, cgLen, genLen := cp.PlanOptimize()
 	require.Equal(t, []tsm1.CompactionGroup(nil), cgroup, "compaction group")
@@ -2712,19 +2717,20 @@ func TestDefaultPlanner_FullyCompacted_ManySingleGenLessThen2GBMaxAggrBlocks(t *
 	require.True(t, compacted, "is fully compacted")
 
 	_, cgLen := cp.PlanLevel(1)
-	require.Equal(t, int64(0), cgLen, "compaction group length; PlanLevel(1)")
+	require.Zero(t, cgLen, "compaction group length; PlanLevel(1)")
 	_, cgLen = cp.PlanLevel(2)
-	require.Equal(t, int64(0), cgLen, "compaction group length; PlanLevel(2)")
+	require.Zero(t, cgLen, "compaction group length; PlanLevel(2)")
 	_, cgLen = cp.PlanLevel(3)
-	require.Equal(t, int64(0), cgLen, "compaction group length; PlanLevel(3)")
+	require.Zero(t, cgLen, "compaction group length; PlanLevel(3)")
 
-	_, cgLen = cp.Plan(time.Now().Add(-1))
-	require.Equal(t, int64(0), cgLen, "compaction group length; Plan()")
+	tsmP, pLenP := cp.Plan(time.Now().Add(-time.Second))
+	require.Zero(t, len(tsmP), "compaction group; Plan()")
+	require.Zero(t, pLenP, "compaction group length; Plan()")
 
 	cgroup, cgLen, genLen := cp.PlanOptimize()
 	require.Equal(t, []tsm1.CompactionGroup(nil), cgroup, "compaction group")
-	require.Equal(t, int64(0), cgLen, "compaction group length")
-	require.Equal(t, int64(0), genLen, "generation count")
+	require.Zero(t, cgLen, "compaction group length")
+	require.Zero(t, genLen, "generation count")
 }
 
 // This test is added to account for a single generation that has a group size
@@ -2768,14 +2774,15 @@ func TestDefaultPlanner_FullyCompacted_ManySingleGenLessThen2GBNotMaxAggrBlocks(
 	require.False(t, compacted, "is fully compacted")
 
 	_, cgLen := cp.PlanLevel(1)
-	require.Equal(t, int64(0), cgLen, "compaction group length; PlanLevel(1)")
+	require.Zero(t, cgLen, "compaction group length; PlanLevel(1)")
 	_, cgLen = cp.PlanLevel(2)
-	require.Equal(t, int64(0), cgLen, "compaction group length; PlanLevel(2)")
+	require.Zero(t, cgLen, "compaction group length; PlanLevel(2)")
 	_, cgLen = cp.PlanLevel(3)
-	require.Equal(t, int64(0), cgLen, "compaction group length; PlanLevel(3)")
+	require.Zero(t, cgLen, "compaction group length; PlanLevel(3)")
 
-	_, cgLen = cp.Plan(time.Now().Add(-1))
-	require.Equal(t, int64(0), cgLen, "compaction group length; Plan()")
+	tsmP, pLenP := cp.Plan(time.Now().Add(-time.Second))
+	require.Zero(t, len(tsmP), "compaction group; Plan()")
+	require.Zero(t, pLenP, "compaction group length; Plan()")
 
 	_, cgLen, genLen := cp.PlanOptimize()
 	require.Equal(t, int64(1), cgLen, "compaction group length")
@@ -2853,14 +2860,15 @@ func TestDefaultPlanner_FullyCompacted_ManySingleGen2GBLastLevel2(t *testing.T)
 	}
 
 	_, cgLen := cp.PlanLevel(1)
-	require.Equal(t, int64(0), cgLen, "compaction group length; PlanLevel(1)")
+	require.Zero(t, cgLen, "compaction group length; PlanLevel(1)")
 	_, cgLen = cp.PlanLevel(2)
-	require.Equal(t, int64(0), cgLen, "compaction group length; PlanLevel(2)")
+	require.Zero(t, cgLen, "compaction group length; PlanLevel(2)")
 	_, cgLen = cp.PlanLevel(3)
-	require.Equal(t, int64(0), cgLen, "compaction group length; PlanLevel(3)")
+	require.Zero(t, cgLen, "compaction group length; PlanLevel(3)")
 
-	_, cgLen = cp.Plan(time.Now().Add(-1))
-	require.Equal(t, int64(0), cgLen, "compaction group length; Plan()")
+	tsmP, pLenP := cp.Plan(time.Now().Add(-time.Second))
+	require.Zero(t, len(tsmP), "compaction group; Plan()")
+	require.Zero(t, pLenP, "compaction group length; Plan()")
 
 	tsm, cgLen, genLen := cp.PlanOptimize()
 	require.Equal(t, int64(1), cgLen, "compaction group length")

From 5e4e2da881522616be679635ccbd1db864e60ce3 Mon Sep 17 00:00:00 2001
From: Devan <devandbenz@gmail.com>
Date: Thu, 26 Dec 2024 12:23:18 -0600
Subject: [PATCH 21/31] feat: Adds test for planning lower level TSMs with
 block sizes at aggressive block count

---
 tsdb/engine/tsm1/compact_test.go | 60 ++++++++++++++++++++++++++++++++
 1 file changed, 60 insertions(+)

diff --git a/tsdb/engine/tsm1/compact_test.go b/tsdb/engine/tsm1/compact_test.go
index 32258104010..240c977ff44 100644
--- a/tsdb/engine/tsm1/compact_test.go
+++ b/tsdb/engine/tsm1/compact_test.go
@@ -2876,6 +2876,66 @@ func TestDefaultPlanner_FullyCompacted_ManySingleGen2GBLastLevel2(t *testing.T)
 	require.Equal(t, len(expFiles), len(tsm[0]), "tsm files in compaction group")
 }
 
+// This test will check to ensure that any TSM generations planned with the default planner
+// using Plan() and PlanLevel() over default block size are skipped.
+func TestDefaultPlanner_PlanOverAggressiveBlocks(t *testing.T) {
+	data := []tsm1.FileStat{
+		{
+			Path: "01-02.tsm1",
+			Size: 251 * 1024 * 1024,
+		},
+		{
+			Path: "01-03.tsm1",
+			Size: 1 * 1024 * 1024,
+		},
+		{
+			Path: "02-02.tsm1",
+			Size: 251 * 1024 * 1024,
+		},
+		{
+			Path: "02-03.tsm1",
+			Size: 1 * 1024 * 1024,
+		},
+		{
+			Path: "03-02.tsm1",
+			Size: 251 * 1024 * 1024,
+		},
+		{
+			Path: "03-03.tsm1",
+			Size: 1 * 1024 * 1024,
+		},
+	}
+
+	fs := &fakeFileStore{
+		PathsFn: func() []tsm1.FileStat {
+			return data
+		},
+	}
+	blocks := []int{
+		tsdb.AggressiveMaxPointsPerBlock,
+		tsdb.AggressiveMaxPointsPerBlock,
+		tsdb.AggressiveMaxPointsPerBlock,
+		tsdb.AggressiveMaxPointsPerBlock,
+		tsdb.AggressiveMaxPointsPerBlock,
+		tsdb.AggressiveMaxPointsPerBlock,
+	}
+	err := fs.SetBlockCounts(blocks)
+	require.NoError(t, err, "SetBlockCounts")
+
+	cp := tsm1.NewDefaultPlanner(fs, tsdb.DefaultCompactFullWriteColdDuration)
+
+	_, cgLen := cp.PlanLevel(1)
+	require.Zero(t, cgLen, "compaction group length; PlanLevel(1)")
+	_, cgLen = cp.PlanLevel(2)
+	require.Zero(t, cgLen, "compaction group length; PlanLevel(2)")
+	_, cgLen = cp.PlanLevel(3)
+	require.Zero(t, cgLen, "compaction group length; PlanLevel(3)")
+
+	tsmP, pLenP := cp.Plan(time.Now().Add(-time.Second))
+	require.Zero(t, len(tsmP), "compaction group; Plan()")
+	require.Zero(t, pLenP, "compaction group length; Plan()")
+}
+
 func TestDefaultPlanner_PlanOptimize_Tombstones(t *testing.T) {
 	data := []tsm1.FileStat{
 		{

From c315b1f2d2f3a8408a823016f423b01bb5196055 Mon Sep 17 00:00:00 2001
From: Devan <devandbenz@gmail.com>
Date: Thu, 26 Dec 2024 13:34:03 -0600
Subject: [PATCH 22/31] chore: rerun ci


From eb0a77dd35902aeec25a24d85a50d4562106c949 Mon Sep 17 00:00:00 2001
From: Devan <devandbenz@gmail.com>
Date: Thu, 26 Dec 2024 14:27:49 -0600
Subject: [PATCH 23/31] feat: Add a mock backfill test with mixed generations,
 mixed levels, and mixed block counts

---
 tsdb/engine/tsm1/compact_test.go | 109 +++++++++++++++++++++++++++++++
 1 file changed, 109 insertions(+)

diff --git a/tsdb/engine/tsm1/compact_test.go b/tsdb/engine/tsm1/compact_test.go
index 240c977ff44..95ce7de1177 100644
--- a/tsdb/engine/tsm1/compact_test.go
+++ b/tsdb/engine/tsm1/compact_test.go
@@ -2936,6 +2936,115 @@ func TestDefaultPlanner_PlanOverAggressiveBlocks(t *testing.T) {
 	require.Zero(t, pLenP, "compaction group length; Plan()")
 }
 
+// This test will mock a 'backfill' condition where we have a single
+// shard with many generations. The initial generation should be fully
+// compacted, but we have some new generations that are not. We need to ensure
+// the optimize planner will pick these up and compact everything together.
+func TestDefaultPlanner_BackfillMock(t *testing.T) {
+	// > 2 GB total group size
+	// 50% of files are at aggressive max block size
+	data := []tsm1.FileStat{
+		{
+			Path: "01-05.tsm1",
+			Size: 2048 * 1024 * 1024,
+		},
+		{
+			Path: "01-06.tsm1",
+			Size: 2048 * 1024 * 1024,
+		},
+		{
+			Path: "01-07.tsm1",
+			Size: 2048 * 1024 * 1024,
+		},
+		{
+			Path: "02-04.tsm1",
+			Size: 700 * 1024 * 1024,
+		},
+		{
+			Path: "02-05.tsm1",
+			Size: 500 * 1024 * 1024,
+		},
+		{
+			Path: "02-06.tsm1",
+			Size: 400 * 1024 * 1024,
+		},
+		{
+			Path: "03-02.tsm1",
+			Size: 700 * 1024 * 1024,
+		},
+		{
+			Path: "03-03.tsm1",
+			Size: 500 * 1024 * 1024,
+		},
+		{
+			Path: "03-04.tsm1",
+			Size: 400 * 1024 * 1024,
+		},
+		{
+			Path: "04-01.tsm1",
+			Size: 700 * 1024 * 1024,
+		},
+		{
+			Path: "04-02.tsm1",
+			Size: 500 * 1024 * 1024,
+		},
+		{
+			Path: "03-03.tsm1",
+			Size: 400 * 1024 * 1024,
+		},
+	}
+
+	fs := &fakeFileStore{
+		PathsFn: func() []tsm1.FileStat {
+			return data
+		},
+	}
+	blocks := []int{
+		tsdb.AggressiveMaxPointsPerBlock,
+		tsdb.AggressiveMaxPointsPerBlock,
+		tsdb.AggressiveMaxPointsPerBlock,
+
+		tsdb.AggressiveMaxPointsPerBlock,
+		tsdb.AggressiveMaxPointsPerBlock,
+		tsdb.DefaultMaxPointsPerBlock,
+
+		tsdb.AggressiveMaxPointsPerBlock,
+		tsdb.AggressiveMaxPointsPerBlock,
+		tsdb.DefaultMaxPointsPerBlock,
+
+		tsdb.DefaultMaxPointsPerBlock,
+		// Use some magic numbers but these are just small values for block counts
+		100,
+		10,
+	}
+	err := fs.SetBlockCounts(blocks)
+	require.NoError(t, err, "SetBlockCounts")
+
+	cp := tsm1.NewDefaultPlanner(fs, tsdb.DefaultCompactFullWriteColdDuration)
+
+	_, cgLen := cp.PlanLevel(1)
+	require.Zero(t, cgLen, "compaction group length; PlanLevel(1)")
+	_, cgLen = cp.PlanLevel(2)
+	require.Zero(t, cgLen, "compaction group length; PlanLevel(2)")
+	_, cgLen = cp.PlanLevel(3)
+	require.Zero(t, cgLen, "compaction group length; PlanLevel(3)")
+
+	tsmP, pLenP := cp.Plan(time.Now().Add(-time.Second))
+	require.Zero(t, len(tsmP), "compaction group; Plan()")
+	require.Zero(t, pLenP, "compaction group length; Plan()")
+
+	expFiles := make([]tsm1.FileStat, 0)
+	for _, file := range data {
+		expFiles = append(expFiles, file)
+	}
+
+	tsm, cgLen, genLen := cp.PlanOptimize()
+	require.Equal(t, len(expFiles), len(tsm[0]), "expected TSM files")
+	require.Equal(t, int64(1), cgLen, "compaction group length")
+	// Should pick up 4 generations for compaction
+	require.Equal(t, int64(4), genLen, "generation count")
+}
+
 func TestDefaultPlanner_PlanOptimize_Tombstones(t *testing.T) {
 	data := []tsm1.FileStat{
 		{

From 371f96066474a87620055dcfc60a980bd16c4a2e Mon Sep 17 00:00:00 2001
From: Devan <devandbenz@gmail.com>
Date: Mon, 6 Jan 2025 15:57:20 -0600
Subject: [PATCH 24/31] feat: Fix a merge conflict where a var was renamed from
 fs -> fss

---
 tsdb/engine/tsm1/compact_test.go | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/tsdb/engine/tsm1/compact_test.go b/tsdb/engine/tsm1/compact_test.go
index cf00144fe11..9e9f8ae3b2a 100644
--- a/tsdb/engine/tsm1/compact_test.go
+++ b/tsdb/engine/tsm1/compact_test.go
@@ -14,8 +14,8 @@ import (
 
 	"github.com/influxdata/influxdb/tsdb"
 	"github.com/influxdata/influxdb/tsdb/engine/tsm1"
-	"github.com/stretchr/testify/require"
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
 	"go.uber.org/zap"
 )
 
@@ -3256,7 +3256,7 @@ func TestDefaultPlanner_Plan_SkipPlanningAfterFull(t *testing.T) {
 
 	err := ffs.SetBlockCounts([]int{tsdb.DefaultMaxPointsPerBlock, tsdb.DefaultMaxPointsPerBlock, tsdb.DefaultMaxPointsPerBlock, tsdb.DefaultMaxPointsPerBlock})
 	require.NoError(t, err, "SetBlockCounts")
-  
+
 	cp := tsm1.NewDefaultPlanner(ffs, time.Nanosecond)
 
 	plan, pLen := cp.Plan(time.Now().Add(-time.Second))
@@ -3441,7 +3441,7 @@ func TestDefaultPlanner_Plan_NotFullOverMaxsize(t *testing.T) {
 		bcs = append(bcs, 100)
 	}
 
-	err := fs.SetBlockCounts(bcs)
+	err := ffs.SetBlockCounts(bcs)
 	require.NoError(t, err, "SetBlockCounts")
 
 	cp := tsm1.NewDefaultPlanner(

From 5a614c491275a103e282ad0bc045e7f0aef4a2ac Mon Sep 17 00:00:00 2001
From: devanbenz <devandbenz@gmail.com>
Date: Thu, 9 Jan 2025 15:52:36 -0600
Subject: [PATCH 25/31] feat: Adding more tests reversing and mixing up some of
 the file sizes and block counts

---
 tsdb/engine/tsm1/compact_test.go | 686 +++++++++++++++++++++++++++----
 1 file changed, 614 insertions(+), 72 deletions(-)

diff --git a/tsdb/engine/tsm1/compact_test.go b/tsdb/engine/tsm1/compact_test.go
index 9e9f8ae3b2a..f0faa7e5c0b 100644
--- a/tsdb/engine/tsm1/compact_test.go
+++ b/tsdb/engine/tsm1/compact_test.go
@@ -2345,6 +2345,80 @@ func TestDefaultPlanner_PlanOptimize_LargeMultiGeneration(t *testing.T) {
 	require.Equal(t, len(expFiles), len(tsm[0]), "expected TSM files")
 }
 
+func TestDefaultPlanner_PlanOptimize_LargeMultiGenerationSmallFirstFiles(t *testing.T) {
+	data := []tsm1.FileStat{
+		{
+			Path: "03-04.tsm1",
+			Size: 2048 * 1024 * 1024,
+		},
+		{
+			Path: "03-05.tsm1",
+			Size: 512 * 1024 * 1024,
+		},
+		{
+			Path: "01-05.tsm1",
+			Size: 2048 * 1024 * 1024,
+		},
+		{
+			Path: "01-06.tsm1",
+			Size: 2048 * 1024 * 1024,
+		},
+		{
+			Path: "01-07.tsm1",
+			Size: 2048 * 1024 * 1024,
+		},
+		{
+			Path: "01-08.tsm1",
+			Size: 1048 * 1024 * 1024,
+		},
+		{
+			Path: "02-05.tsm1",
+			Size: 2048 * 1024 * 1024,
+		},
+		{
+			Path: "02-06.tsm1",
+			Size: 2048 * 1024 * 1024,
+		},
+		{
+			Path: "02-07.tsm1",
+			Size: 2048 * 1024 * 1024,
+		},
+		{
+			Path: "02-08.tsm1",
+			Size: 1048 * 1024 * 1024,
+		},
+	}
+
+	cp := tsm1.NewDefaultPlanner(
+		&fakeFileStore{
+			PathsFn: func() []tsm1.FileStat {
+				return data
+			},
+		}, tsdb.DefaultCompactFullWriteColdDuration,
+	)
+
+	expFiles := make([]tsm1.FileStat, 0)
+	for _, file := range data {
+		expFiles = append(expFiles, file)
+	}
+
+	_, cgLen := cp.PlanLevel(1)
+	require.Zero(t, cgLen, "compaction group length; PlanLevel(1)")
+	_, cgLen = cp.PlanLevel(2)
+	require.Zero(t, cgLen, "compaction group length; PlanLevel(2)")
+	_, cgLen = cp.PlanLevel(3)
+	require.Zero(t, cgLen, "compaction group length; PlanLevel(3)")
+
+	tsmP, pLenP := cp.Plan(time.Now().Add(-time.Second))
+	require.Zero(t, len(tsmP), "compaction group; Plan()")
+	require.Zero(t, pLenP, "compaction group length; Plan()")
+
+	tsm, pLen, _ := cp.PlanOptimize()
+	require.Equal(t, 1, len(tsm), "compaction group")
+	require.Equal(t, int64(len(tsm)), pLen, "compaction group length")
+	require.Equal(t, len(expFiles), len(tsm[0]), "expected TSM files")
+}
+
 // This test is added to account for a single generation that has a group size
 // under 2 GB so it should be further compacted to a single file.
 func TestDefaultPlanner_PlanOptimize_SmallSingleGeneration(t *testing.T) {
@@ -2399,6 +2473,58 @@ func TestDefaultPlanner_PlanOptimize_SmallSingleGeneration(t *testing.T) {
 	require.Equal(t, len(expFiles), len(tsm[0]), "expected TSM files")
 }
 
+func TestDefaultPlanner_PlanOptimize_SmallSingleGenerationReverse(t *testing.T) {
+	// ~650 MB total group size
+	data := []tsm1.FileStat{
+		{
+			Path: "01-05.tsm1",
+			Size: 50 * 1024 * 1024,
+		},
+		{
+			Path: "01-06.tsm1",
+			Size: 100 * 1024 * 1024,
+		},
+		{
+			Path: "01-07.tsm1",
+			Size: 200 * 1024 * 1024,
+		},
+		{
+			Path: "01-08.tsm1",
+			Size: 300 * 1024 * 1024,
+		},
+	}
+
+	cp := tsm1.NewDefaultPlanner(
+		&fakeFileStore{
+			PathsFn: func() []tsm1.FileStat {
+				return data
+			},
+		}, tsdb.DefaultCompactFullWriteColdDuration,
+	)
+
+	expFiles := make([]tsm1.FileStat, 0)
+	for _, file := range data {
+		expFiles = append(expFiles, file)
+	}
+
+	_, cgLen := cp.PlanLevel(1)
+	require.Zero(t, cgLen, "compaction group length; PlanLevel(1)")
+	_, cgLen = cp.PlanLevel(2)
+	require.Zero(t, cgLen, "compaction group length; PlanLevel(2)")
+	_, cgLen = cp.PlanLevel(3)
+	require.Zero(t, cgLen, "compaction group length; PlanLevel(3)")
+
+	tsmP, pLenP := cp.Plan(time.Now().Add(-time.Second))
+	require.Zero(t, len(tsmP), "compaction group; Plan()")
+	require.Zero(t, pLenP, "compaction group length; Plan()")
+
+	tsm, pLen, gLen := cp.PlanOptimize()
+	require.Equal(t, 1, len(tsm), "compaction group")
+	require.Equal(t, int64(len(tsm)), pLen, "compaction group length")
+	require.Equal(t, int64(1), gLen, "generation of TSM files")
+	require.Equal(t, len(expFiles), len(tsm[0]), "expected TSM files")
+}
+
 // This test is added to account for a single generation that has a group size
 // under 2 GB and has less then level 4 files it should be further compacted to a single file.
 func TestDefaultPlanner_PlanOptimize_SmallSingleGenerationUnderLevel4(t *testing.T) {
@@ -2449,6 +2575,54 @@ func TestDefaultPlanner_PlanOptimize_SmallSingleGenerationUnderLevel4(t *testing
 	require.Equal(t, len(expFiles), len(tsm[0]), "expected TSM files")
 }
 
+func TestDefaultPlanner_PlanOptimize_SmallSingleGenerationUnderLevel4SmallFileFirst(t *testing.T) {
+	// ~650 MB total group size
+	data := []tsm1.FileStat{
+		{
+			Path: "01-02.tsm1",
+			Size: 100 * 1024 * 1024,
+		},
+		{
+			Path: "01-03.tsm1",
+			Size: 200 * 1024 * 1024,
+		},
+		{
+			Path: "01-04.tsm1",
+			Size: 300 * 1024 * 1024,
+		},
+	}
+
+	cp := tsm1.NewDefaultPlanner(
+		&fakeFileStore{
+			PathsFn: func() []tsm1.FileStat {
+				return data
+			},
+		}, tsdb.DefaultCompactFullWriteColdDuration,
+	)
+
+	expFiles := make([]tsm1.FileStat, 0)
+	for _, file := range data {
+		expFiles = append(expFiles, file)
+	}
+
+	_, cgLen := cp.PlanLevel(1)
+	require.Zero(t, cgLen, "compaction group length; PlanLevel(1)")
+	_, cgLen = cp.PlanLevel(2)
+	require.Zero(t, cgLen, "compaction group length; PlanLevel(2)")
+	_, cgLen = cp.PlanLevel(3)
+	require.Zero(t, cgLen, "compaction group length; PlanLevel(3)")
+
+	tsmP, pLenP := cp.Plan(time.Now().Add(-time.Second))
+	require.Zero(t, len(tsmP), "compaction group; Plan()")
+	require.Zero(t, pLenP, "compaction group length; Plan()")
+
+	tsm, pLen, gLen := cp.PlanOptimize()
+	require.Equal(t, 1, len(tsm), "compaction group")
+	require.Equal(t, int64(len(tsm)), pLen, "compaction group length")
+	require.Equal(t, int64(1), gLen, "generation of TSM files")
+	require.Equal(t, len(expFiles), len(tsm[0]), "expected TSM files")
+}
+
 // This test is added to account for a single generation that has a group size
 // under 2 GB and all files at max default points per block of 1000.
 // This should be planned for compaction at a more aggressive points per block.
@@ -2483,7 +2657,258 @@ func TestDefaultPlanner_FullyCompacted_SmallSingleGeneration(t *testing.T) {
 	require.NoError(t, err, "SetBlockCounts")
 
 	cp := tsm1.NewDefaultPlanner(fs, tsdb.DefaultCompactFullWriteColdDuration)
-
+
+	compacted, reason := cp.FullyCompacted()
+	require.Equal(t, reason, tsdb.SingleGenerationReasonText, "fullyCompacted reason")
+	require.False(t, compacted, "is fully compacted")
+
+	_, cgLen := cp.PlanLevel(1)
+	require.Zero(t, cgLen, "compaction group length; PlanLevel(1)")
+	_, cgLen = cp.PlanLevel(2)
+	require.Zero(t, cgLen, "compaction group length; PlanLevel(2)")
+	_, cgLen = cp.PlanLevel(3)
+	require.Zero(t, cgLen, "compaction group length; PlanLevel(3)")
+
+	tsmP, pLenP := cp.Plan(time.Now().Add(-time.Second))
+	require.Zero(t, len(tsmP), "compaction group; Plan()")
+	require.Zero(t, pLenP, "compaction group length; Plan()")
+
+	_, cgLen, genLen := cp.PlanOptimize()
+	require.Equal(t, int64(1), cgLen, "compaction group length")
+	require.Equal(t, int64(1), genLen, "generation count")
+}
+
+func TestDefaultPlanner_FullyCompacted_SmallSingleGenerationSmallFileFirst(t *testing.T) {
+	// ~650 MB total group size
+	data := []tsm1.FileStat{
+		{
+			Path: "01-05.tsm1",
+			Size: 50 * 1024 * 1024,
+		},
+		{
+			Path: "01-06.tsm1",
+			Size: 100 * 1024 * 1024,
+		},
+		{
+			Path: "01-07.tsm1",
+			Size: 200 * 1024 * 1024,
+		},
+		{
+			Path: "01-08.tsm1",
+			Size: 300 * 1024 * 1024,
+		},
+	}
+
+	fs := &fakeFileStore{
+		PathsFn: func() []tsm1.FileStat {
+			return data
+		},
+	}
+
+	err := fs.SetBlockCounts([]int{tsdb.DefaultMaxPointsPerBlock, tsdb.DefaultMaxPointsPerBlock, tsdb.DefaultMaxPointsPerBlock, tsdb.DefaultMaxPointsPerBlock})
+	require.NoError(t, err, "SetBlockCounts")
+
+	cp := tsm1.NewDefaultPlanner(fs, tsdb.DefaultCompactFullWriteColdDuration)
+
+	compacted, reason := cp.FullyCompacted()
+	require.Equal(t, reason, tsdb.SingleGenerationReasonText, "fullyCompacted reason")
+	require.False(t, compacted, "is fully compacted")
+
+	_, cgLen := cp.PlanLevel(1)
+	require.Zero(t, cgLen, "compaction group length; PlanLevel(1)")
+	_, cgLen = cp.PlanLevel(2)
+	require.Zero(t, cgLen, "compaction group length; PlanLevel(2)")
+	_, cgLen = cp.PlanLevel(3)
+	require.Zero(t, cgLen, "compaction group length; PlanLevel(3)")
+
+	tsmP, pLenP := cp.Plan(time.Now().Add(-time.Second))
+	require.Zero(t, len(tsmP), "compaction group; Plan()")
+	require.Zero(t, pLenP, "compaction group length; Plan()")
+
+	_, cgLen, genLen := cp.PlanOptimize()
+	require.Equal(t, int64(1), cgLen, "compaction group length")
+	require.Equal(t, int64(1), genLen, "generation count")
+}
+
+// This test is added to account for halting state after
+// TestDefaultPlanner_FullyCompacted_SmallSingleGeneration
+// will need to ensure that once we have single TSM file under 2 GB we stop
+func TestDefaultPlanner_FullyCompacted_SmallSingleGeneration_Halt(t *testing.T) {
+	// ~650 MB total group size
+	data := []tsm1.FileStat{
+		{
+			Path: "01-09.tsm1",
+			Size: 650 * 1024 * 1024,
+		},
+	}
+
+	cp := tsm1.NewDefaultPlanner(
+		&fakeFileStore{
+			PathsFn: func() []tsm1.FileStat {
+				return data
+			},
+		}, tsdb.DefaultCompactFullWriteColdDuration,
+	)
+
+	compacted, reason := cp.FullyCompacted()
+	reasonExp := ""
+	require.Equal(t, reason, reasonExp, "fullyCompacted reason")
+	require.True(t, compacted, "is fully compacted")
+
+	_, cgLen := cp.PlanLevel(1)
+	require.Zero(t, cgLen, "compaction group length; PlanLevel(1)")
+	_, cgLen = cp.PlanLevel(2)
+	require.Zero(t, cgLen, "compaction group length; PlanLevel(2)")
+	_, cgLen = cp.PlanLevel(3)
+	require.Zero(t, cgLen, "compaction group length; PlanLevel(3)")
+
+	tsmP, pLenP := cp.Plan(time.Now().Add(-time.Second))
+	require.Zero(t, len(tsmP), "compaction group; Plan()")
+	require.Zero(t, pLenP, "compaction group length; Plan()")
+
+	cgroup, cgLen, genLen := cp.PlanOptimize()
+	require.Equal(t, []tsm1.CompactionGroup(nil), cgroup, "compaction group")
+	require.Zero(t, cgLen, "compaction group length")
+	require.Zero(t, genLen, "generation count")
+}
+
+// This test is added to account for a single generation that has a group size
+// under 2 GB and a mix of aggressive max blocks and default max blocks
+// it should be further compacted.
+func TestDefaultPlanner_FullyCompacted_LargeSingleGenerationUnderAggressiveBlocks(t *testing.T) {
+	// > 2 GB total group size
+	// 50% of files are at aggressive max block size
+	data := []tsm1.FileStat{
+		{
+			Path: "01-05.tsm1",
+			Size: 700 * 1024 * 1024,
+		},
+		{
+			Path: "01-06.tsm1",
+			Size: 500 * 1024 * 1024,
+		},
+		{
+			Path: "01-07.tsm1",
+			Size: 400 * 1024 * 1024,
+		},
+		{
+			Path: "01-08.tsm1",
+			Size: 300 * 1024 * 1024,
+		},
+		{
+			Path: "01-09.tsm1",
+			Size: 200 * 1024 * 1024,
+		},
+		{
+			Path: "01-10.tsm1",
+			Size: 100 * 1024 * 1024,
+		},
+		{
+			Path: "01-11.tsm1",
+			Size: 50 * 1024 * 1024,
+		},
+		{
+			Path: "01-12.tsm1",
+			Size: 25 * 1024 * 1024,
+		},
+	}
+
+	fs := &fakeFileStore{
+		PathsFn: func() []tsm1.FileStat {
+			return data
+		},
+	}
+	blocks := []int{
+		tsdb.AggressiveMaxPointsPerBlock,
+		tsdb.AggressiveMaxPointsPerBlock,
+		tsdb.AggressiveMaxPointsPerBlock,
+		tsdb.AggressiveMaxPointsPerBlock,
+		tsdb.DefaultMaxPointsPerBlock,
+		tsdb.DefaultMaxPointsPerBlock,
+		tsdb.DefaultMaxPointsPerBlock,
+		tsdb.DefaultMaxPointsPerBlock,
+	}
+	err := fs.SetBlockCounts(blocks)
+	require.NoError(t, err, "SetBlockCounts")
+
+	cp := tsm1.NewDefaultPlanner(fs, tsdb.DefaultCompactFullWriteColdDuration)
+	compacted, reason := cp.FullyCompacted()
+	require.Equal(t, reason, tsdb.SingleGenerationReasonText, "fullyCompacted reason")
+	require.False(t, compacted, "is fully compacted")
+
+	_, cgLen := cp.PlanLevel(1)
+	require.Zero(t, cgLen, "compaction group length; PlanLevel(1)")
+	_, cgLen = cp.PlanLevel(2)
+	require.Zero(t, cgLen, "compaction group length; PlanLevel(2)")
+	_, cgLen = cp.PlanLevel(3)
+	require.Zero(t, cgLen, "compaction group length; PlanLevel(3)")
+
+	tsmP, pLenP := cp.Plan(time.Now().Add(-time.Second))
+	require.Zero(t, len(tsmP), "compaction group; Plan()")
+	require.Zero(t, pLenP, "compaction group length; Plan()")
+
+	_, cgLen, genLen := cp.PlanOptimize()
+	require.Equal(t, int64(1), cgLen, "compaction group length")
+	require.Equal(t, int64(1), genLen, "generation count")
+}
+
+func TestDefaultPlanner_FullyCompacted_LargeSingleGenerationUnderAggressiveBlocksMixFileSize(t *testing.T) {
+	// > 2 GB total group size
+	// 50% of files are at aggressive max block size
+	data := []tsm1.FileStat{
+		{
+			Path: "01-05.tsm1",
+			Size: 300 * 1024 * 1024,
+		},
+		{
+			Path: "01-06.tsm1",
+			Size: 700 * 1024 * 1024,
+		},
+		{
+			Path: "01-07.tsm1",
+			Size: 25 * 1024 * 1024,
+		},
+		{
+			Path: "01-08.tsm1",
+			Size: 400 * 1024 * 1024,
+		},
+		{
+			Path: "01-09.tsm1",
+			Size: 50 * 1024 * 1024,
+		},
+		{
+			Path: "01-10.tsm1",
+			Size: 100 * 1024 * 1024,
+		},
+		{
+			Path: "01-11.tsm1",
+			Size: 200 * 1024 * 1024,
+		},
+		{
+			Path: "01-12.tsm1",
+			Size: 500 * 1024 * 1024,
+		},
+	}
+
+	fs := &fakeFileStore{
+		PathsFn: func() []tsm1.FileStat {
+			return data
+		},
+	}
+	blocks := []int{
+		tsdb.AggressiveMaxPointsPerBlock,
+		tsdb.AggressiveMaxPointsPerBlock,
+		tsdb.AggressiveMaxPointsPerBlock,
+		tsdb.AggressiveMaxPointsPerBlock,
+		tsdb.DefaultMaxPointsPerBlock,
+		tsdb.DefaultMaxPointsPerBlock,
+		tsdb.DefaultMaxPointsPerBlock,
+		tsdb.DefaultMaxPointsPerBlock,
+	}
+	err := fs.SetBlockCounts(blocks)
+	require.NoError(t, err, "SetBlockCounts")
+
+	cp := tsm1.NewDefaultPlanner(fs, tsdb.DefaultCompactFullWriteColdDuration)
 	compacted, reason := cp.FullyCompacted()
 	require.Equal(t, reason, tsdb.SingleGenerationReasonText, "fullyCompacted reason")
 	require.False(t, compacted, "is fully compacted")
@@ -2504,25 +2929,36 @@ func TestDefaultPlanner_FullyCompacted_SmallSingleGeneration(t *testing.T) {
 	require.Equal(t, int64(1), genLen, "generation count")
 }
 
-// This test is added to account for halting state after
-// TestDefaultPlanner_FullyCompacted_SmallSingleGeneration
-// will need to ensure that once we have single TSM file under 2 GB we stop
-func TestDefaultPlanner_FullyCompacted_SmallSingleGeneration_Halt(t *testing.T) {
-	// ~650 MB total group size
+// This test is added to account for a single generation that has a group size
+// over 2 GB with 1 file under 2 GB all at max points per block with aggressive compaction.
+// It should not compact any further.
+func TestDefaultPlanner_FullyCompacted_LargeSingleGenerationMaxAggressiveBlocks(t *testing.T) {
+	// > 2 GB total group size
+	// 100% of files are at aggressive max block size
 	data := []tsm1.FileStat{
 		{
-			Path: "01-09.tsm1",
-			Size: 650 * 1024 * 1024,
+			Path: "01-13.tsm1",
+			Size: 2048 * 1024 * 1024,
+		},
+		{
+			Path: "01-14.tsm1",
+			Size: 691 * 1024 * 1024,
 		},
 	}
 
-	cp := tsm1.NewDefaultPlanner(
-		&fakeFileStore{
-			PathsFn: func() []tsm1.FileStat {
-				return data
-			},
-		}, tsdb.DefaultCompactFullWriteColdDuration,
-	)
+	fs := &fakeFileStore{
+		PathsFn: func() []tsm1.FileStat {
+			return data
+		},
+	}
+	blocks := []int{
+		tsdb.AggressiveMaxPointsPerBlock,
+		tsdb.AggressiveMaxPointsPerBlock,
+	}
+	err := fs.SetBlockCounts(blocks)
+	require.NoError(t, err, "SetBlockCounts")
+
+	cp := tsm1.NewDefaultPlanner(fs, tsdb.DefaultCompactFullWriteColdDuration)
 
 	compacted, reason := cp.FullyCompacted()
 	reasonExp := ""
@@ -2542,48 +2978,21 @@ func TestDefaultPlanner_FullyCompacted_SmallSingleGeneration_Halt(t *testing.T)
 
 	cgroup, cgLen, genLen := cp.PlanOptimize()
 	require.Equal(t, []tsm1.CompactionGroup(nil), cgroup, "compaction group")
-	require.Zero(t, cgLen, "compaction group length")
-	require.Zero(t, genLen, "generation count")
+	require.Equal(t, int64(0), cgLen, "compaction group length")
+	require.Equal(t, int64(0), genLen, "generation count")
 }
 
-// This test is added to account for a single generation that has a group size
-// under 2 GB and a mix of aggressive max blocks and default max blocks
-// it should be further compacted.
-func TestDefaultPlanner_FullyCompacted_LargeSingleGenerationUnderAggressiveBlocks(t *testing.T) {
+func TestDefaultPlanner_FullyCompacted_LargeSingleGenerationMaxAggressiveBlocksSmallFileFirst(t *testing.T) {
 	// > 2 GB total group size
-	// 50% of files are at aggressive max block size
+	// 100% of files are at aggressive max block size
 	data := []tsm1.FileStat{
 		{
-			Path: "01-05.tsm1",
-			Size: 700 * 1024 * 1024,
-		},
-		{
-			Path: "01-06.tsm1",
-			Size: 500 * 1024 * 1024,
-		},
-		{
-			Path: "01-07.tsm1",
-			Size: 400 * 1024 * 1024,
-		},
-		{
-			Path: "01-08.tsm1",
-			Size: 300 * 1024 * 1024,
-		},
-		{
-			Path: "01-09.tsm1",
-			Size: 200 * 1024 * 1024,
-		},
-		{
-			Path: "01-10.tsm1",
-			Size: 100 * 1024 * 1024,
-		},
-		{
-			Path: "01-11.tsm1",
-			Size: 50 * 1024 * 1024,
+			Path: "01-13.tsm1",
+			Size: 691 * 1024 * 1024,
 		},
 		{
-			Path: "01-12.tsm1",
-			Size: 25 * 1024 * 1024,
+			Path: "01-14.tsm1",
+			Size: 2048 * 1024 * 1024,
 		},
 	}
 
@@ -2595,20 +3004,16 @@ func TestDefaultPlanner_FullyCompacted_LargeSingleGenerationUnderAggressiveBlock
 	blocks := []int{
 		tsdb.AggressiveMaxPointsPerBlock,
 		tsdb.AggressiveMaxPointsPerBlock,
-		tsdb.AggressiveMaxPointsPerBlock,
-		tsdb.AggressiveMaxPointsPerBlock,
-		tsdb.DefaultMaxPointsPerBlock,
-		tsdb.DefaultMaxPointsPerBlock,
-		tsdb.DefaultMaxPointsPerBlock,
-		tsdb.DefaultMaxPointsPerBlock,
 	}
 	err := fs.SetBlockCounts(blocks)
 	require.NoError(t, err, "SetBlockCounts")
 
 	cp := tsm1.NewDefaultPlanner(fs, tsdb.DefaultCompactFullWriteColdDuration)
+
 	compacted, reason := cp.FullyCompacted()
-	require.Equal(t, reason, tsdb.SingleGenerationReasonText, "fullyCompacted reason")
-	require.False(t, compacted, "is fully compacted")
+	reasonExp := ""
+	require.Equal(t, reason, reasonExp, "fullyCompacted reason")
+	require.True(t, compacted, "is fully compacted")
 
 	_, cgLen := cp.PlanLevel(1)
 	require.Zero(t, cgLen, "compaction group length; PlanLevel(1)")
@@ -2621,15 +3026,17 @@ func TestDefaultPlanner_FullyCompacted_LargeSingleGenerationUnderAggressiveBlock
 	require.Zero(t, len(tsmP), "compaction group; Plan()")
 	require.Zero(t, pLenP, "compaction group length; Plan()")
 
-	_, cgLen, genLen := cp.PlanOptimize()
-	require.Equal(t, int64(1), cgLen, "compaction group length")
-	require.Equal(t, int64(1), genLen, "generation count")
+	cgroup, cgLen, genLen := cp.PlanOptimize()
+	require.Equal(t, []tsm1.CompactionGroup(nil), cgroup, "compaction group")
+	require.Equal(t, int64(0), cgLen, "compaction group length")
+	require.Equal(t, int64(0), genLen, "generation count")
 }
 
 // This test is added to account for a single generation that has a group size
-// over 2 GB with 1 file under 2 GB all at max points per block with aggressive compaction.
+// over 2 GB at max points per block with aggressive compaction, and, 1 file
+// under 2 GB at default max points per block.
 // It should not compact any further.
-func TestDefaultPlanner_FullyCompacted_LargeSingleGenerationMaxAggressiveBlocks(t *testing.T) {
+func TestDefaultPlanner_FullyCompacted_LargeSingleGenerationNoMaxAggrBlocks(t *testing.T) {
 	// > 2 GB total group size
 	// 100% of files are at aggressive max block size
 	data := []tsm1.FileStat{
@@ -2650,7 +3057,7 @@ func TestDefaultPlanner_FullyCompacted_LargeSingleGenerationMaxAggressiveBlocks(
 	}
 	blocks := []int{
 		tsdb.AggressiveMaxPointsPerBlock,
-		tsdb.AggressiveMaxPointsPerBlock,
+		tsdb.DefaultMaxPointsPerBlock,
 	}
 	err := fs.SetBlockCounts(blocks)
 	require.NoError(t, err, "SetBlockCounts")
@@ -2679,21 +3086,17 @@ func TestDefaultPlanner_FullyCompacted_LargeSingleGenerationMaxAggressiveBlocks(
 	require.Equal(t, int64(0), genLen, "generation count")
 }
 
-// This test is added to account for a single generation that has a group size
-// over 2 GB at max points per block with aggressive compaction, and, 1 file
-// under 2 GB at default max points per block.
-// It should not compact any further.
-func TestDefaultPlanner_FullyCompacted_LargeSingleGenerationNoMaxAggrBlocks(t *testing.T) {
+func TestDefaultPlanner_FullyCompacted_LargeSingleGenerationNoMaxAggrBlocksSmallFileFirst(t *testing.T) {
 	// > 2 GB total group size
 	// 100% of files are at aggressive max block size
 	data := []tsm1.FileStat{
 		{
 			Path: "01-13.tsm1",
-			Size: 2048 * 1024 * 1024,
+			Size: 691 * 1024 * 1024,
 		},
 		{
 			Path: "01-14.tsm1",
-			Size: 691 * 1024 * 1024,
+			Size: 2048 * 1024 * 1024,
 		},
 	}
 
@@ -2703,8 +3106,8 @@ func TestDefaultPlanner_FullyCompacted_LargeSingleGenerationNoMaxAggrBlocks(t *t
 		},
 	}
 	blocks := []int{
-		tsdb.AggressiveMaxPointsPerBlock,
 		tsdb.DefaultMaxPointsPerBlock,
+		tsdb.AggressiveMaxPointsPerBlock,
 	}
 	err := fs.SetBlockCounts(blocks)
 	require.NoError(t, err, "SetBlockCounts")
@@ -2790,6 +3193,61 @@ func TestDefaultPlanner_FullyCompacted_ManySingleGenLessThen2GBMaxAggrBlocks(t *
 	require.Zero(t, genLen, "generation count")
 }
 
+func TestDefaultPlanner_FullyCompacted_ManySingleGenLessThen2GBMaxAggrBlocksSmallFileFirst(t *testing.T) {
+	// > 2 GB total group size
+	// 100% of files are at aggressive max block size
+	data := []tsm1.FileStat{
+		{
+			Path: "01-13.tsm1",
+			Size: 450 * 1024 * 1024,
+		},
+		{
+			Path: "01-14.tsm1",
+			Size: 650 * 1024 * 1024,
+		},
+		{
+			Path: "01-15.tsm1",
+			Size: 2048 * 1024 * 1024,
+		},
+	}
+
+	fs := &fakeFileStore{
+		PathsFn: func() []tsm1.FileStat {
+			return data
+		},
+	}
+	blocks := []int{
+		tsdb.AggressiveMaxPointsPerBlock,
+		tsdb.AggressiveMaxPointsPerBlock,
+		tsdb.AggressiveMaxPointsPerBlock,
+	}
+	err := fs.SetBlockCounts(blocks)
+	require.NoError(t, err, "SetBlockCounts")
+
+	cp := tsm1.NewDefaultPlanner(fs, tsdb.DefaultCompactFullWriteColdDuration)
+
+	compacted, reason := cp.FullyCompacted()
+	reasonExp := ""
+	require.Equal(t, reason, reasonExp, "fullyCompacted reason")
+	require.True(t, compacted, "is fully compacted")
+
+	_, cgLen := cp.PlanLevel(1)
+	require.Zero(t, cgLen, "compaction group length; PlanLevel(1)")
+	_, cgLen = cp.PlanLevel(2)
+	require.Zero(t, cgLen, "compaction group length; PlanLevel(2)")
+	_, cgLen = cp.PlanLevel(3)
+	require.Zero(t, cgLen, "compaction group length; PlanLevel(3)")
+
+	tsmP, pLenP := cp.Plan(time.Now().Add(-time.Second))
+	require.Zero(t, len(tsmP), "compaction group; Plan()")
+	require.Zero(t, pLenP, "compaction group length; Plan()")
+
+	cgroup, cgLen, genLen := cp.PlanOptimize()
+	require.Equal(t, []tsm1.CompactionGroup(nil), cgroup, "compaction group")
+	require.Zero(t, cgLen, "compaction group length")
+	require.Zero(t, genLen, "generation count")
+}
+
 // This test is added to account for a single generation that has a group size
 // over 2 GB and multiple files under 2 GB with multiple files under aggressive
 // max points per block. This should further compact.
@@ -2933,6 +3391,90 @@ func TestDefaultPlanner_FullyCompacted_ManySingleGen2GBLastLevel2(t *testing.T)
 	require.Equal(t, len(expFiles), len(tsm[0]), "tsm files in compaction group")
 }
 
+func TestDefaultPlanner_FullyCompacted_ManySingleGen2GBLastLevel2MixedSizes(t *testing.T) {
+	// > 2 GB total group size
+	// 100% of files are at aggressive max block size
+	data := []tsm1.FileStat{
+		{
+			Path: "01-05.tsm1",
+			Size: 2048 * 1024 * 1024,
+		},
+		{
+			Path: "01-06.tsm1",
+			Size: 2048 * 1024 * 1024,
+		},
+		{
+			Path: "01-07.tsm1",
+			Size: 2048 * 1024 * 1024,
+		},
+		{
+			Path: "01-08.tsm1",
+			Size: 1048 * 1024 * 1024,
+		},
+		{
+			Path: "02-05.tsm1",
+			Size: 2048 * 1024 * 1024,
+		},
+		{
+			Path: "02-06.tsm1",
+			Size: 2048 * 1024 * 1024,
+		},
+		{
+			Path: "02-07.tsm1",
+			Size: 2048 * 1024 * 1024,
+		},
+		{
+			Path: "02-08.tsm1",
+			Size: 1048 * 1024 * 1024,
+		},
+		{
+			Path: "03-03.tsm1",
+			Size: 600 * 1024 * 1024,
+		},
+		{
+			Path: "03-04.tsm1",
+			Size: 500 * 1024 * 1024,
+		},
+		{
+			Path: "03-04.tsm1",
+			Size: 2048 * 1024 * 1024,
+		},
+		{
+			Path: "03-06.tsm1",
+			Size: 2048 * 1024 * 1024,
+		},
+	}
+
+	fs := &fakeFileStore{
+		PathsFn: func() []tsm1.FileStat {
+			return data
+		},
+	}
+
+	cp := tsm1.NewDefaultPlanner(fs, tsdb.DefaultCompactFullWriteColdDuration)
+
+	expFiles := make([]tsm1.FileStat, 0)
+	for _, file := range data {
+		expFiles = append(expFiles, file)
+	}
+
+	_, cgLen := cp.PlanLevel(1)
+	require.Zero(t, cgLen, "compaction group length; PlanLevel(1)")
+	_, cgLen = cp.PlanLevel(2)
+	require.Zero(t, cgLen, "compaction group length; PlanLevel(2)")
+	_, cgLen = cp.PlanLevel(3)
+	require.Zero(t, cgLen, "compaction group length; PlanLevel(3)")
+
+	tsmP, pLenP := cp.Plan(time.Now().Add(-time.Second))
+	require.Zero(t, len(tsmP), "compaction group; Plan()")
+	require.Zero(t, pLenP, "compaction group length; Plan()")
+
+	tsm, cgLen, genLen := cp.PlanOptimize()
+	require.Equal(t, int64(1), cgLen, "compaction group length")
+	require.Equal(t, int64(3), genLen, "generation count")
+	require.Equal(t, len(expFiles), len(tsm[0]), "tsm files in compaction group")
+}
+
 // This test will check to ensure that any TSM generations planned with the default planner
 // using Plan() and PlanLevel() over default block size are skipped.
 func TestDefaultPlanner_PlanOverAggressiveBlocks(t *testing.T) {

From 3748c364a0c78f4860bb1e1f5b785573f16941dc Mon Sep 17 00:00:00 2001
From: devanbenz <devandbenz@gmail.com>
Date: Mon, 13 Jan 2025 13:41:44 -0600
Subject: [PATCH 26/31] feat: Begin 'compacting' tests in to single test

---
 tsdb/engine/tsm1/compact_test.go | 1379 +++++++++---------------------
 1 file changed, 413 insertions(+), 966 deletions(-)

diff --git a/tsdb/engine/tsm1/compact_test.go b/tsdb/engine/tsm1/compact_test.go
index f0faa7e5c0b..05e8022990a 100644
--- a/tsdb/engine/tsm1/compact_test.go
+++ b/tsdb/engine/tsm1/compact_test.go
@@ -3,6 +3,7 @@ package tsm1_test
 import (
 	"errors"
 	"fmt"
+	"golang.org/x/exp/slices"
 	"io/fs"
 	"math"
 	"os"
@@ -2269,465 +2270,380 @@ func TestDefaultPlanner_PlanOptimize_NoLevel4(t *testing.T) {
 	}
 }
 
-// This test is added to acount for many TSM files within a group being over 2 GB
-// we want to ensure that the shard will be planned.
-func TestDefaultPlanner_PlanOptimize_LargeMultiGeneration(t *testing.T) {
-	data := []tsm1.FileStat{
-		{
-			Path: "01-05.tsm1",
-			Size: 2048 * 1024 * 1024,
-		},
-		{
-			Path: "01-06.tsm1",
-			Size: 2048 * 1024 * 1024,
-		},
-		{
-			Path: "01-07.tsm1",
-			Size: 2048 * 1024 * 1024,
-		},
-		{
-			Path: "01-08.tsm1",
-			Size: 1048 * 1024 * 1024,
-		},
-		{
-			Path: "02-05.tsm1",
-			Size: 2048 * 1024 * 1024,
-		},
-		{
-			Path: "02-06.tsm1",
-			Size: 2048 * 1024 * 1024,
-		},
-		{
-			Path: "02-07.tsm1",
-			Size: 2048 * 1024 * 1024,
-		},
-		{
-			Path: "02-08.tsm1",
-			Size: 1048 * 1024 * 1024,
-		},
-		{
-			Path: "03-04.tsm1",
-			Size: 2048 * 1024 * 1024,
-		},
-		{
-			Path: "03-05.tsm1",
-			Size: 512 * 1024 * 1024,
-		},
-	}
-
-	cp := tsm1.NewDefaultPlanner(
-		&fakeFileStore{
-			PathsFn: func() []tsm1.FileStat {
-				return data
-			},
-		}, tsdb.DefaultCompactFullWriteColdDuration,
-	)
-
-	expFiles := make([]tsm1.FileStat, 0)
-	for _, file := range data {
-		expFiles = append(expFiles, file)
-	}
-
-	_, cgLen := cp.PlanLevel(1)
-	require.Zero(t, cgLen, "compaction group length; PlanLevel(1)")
-	_, cgLen = cp.PlanLevel(2)
-	require.Zero(t, cgLen, "compaction group length; PlanLevel(2)")
-	_, cgLen = cp.PlanLevel(3)
-	require.Zero(t, cgLen, "compaction group length; PlanLevel(3)")
-
-	tsmP, pLenP := cp.Plan(time.Now().Add(-time.Second))
-	require.Zero(t, len(tsmP), "compaction group; Plan()")
-	require.Zero(t, pLenP, "compaction group length; Plan()")
-
-	tsm, pLen, _ := cp.PlanOptimize()
-	require.Equal(t, 1, len(tsm), "compaction group")
-	require.Equal(t, int64(len(tsm)), pLen, "compaction group length")
-	require.Equal(t, len(expFiles), len(tsm[0]), "expected TSM files")
-}
-
-func TestDefaultPlanner_PlanOptimize_LargeMultiGenerationSmallFirstFiles(t *testing.T) {
-	data := []tsm1.FileStat{
-		{
-			Path: "03-04.tsm1",
-			Size: 2048 * 1024 * 1024,
-		},
-		{
-			Path: "03-05.tsm1",
-			Size: 512 * 1024 * 1024,
-		},
-		{
-			Path: "01-05.tsm1",
-			Size: 2048 * 1024 * 1024,
-		},
-		{
-			Path: "01-06.tsm1",
-			Size: 2048 * 1024 * 1024,
-		},
-		{
-			Path: "01-07.tsm1",
-			Size: 2048 * 1024 * 1024,
-		},
-		{
-			Path: "01-08.tsm1",
-			Size: 1048 * 1024 * 1024,
-		},
-		{
-			Path: "02-05.tsm1",
-			Size: 2048 * 1024 * 1024,
-		},
-		{
-			Path: "02-06.tsm1",
-			Size: 2048 * 1024 * 1024,
-		},
-		{
-			Path: "02-07.tsm1",
-			Size: 2048 * 1024 * 1024,
-		},
-		{
-			Path: "02-08.tsm1",
-			Size: 1048 * 1024 * 1024,
-		},
-	}
-
-	cp := tsm1.NewDefaultPlanner(
-		&fakeFileStore{
-			PathsFn: func() []tsm1.FileStat {
-				return data
-			},
-		}, tsdb.DefaultCompactFullWriteColdDuration,
-	)
-
-	expFiles := make([]tsm1.FileStat, 0)
-	for _, file := range data {
-		expFiles = append(expFiles, file)
-	}
-
-	_, cgLen := cp.PlanLevel(1)
-	require.Zero(t, cgLen, "compaction group length; PlanLevel(1)")
-	_, cgLen = cp.PlanLevel(2)
-	require.Zero(t, cgLen, "compaction group length; PlanLevel(2)")
-	_, cgLen = cp.PlanLevel(3)
-	require.Zero(t, cgLen, "compaction group length; PlanLevel(3)")
-
-	tsmP, pLenP := cp.Plan(time.Now().Add(-time.Second))
-	require.Zero(t, len(tsmP), "compaction group; Plan()")
-	require.Zero(t, pLenP, "compaction group length; Plan()")
-
-	tsm, pLen, _ := cp.PlanOptimize()
-	require.Equal(t, 1, len(tsm), "compaction group")
-	require.Equal(t, int64(len(tsm)), pLen, "compaction group length")
-	require.Equal(t, len(expFiles), len(tsm[0]), "expected TSM files")
-}
-
-// This test is added to account for a single generation that has a group size
-// under 2 GB so it should be further compacted to a single file.
-func TestDefaultPlanner_PlanOptimize_SmallSingleGeneration(t *testing.T) {
-	// ~650 MB total group size
-	data := []tsm1.FileStat{
-		{
-			Path: "01-05.tsm1",
-			Size: 300 * 1024 * 1024,
-		},
-		{
-			Path: "01-06.tsm1",
-			Size: 200 * 1024 * 1024,
-		},
-		{
-			Path: "01-07.tsm1",
-			Size: 100 * 1024 * 1024,
-		},
-		{
-			Path: "01-08.tsm1",
-			Size: 50 * 1024 * 1024,
-		},
-	}
-
-	cp := tsm1.NewDefaultPlanner(
-		&fakeFileStore{
-			PathsFn: func() []tsm1.FileStat {
-				return data
-			},
-		}, tsdb.DefaultCompactFullWriteColdDuration,
-	)
-
-	expFiles := make([]tsm1.FileStat, 0)
-	for _, file := range data {
-		expFiles = append(expFiles, file)
-	}
-
-	_, cgLen := cp.PlanLevel(1)
-	require.Zero(t, cgLen, "compaction group length; PlanLevel(1)")
-	_, cgLen = cp.PlanLevel(2)
-	require.Zero(t, cgLen, "compaction group length; PlanLevel(2)")
-	_, cgLen = cp.PlanLevel(3)
-	require.Zero(t, cgLen, "compaction group length; PlanLevel(3)")
-
-	tsmP, pLenP := cp.Plan(time.Now().Add(-time.Second))
-	require.Zero(t, len(tsmP), "compaction group; Plan()")
-	require.Zero(t, pLenP, "compaction group length; Plan()")
-
-	tsm, pLen, gLen := cp.PlanOptimize()
-	require.Equal(t, 1, len(tsm), "compaction group")
-	require.Equal(t, int64(len(tsm)), pLen, "compaction group length")
-	require.Equal(t, int64(1), gLen, "generation of TSM files")
-	require.Equal(t, len(expFiles), len(tsm[0]), "expected TSM files")
-}
-
-func TestDefaultPlanner_PlanOptimize_SmallSingleGenerationReverse(t *testing.T) {
-	// ~650 MB total group size
-	data := []tsm1.FileStat{
-		{
-			Path: "01-05.tsm1",
-			Size: 50 * 1024 * 1024,
-		},
-		{
-			Path: "01-06.tsm1",
-			Size: 100 * 1024 * 1024,
-		},
-		{
-			Path: "01-07.tsm1",
-			Size: 200 * 1024 * 1024,
-		},
-		{
-			Path: "01-08.tsm1",
-			Size: 300 * 1024 * 1024,
-		},
-	}
-
-	cp := tsm1.NewDefaultPlanner(
-		&fakeFileStore{
-			PathsFn: func() []tsm1.FileStat {
-				return data
-			},
-		}, tsdb.DefaultCompactFullWriteColdDuration,
-	)
-
-	expFiles := make([]tsm1.FileStat, 0)
-	for _, file := range data {
-		expFiles = append(expFiles, file)
-	}
-
-	_, cgLen := cp.PlanLevel(1)
-	require.Zero(t, cgLen, "compaction group length; PlanLevel(1)")
-	_, cgLen = cp.PlanLevel(2)
-	require.Zero(t, cgLen, "compaction group length; PlanLevel(2)")
-	_, cgLen = cp.PlanLevel(3)
-	require.Zero(t, cgLen, "compaction group length; PlanLevel(3)")
-
-	tsmP, pLenP := cp.Plan(time.Now().Add(-time.Second))
-	require.Zero(t, len(tsmP), "compaction group; Plan()")
-	require.Zero(t, pLenP, "compaction group length; Plan()")
-
-	tsm, pLen, gLen := cp.PlanOptimize()
-	require.Equal(t, 1, len(tsm), "compaction group")
-	require.Equal(t, int64(len(tsm)), pLen, "compaction group length")
-	require.Equal(t, int64(1), gLen, "generation of TSM files")
-	require.Equal(t, len(expFiles), len(tsm[0]), "expected TSM files")
-}
-
-// This test is added to account for a single generation that has a group size
-// under 2 GB and has less then level 4 files it should be further compacted to a single file.
-func TestDefaultPlanner_PlanOptimize_SmallSingleGenerationUnderLevel4(t *testing.T) {
-	// ~650 MB total group size
-	data := []tsm1.FileStat{
-		{
-			Path: "01-02.tsm1",
-			Size: 300 * 1024 * 1024,
-		},
-		{
-			Path: "01-03.tsm1",
-			Size: 200 * 1024 * 1024,
-		},
-		{
-			Path: "01-04.tsm1",
-			Size: 100 * 1024 * 1024,
-		},
-	}
-
-	cp := tsm1.NewDefaultPlanner(
-		&fakeFileStore{
-			PathsFn: func() []tsm1.FileStat {
-				return data
-			},
-		}, tsdb.DefaultCompactFullWriteColdDuration,
-	)
-
-	expFiles := make([]tsm1.FileStat, 0)
-	for _, file := range data {
-		expFiles = append(expFiles, file)
-	}
-
-	_, cgLen := cp.PlanLevel(1)
-	require.Zero(t, cgLen, "compaction group length; PlanLevel(1)")
-	_, cgLen = cp.PlanLevel(2)
-	require.Zero(t, cgLen, "compaction group length; PlanLevel(2)")
-	_, cgLen = cp.PlanLevel(3)
-	require.Zero(t, cgLen, "compaction group length; PlanLevel(3)")
-
-	tsmP, pLenP := cp.Plan(time.Now().Add(-time.Second))
-	require.Zero(t, len(tsmP), "compaction group; Plan()")
-	require.Zero(t, pLenP, "compaction group length; Plan()")
-
-	tsm, pLen, gLen := cp.PlanOptimize()
-	require.Equal(t, 1, len(tsm), "compaction group")
-	require.Equal(t, int64(len(tsm)), pLen, "compaction group length")
-	require.Equal(t, int64(1), gLen, "generation of TSM files")
-	require.Equal(t, len(expFiles), len(tsm[0]), "expected TSM files")
-}
-
-func TestDefaultPlanner_PlanOptimize_SmallSingleGenerationUnderLevel4SmallFileFirst(t *testing.T) {
-	// ~650 MB total group size
-	data := []tsm1.FileStat{
-		{
-			Path: "01-02.tsm1",
-			Size: 100 * 1024 * 1024,
-		},
-		{
-			Path: "01-03.tsm1",
-			Size: 200 * 1024 * 1024,
-		},
-		{
-			Path: "01-04.tsm1",
-			Size: 300 * 1024 * 1024,
-		},
-	}
-
-	cp := tsm1.NewDefaultPlanner(
-		&fakeFileStore{
-			PathsFn: func() []tsm1.FileStat {
-				return data
-			},
-		}, tsdb.DefaultCompactFullWriteColdDuration,
-	)
-
-	expFiles := make([]tsm1.FileStat, 0)
-	for _, file := range data {
-		expFiles = append(expFiles, file)
-	}
-
-	_, cgLen := cp.PlanLevel(1)
-	require.Zero(t, cgLen, "compaction group length; PlanLevel(1)")
-	_, cgLen = cp.PlanLevel(2)
-	require.Zero(t, cgLen, "compaction group length; PlanLevel(2)")
-	_, cgLen = cp.PlanLevel(3)
-	require.Zero(t, cgLen, "compaction group length; PlanLevel(3)")
-
-	tsmP, pLenP := cp.Plan(time.Now().Add(-time.Second))
-	require.Zero(t, len(tsmP), "compaction group; Plan()")
-	require.Zero(t, pLenP, "compaction group length; Plan()")
-
-	tsm, pLen, gLen := cp.PlanOptimize()
-	require.Equal(t, 1, len(tsm), "compaction group")
-	require.Equal(t, int64(len(tsm)), pLen, "compaction group length")
-	require.Equal(t, int64(1), gLen, "generation of TSM files")
-	require.Equal(t, len(expFiles), len(tsm[0]), "expected TSM files")
-}
-
-// This test is added to account for a single generation that has a group size
-// under 2 GB and all files at max default points per block of 1000.
-// This should be planned for compaction at a more aggressive points per block.
-func TestDefaultPlanner_FullyCompacted_SmallSingleGeneration(t *testing.T) {
-	// ~650 MB total group size
-	data := []tsm1.FileStat{
-		{
-			Path: "01-05.tsm1",
-			Size: 300 * 1024 * 1024,
-		},
-		{
-			Path: "01-06.tsm1",
-			Size: 200 * 1024 * 1024,
-		},
-		{
-			Path: "01-07.tsm1",
-			Size: 100 * 1024 * 1024,
-		},
-		{
-			Path: "01-08.tsm1",
-			Size: 50 * 1024 * 1024,
-		},
-	}
-
-	fs := &fakeFileStore{
-		PathsFn: func() []tsm1.FileStat {
-			return data
-		},
-	}
-
-	err := fs.SetBlockCounts([]int{tsdb.DefaultMaxPointsPerBlock, tsdb.DefaultMaxPointsPerBlock, tsdb.DefaultMaxPointsPerBlock, tsdb.DefaultMaxPointsPerBlock})
-	require.NoError(t, err, "SetBlockCounts")
-
-	cp := tsm1.NewDefaultPlanner(fs, tsdb.DefaultCompactFullWriteColdDuration)
-
-	compacted, reason := cp.FullyCompacted()
-	require.Equal(t, reason, tsdb.SingleGenerationReasonText, "fullyCompacted reason")
-	require.False(t, compacted, "is fully compacted")
-
-	_, cgLen := cp.PlanLevel(1)
-	require.Zero(t, cgLen, "compaction group length; PlanLevel(1)")
-	_, cgLen = cp.PlanLevel(2)
-	require.Zero(t, cgLen, "compaction group length; PlanLevel(2)")
-	_, cgLen = cp.PlanLevel(3)
-	require.Zero(t, cgLen, "compaction group length; PlanLevel(3)")
-
-	tsmP, pLenP := cp.Plan(time.Now().Add(-time.Second))
-	require.Zero(t, len(tsmP), "compaction group; Plan()")
-	require.Zero(t, pLenP, "compaction group length; Plan()")
-
-	_, cgLen, genLen := cp.PlanOptimize()
-	require.Equal(t, int64(1), cgLen, "compaction group length")
-	require.Equal(t, int64(1), genLen, "generation count")
-}
-
-func TestDefaultPlanner_FullyCompacted_SmallSingleGenerationSmallFileFirst(t *testing.T) {
-	// ~650 MB total group size
-	data := []tsm1.FileStat{
-		{
-			Path: "01-05.tsm1",
-			Size: 50 * 1024 * 1024,
-		},
-		{
-			Path: "01-06.tsm1",
-			Size: 100 * 1024 * 1024,
-		},
-		{
-			Path: "01-07.tsm1",
-			Size: 200 * 1024 * 1024,
-		},
-		{
-			Path: "01-08.tsm1",
-			Size: 300 * 1024 * 1024,
-		},
-	}
-
-	fs := &fakeFileStore{
-		PathsFn: func() []tsm1.FileStat {
-			return data
-		},
-	}
-
-	err := fs.SetBlockCounts([]int{tsdb.DefaultMaxPointsPerBlock, tsdb.DefaultMaxPointsPerBlock, tsdb.DefaultMaxPointsPerBlock, tsdb.DefaultMaxPointsPerBlock})
-	require.NoError(t, err, "SetBlockCounts")
-
-	cp := tsm1.NewDefaultPlanner(fs, tsdb.DefaultCompactFullWriteColdDuration)
+func TestDefaultPlanner_PlanOptimize_Test(t *testing.T) {
+	// bc are the block counts for the tsm files
+	// bc can be an empty list
+	type PlanOptimizeTests struct {
+		fs                      []tsm1.FileStat
+		bc                      []int
+		fullyCompactedReasonExp string
+		generationCount         int64
+	}
+
+	fileStats := []PlanOptimizeTests{
+		// Large multi generation group with files at and under 2GB
+		PlanOptimizeTests{
+			[]tsm1.FileStat{
+				{
+					Path: "01-05.tsm1",
+					Size: 2048 * 1024 * 1024,
+				},
+				{
+					Path: "01-06.tsm1",
+					Size: 2048 * 1024 * 1024,
+				},
+				{
+					Path: "01-07.tsm1",
+					Size: 2048 * 1024 * 1024,
+				},
+				{
+					Path: "01-08.tsm1",
+					Size: 1048 * 1024 * 1024,
+				},
+				{
+					Path: "02-05.tsm1",
+					Size: 2048 * 1024 * 1024,
+				},
+				{
+					Path: "02-06.tsm1",
+					Size: 2048 * 1024 * 1024,
+				},
+				{
+					Path: "02-07.tsm1",
+					Size: 2048 * 1024 * 1024,
+				},
+				{
+					Path: "02-08.tsm1",
+					Size: 1048 * 1024 * 1024,
+				},
+				{
+					Path: "03-04.tsm1",
+					Size: 2048 * 1024 * 1024,
+				},
+				{
+					Path: "03-05.tsm1",
+					Size: 512 * 1024 * 1024,
+				},
+			},
+			[]int{},
+			"not fully compacted and not idle because of more than one generation",
+			3,
+		},
+		// ~650mb group size
+		PlanOptimizeTests{
+			[]tsm1.FileStat{
+				{
+					Path: "01-05.tsm1",
+					Size: 300 * 1024 * 1024,
+				},
+				{
+					Path: "01-06.tsm1",
+					Size: 200 * 1024 * 1024,
+				},
+				{
+					Path: "01-07.tsm1",
+					Size: 100 * 1024 * 1024,
+				},
+				{
+					Path: "01-08.tsm1",
+					Size: 50 * 1024 * 1024,
+				},
+			},
+			[]int{},
+			tsdb.SingleGenerationReasonText,
+			1,
+		},
+		// ~650 MB total group size with generations under 4
+		PlanOptimizeTests{
+			[]tsm1.FileStat{
+				{
+					Path: "01-02.tsm1",
+					Size: 300 * 1024 * 1024,
+				},
+				{
+					Path: "01-03.tsm1",
+					Size: 200 * 1024 * 1024,
+				},
+				{
+					Path: "01-04.tsm1",
+					Size: 100 * 1024 * 1024,
+				},
+			},
+			[]int{},
+			tsdb.SingleGenerationReasonText,
+			1,
+		},
+		{
+			[]tsm1.FileStat{
+				{
+					Path: "01-05.tsm1",
+					Size: 300 * 1024 * 1024,
+				},
+				{
+					Path: "01-06.tsm1",
+					Size: 200 * 1024 * 1024,
+				},
+				{
+					Path: "01-07.tsm1",
+					Size: 100 * 1024 * 1024,
+				},
+				{
+					Path: "01-08.tsm1",
+					Size: 50 * 1024 * 1024,
+				},
+			}, []int{tsdb.DefaultMaxPointsPerBlock, tsdb.DefaultMaxPointsPerBlock, tsdb.DefaultMaxPointsPerBlock, tsdb.DefaultMaxPointsPerBlock},
+			tsdb.SingleGenerationReasonText,
+			1,
+		},
+
+		// > 2 GB total group size
+		// 50% of files are at aggressive max block size
+		{
+			[]tsm1.FileStat{
+				{
+					Path: "01-05.tsm1",
+					Size: 700 * 1024 * 1024,
+				},
+				{
+					Path: "01-06.tsm1",
+					Size: 500 * 1024 * 1024,
+				},
+				{
+					Path: "01-07.tsm1",
+					Size: 400 * 1024 * 1024,
+				},
+				{
+					Path: "01-08.tsm1",
+					Size: 300 * 1024 * 1024,
+				},
+				{
+					Path: "01-09.tsm1",
+					Size: 200 * 1024 * 1024,
+				},
+				{
+					Path: "01-10.tsm1",
+					Size: 100 * 1024 * 1024,
+				},
+				{
+					Path: "01-11.tsm1",
+					Size: 50 * 1024 * 1024,
+				},
+				{
+					Path: "01-12.tsm1",
+					Size: 25 * 1024 * 1024,
+				},
+			},
+			[]int{
+				tsdb.AggressiveMaxPointsPerBlock,
+				tsdb.AggressiveMaxPointsPerBlock,
+				tsdb.AggressiveMaxPointsPerBlock,
+				tsdb.AggressiveMaxPointsPerBlock,
+				tsdb.DefaultMaxPointsPerBlock,
+				tsdb.DefaultMaxPointsPerBlock,
+				tsdb.DefaultMaxPointsPerBlock,
+				tsdb.DefaultMaxPointsPerBlock,
+			},
+			tsdb.SingleGenerationReasonText,
+			1,
+		},
+		{
+			// > 2 GB total group size
+			// 100% of files are at aggressive max block size
+			[]tsm1.FileStat{
+				{
+					Path: "01-13.tsm1",
+					Size: 2048 * 1024 * 1024,
+				},
+				{
+					Path: "01-14.tsm1",
+					Size: 650 * 1024 * 1024,
+				},
+				{
+					Path: "01-15.tsm1",
+					Size: 450 * 1024 * 1024,
+				},
+			}, []int{
+			tsdb.AggressiveMaxPointsPerBlock,
+			tsdb.DefaultMaxPointsPerBlock,
+			tsdb.DefaultMaxPointsPerBlock,
+		},
+			tsdb.SingleGenerationReasonText,
+			1,
+		},
+		{
+			// Last files are lower than first files generations
+			// Mix of less than 4 level files and > level 4 files
+			[]tsm1.FileStat{
+				{
+					Path: "01-05.tsm1",
+					Size: 2048 * 1024 * 1024,
+				},
+				{
+					Path: "01-06.tsm1",
+					Size: 2048 * 1024 * 1024,
+				},
+				{
+					Path: "01-07.tsm1",
+					Size: 2048 * 1024 * 1024,
+				},
+				{
+					Path: "01-08.tsm1",
+					Size: 1048 * 1024 * 1024,
+				},
+				{
+					Path: "02-05.tsm1",
+					Size: 2048 * 1024 * 1024,
+				},
+				{
+					Path: "02-06.tsm1",
+					Size: 2048 * 1024 * 1024,
+				},
+				{
+					Path: "02-07.tsm1",
+					Size: 2048 * 1024 * 1024,
+				},
+				{
+					Path: "02-08.tsm1",
+					Size: 1048 * 1024 * 1024,
+				},
+				{
+					Path: "03-03.tsm1",
+					Size: 2048 * 1024 * 1024,
+				},
+				{
+					Path: "03-04.tsm1",
+					Size: 2048 * 1024 * 1024,
+				},
+				{
+					Path: "03-04.tsm1",
+					Size: 600 * 1024 * 1024,
+				},
+				{
+					Path: "03-06.tsm1",
+					Size: 500 * 1024 * 1024,
+				},
+			}, []int{}, "not fully compacted and not idle because of more than one generation", 3,
+		},
+		{
+			// This test will mock a 'backfill' condition where we have a single
+			// shard with many generations. The initial generation should be fully
+			// compacted, but we have some new generations that are not. We need to ensure
+			// the optimize planner will pick these up and compact everything together.
+			[]tsm1.FileStat{
+				{
+					Path: "01-05.tsm1",
+					Size: 2048 * 1024 * 1024,
+				},
+				{
+					Path: "01-06.tsm1",
+					Size: 2048 * 1024 * 1024,
+				},
+				{
+					Path: "01-07.tsm1",
+					Size: 2048 * 1024 * 1024,
+				},
+				{
+					Path: "02-04.tsm1",
+					Size: 700 * 1024 * 1024,
+				},
+				{
+					Path: "02-05.tsm1",
+					Size: 500 * 1024 * 1024,
+				},
+				{
+					Path: "02-06.tsm1",
+					Size: 400 * 1024 * 1024,
+				},
+				{
+					Path: "03-02.tsm1",
+					Size: 700 * 1024 * 1024,
+				},
+				{
+					Path: "03-03.tsm1",
+					Size: 500 * 1024 * 1024,
+				},
+				{
+					Path: "03-04.tsm1",
+					Size: 400 * 1024 * 1024,
+				},
+				{
+					Path: "04-01.tsm1",
+					Size: 700 * 1024 * 1024,
+				},
+				{
+					Path: "04-02.tsm1",
+					Size: 500 * 1024 * 1024,
+				},
+				{
+					Path: "03-03.tsm1",
+					Size: 400 * 1024 * 1024,
+				},
+			}, []int{
+			tsdb.AggressiveMaxPointsPerBlock,
+			tsdb.AggressiveMaxPointsPerBlock,
+			tsdb.AggressiveMaxPointsPerBlock,
+
+			tsdb.AggressiveMaxPointsPerBlock,
+			tsdb.AggressiveMaxPointsPerBlock,
+			tsdb.DefaultMaxPointsPerBlock,
+
+			tsdb.AggressiveMaxPointsPerBlock,
+			tsdb.AggressiveMaxPointsPerBlock,
+			tsdb.DefaultMaxPointsPerBlock,
+
+			tsdb.DefaultMaxPointsPerBlock,
+			// Use some magic numbers but these are just small values for block counts
+			100,
+			10,
+		},
+			"not fully compacted and not idle because of more than one generation",
+			4,
+		},
+	}
+
+	expected := func(cp *tsm1.DefaultPlanner, reasonExp string, generationCountExp int64) {
+		compacted, reason := cp.FullyCompacted()
+		require.Equal(t, reason, reasonExp, "fullyCompacted reason")
+		require.False(t, compacted, "is fully compacted")
+
+		_, cgLen := cp.PlanLevel(1)
+		require.Zero(t, cgLen, "compaction group length; PlanLevel(1)")
+		_, cgLen = cp.PlanLevel(2)
+		require.Zero(t, cgLen, "compaction group length; PlanLevel(2)")
+		_, cgLen = cp.PlanLevel(3)
+		require.Zero(t, cgLen, "compaction group length; PlanLevel(3)")
+
+		tsmP, pLenP := cp.Plan(time.Now().Add(-time.Second))
+		require.Zero(t, len(tsmP), "compaction group; Plan()")
+		require.Zero(t, pLenP, "compaction group length; Plan()")
+
+		_, cgLen, genLen := cp.PlanOptimize()
+		require.Equal(t, int64(1), cgLen, "compaction group length")
+		require.Equal(t, generationCountExp, genLen, "generation count")
+
+	}
+
+	for _, fileStat := range fileStats {
+		ffs := &fakeFileStore{
+			PathsFn: func() []tsm1.FileStat {
+				return fileStat.fs
+			},
+		}
 
-	compacted, reason := cp.FullyCompacted()
-	require.Equal(t, reason, tsdb.SingleGenerationReasonText, "fullyCompacted reason")
-	require.False(t, compacted, "is fully compacted")
+		if len(fileStat.bc) > 0 {
+			err := ffs.SetBlockCounts(fileStat.bc)
+			require.NoError(t, err, "setting block counts")
+		}
 
-	_, cgLen := cp.PlanLevel(1)
-	require.Zero(t, cgLen, "compaction group length; PlanLevel(1)")
-	_, cgLen = cp.PlanLevel(2)
-	require.Zero(t, cgLen, "compaction group length; PlanLevel(2)")
-	_, cgLen = cp.PlanLevel(3)
-	require.Zero(t, cgLen, "compaction group length; PlanLevel(3)")
+		cp := tsm1.NewDefaultPlanner(ffs, tsdb.DefaultCompactFullWriteColdDuration)
+		expected(cp, fileStat.fullyCompactedReasonExp, fileStat.generationCount)
 
-	tsmP, pLenP := cp.Plan(time.Now().Add(-time.Second))
-	require.Zero(t, len(tsmP), "compaction group; Plan()")
-	require.Zero(t, pLenP, "compaction group length; Plan()")
+		// Reverse test files and re-run tests
+		slices.Reverse(fileStat.fs)
+		cp = tsm1.NewDefaultPlanner(ffs, tsdb.DefaultCompactFullWriteColdDuration)
+		expected(cp, fileStat.fullyCompactedReasonExp, fileStat.generationCount)
+	}
 
-	_, cgLen, genLen := cp.PlanOptimize()
-	require.Equal(t, int64(1), cgLen, "compaction group length")
-	require.Equal(t, int64(1), genLen, "generation count")
 }
 
 // This test is added to account for halting state after
@@ -2772,163 +2688,6 @@ func TestDefaultPlanner_FullyCompacted_SmallSingleGeneration_Halt(t *testing.T)
 	require.Zero(t, genLen, "generation count")
 }
 
-// This test is added to account for a single generation that has a group size
-// under 2 GB and a mix of aggressive max blocks and default max blocks
-// it should be further compacted.
-func TestDefaultPlanner_FullyCompacted_LargeSingleGenerationUnderAggressiveBlocks(t *testing.T) {
-	// > 2 GB total group size
-	// 50% of files are at aggressive max block size
-	data := []tsm1.FileStat{
-		{
-			Path: "01-05.tsm1",
-			Size: 700 * 1024 * 1024,
-		},
-		{
-			Path: "01-06.tsm1",
-			Size: 500 * 1024 * 1024,
-		},
-		{
-			Path: "01-07.tsm1",
-			Size: 400 * 1024 * 1024,
-		},
-		{
-			Path: "01-08.tsm1",
-			Size: 300 * 1024 * 1024,
-		},
-		{
-			Path: "01-09.tsm1",
-			Size: 200 * 1024 * 1024,
-		},
-		{
-			Path: "01-10.tsm1",
-			Size: 100 * 1024 * 1024,
-		},
-		{
-			Path: "01-11.tsm1",
-			Size: 50 * 1024 * 1024,
-		},
-		{
-			Path: "01-12.tsm1",
-			Size: 25 * 1024 * 1024,
-		},
-	}
-
-	fs := &fakeFileStore{
-		PathsFn: func() []tsm1.FileStat {
-			return data
-		},
-	}
-	blocks := []int{
-		tsdb.AggressiveMaxPointsPerBlock,
-		tsdb.AggressiveMaxPointsPerBlock,
-		tsdb.AggressiveMaxPointsPerBlock,
-		tsdb.AggressiveMaxPointsPerBlock,
-		tsdb.DefaultMaxPointsPerBlock,
-		tsdb.DefaultMaxPointsPerBlock,
-		tsdb.DefaultMaxPointsPerBlock,
-		tsdb.DefaultMaxPointsPerBlock,
-	}
-	err := fs.SetBlockCounts(blocks)
-	require.NoError(t, err, "SetBlockCounts")
-
-	cp := tsm1.NewDefaultPlanner(fs, tsdb.DefaultCompactFullWriteColdDuration)
-	compacted, reason := cp.FullyCompacted()
-	require.Equal(t, reason, tsdb.SingleGenerationReasonText, "fullyCompacted reason")
-	require.False(t, compacted, "is fully compacted")
-
-	_, cgLen := cp.PlanLevel(1)
-	require.Zero(t, cgLen, "compaction group length; PlanLevel(1)")
-	_, cgLen = cp.PlanLevel(2)
-	require.Zero(t, cgLen, "compaction group length; PlanLevel(2)")
-	_, cgLen = cp.PlanLevel(3)
-	require.Zero(t, cgLen, "compaction group length; PlanLevel(3)")
-
-	tsmP, pLenP := cp.Plan(time.Now().Add(-time.Second))
-	require.Zero(t, len(tsmP), "compaction group; Plan()")
-	require.Zero(t, pLenP, "compaction group length; Plan()")
-
-	_, cgLen, genLen := cp.PlanOptimize()
-	require.Equal(t, int64(1), cgLen, "compaction group length")
-	require.Equal(t, int64(1), genLen, "generation count")
-}
-
-func TestDefaultPlanner_FullyCompacted_LargeSingleGenerationUnderAggressiveBlocksMixFileSize(t *testing.T) {
-	// > 2 GB total group size
-	// 50% of files are at aggressive max block size
-	data := []tsm1.FileStat{
-		{
-			Path: "01-05.tsm1",
-			Size: 300 * 1024 * 1024,
-		},
-		{
-			Path: "01-06.tsm1",
-			Size: 700 * 1024 * 1024,
-		},
-		{
-			Path: "01-07.tsm1",
-			Size: 25 * 1024 * 1024,
-		},
-		{
-			Path: "01-08.tsm1",
-			Size: 400 * 1024 * 1024,
-		},
-		{
-			Path: "01-09.tsm1",
-			Size: 50 * 1024 * 1024,
-		},
-		{
-			Path: "01-10.tsm1",
-			Size: 100 * 1024 * 1024,
-		},
-		{
-			Path: "01-11.tsm1",
-			Size: 200 * 1024 * 1024,
-		},
-		{
-			Path: "01-12.tsm1",
-			Size: 500 * 1024 * 1024,
-		},
-	}
-
-	fs := &fakeFileStore{
-		PathsFn: func() []tsm1.FileStat {
-			return data
-		},
-	}
-	blocks := []int{
-		tsdb.AggressiveMaxPointsPerBlock,
-		tsdb.AggressiveMaxPointsPerBlock,
-		tsdb.AggressiveMaxPointsPerBlock,
-		tsdb.AggressiveMaxPointsPerBlock,
-		tsdb.DefaultMaxPointsPerBlock,
-		tsdb.DefaultMaxPointsPerBlock,
-		tsdb.DefaultMaxPointsPerBlock,
-		tsdb.DefaultMaxPointsPerBlock,
-	}
-	err := fs.SetBlockCounts(blocks)
-	require.NoError(t, err, "SetBlockCounts")
-
-	cp := tsm1.NewDefaultPlanner(fs, tsdb.DefaultCompactFullWriteColdDuration)
-	compacted, reason := cp.FullyCompacted()
-	require.Equal(t, reason, tsdb.SingleGenerationReasonText, "fullyCompacted reason")
-	require.False(t, compacted, "is fully compacted")
-
-	_, cgLen := cp.PlanLevel(1)
-	require.Zero(t, cgLen, "compaction group length; PlanLevel(1)")
-	_, cgLen = cp.PlanLevel(2)
-	require.Zero(t, cgLen, "compaction group length; PlanLevel(2)")
-	_, cgLen = cp.PlanLevel(3)
-	require.Zero(t, cgLen, "compaction group length; PlanLevel(3)")
-
-	tsmP, pLenP := cp.Plan(time.Now().Add(-time.Second))
-	require.Zero(t, len(tsmP), "compaction group; Plan()")
-	require.Zero(t, pLenP, "compaction group length; Plan()")
-
-	_, cgLen, genLen := cp.PlanOptimize()
-	require.Equal(t, int64(1), cgLen, "compaction group length")
-	require.Equal(t, int64(1), genLen, "generation count")
-}
-
 // This test is added to account for a single generation that has a group size
 // over 2 GB with 1 file under 2 GB all at max points per block with aggressive compaction.
 // It should not compact any further.
@@ -3045,114 +2804,8 @@ func TestDefaultPlanner_FullyCompacted_LargeSingleGenerationNoMaxAggrBlocks(t *t
 			Size: 2048 * 1024 * 1024,
 		},
 		{
-			Path: "01-14.tsm1",
-			Size: 691 * 1024 * 1024,
-		},
-	}
-
-	fs := &fakeFileStore{
-		PathsFn: func() []tsm1.FileStat {
-			return data
-		},
-	}
-	blocks := []int{
-		tsdb.AggressiveMaxPointsPerBlock,
-		tsdb.DefaultMaxPointsPerBlock,
-	}
-	err := fs.SetBlockCounts(blocks)
-	require.NoError(t, err, "SetBlockCounts")
-
-	cp := tsm1.NewDefaultPlanner(fs, tsdb.DefaultCompactFullWriteColdDuration)
-
-	compacted, reason := cp.FullyCompacted()
-	reasonExp := ""
-	require.Equal(t, reason, reasonExp, "fullyCompacted reason")
-	require.True(t, compacted, "is fully compacted")
-
-	_, cgLen := cp.PlanLevel(1)
-	require.Zero(t, cgLen, "compaction group length; PlanLevel(1)")
-	_, cgLen = cp.PlanLevel(2)
-	require.Zero(t, cgLen, "compaction group length; PlanLevel(2)")
-	_, cgLen = cp.PlanLevel(3)
-	require.Zero(t, cgLen, "compaction group length; PlanLevel(3)")
-
-	tsmP, pLenP := cp.Plan(time.Now().Add(-time.Second))
-	require.Zero(t, len(tsmP), "compaction group; Plan()")
-	require.Zero(t, pLenP, "compaction group length; Plan()")
-
-	cgroup, cgLen, genLen := cp.PlanOptimize()
-	require.Equal(t, []tsm1.CompactionGroup(nil), cgroup, "compaction group")
-	require.Equal(t, int64(0), cgLen, "compaction group length")
-	require.Equal(t, int64(0), genLen, "generation count")
-}
-
-func TestDefaultPlanner_FullyCompacted_LargeSingleGenerationNoMaxAggrBlocksSmallFileFirst(t *testing.T) {
-	// > 2 GB total group size
-	// 100% of files are at aggressive max block size
-	data := []tsm1.FileStat{
-		{
-			Path: "01-13.tsm1",
-			Size: 691 * 1024 * 1024,
-		},
-		{
-			Path: "01-14.tsm1",
-			Size: 2048 * 1024 * 1024,
-		},
-	}
-
-	fs := &fakeFileStore{
-		PathsFn: func() []tsm1.FileStat {
-			return data
-		},
-	}
-	blocks := []int{
-		tsdb.DefaultMaxPointsPerBlock,
-		tsdb.AggressiveMaxPointsPerBlock,
-	}
-	err := fs.SetBlockCounts(blocks)
-	require.NoError(t, err, "SetBlockCounts")
-
-	cp := tsm1.NewDefaultPlanner(fs, tsdb.DefaultCompactFullWriteColdDuration)
-
-	compacted, reason := cp.FullyCompacted()
-	reasonExp := ""
-	require.Equal(t, reason, reasonExp, "fullyCompacted reason")
-	require.True(t, compacted, "is fully compacted")
-
-	_, cgLen := cp.PlanLevel(1)
-	require.Zero(t, cgLen, "compaction group length; PlanLevel(1)")
-	_, cgLen = cp.PlanLevel(2)
-	require.Zero(t, cgLen, "compaction group length; PlanLevel(2)")
-	_, cgLen = cp.PlanLevel(3)
-	require.Zero(t, cgLen, "compaction group length; PlanLevel(3)")
-
-	tsmP, pLenP := cp.Plan(time.Now().Add(-time.Second))
-	require.Zero(t, len(tsmP), "compaction group; Plan()")
-	require.Zero(t, pLenP, "compaction group length; Plan()")
-
-	cgroup, cgLen, genLen := cp.PlanOptimize()
-	require.Equal(t, []tsm1.CompactionGroup(nil), cgroup, "compaction group")
-	require.Equal(t, int64(0), cgLen, "compaction group length")
-	require.Equal(t, int64(0), genLen, "generation count")
-}
-
-// This test is added to account for a single generation that has a group size
-// over 2 GB and multiple files under 2 GB all at max points per block for aggressive compaction.
-func TestDefaultPlanner_FullyCompacted_ManySingleGenLessThen2GBMaxAggrBlocks(t *testing.T) {
-	// > 2 GB total group size
-	// 100% of files are at aggressive max block size
-	data := []tsm1.FileStat{
-		{
-			Path: "01-13.tsm1",
-			Size: 2048 * 1024 * 1024,
-		},
-		{
-			Path: "01-14.tsm1",
-			Size: 650 * 1024 * 1024,
-		},
-		{
-			Path: "01-15.tsm1",
-			Size: 450 * 1024 * 1024,
+			Path: "01-14.tsm1",
+			Size: 691 * 1024 * 1024,
 		},
 	}
 
@@ -3163,8 +2816,7 @@ func TestDefaultPlanner_FullyCompacted_ManySingleGenLessThen2GBMaxAggrBlocks(t *
 	}
 	blocks := []int{
 		tsdb.AggressiveMaxPointsPerBlock,
-		tsdb.AggressiveMaxPointsPerBlock,
-		tsdb.AggressiveMaxPointsPerBlock,
+		tsdb.DefaultMaxPointsPerBlock,
 	}
 	err := fs.SetBlockCounts(blocks)
 	require.NoError(t, err, "SetBlockCounts")
@@ -3189,24 +2841,20 @@ func TestDefaultPlanner_FullyCompacted_ManySingleGenLessThen2GBMaxAggrBlocks(t *
 
 	cgroup, cgLen, genLen := cp.PlanOptimize()
 	require.Equal(t, []tsm1.CompactionGroup(nil), cgroup, "compaction group")
-	require.Zero(t, cgLen, "compaction group length")
-	require.Zero(t, genLen, "generation count")
+	require.Equal(t, int64(0), cgLen, "compaction group length")
+	require.Equal(t, int64(0), genLen, "generation count")
 }
 
-func TestDefaultPlanner_FullyCompacted_ManySingleGenLessThen2GBMaxAggrBlocksSmallFileFirst(t *testing.T) {
+func TestDefaultPlanner_FullyCompacted_LargeSingleGenerationNoMaxAggrBlocksSmallFileFirst(t *testing.T) {
 	// > 2 GB total group size
 	// 100% of files are at aggressive max block size
 	data := []tsm1.FileStat{
 		{
 			Path: "01-13.tsm1",
-			Size: 450 * 1024 * 1024,
+			Size: 691 * 1024 * 1024,
 		},
 		{
 			Path: "01-14.tsm1",
-			Size: 650 * 1024 * 1024,
-		},
-		{
-			Path: "01-15.tsm1",
 			Size: 2048 * 1024 * 1024,
 		},
 	}
@@ -3217,8 +2865,7 @@ func TestDefaultPlanner_FullyCompacted_ManySingleGenLessThen2GBMaxAggrBlocksSmal
 		},
 	}
 	blocks := []int{
-		tsdb.AggressiveMaxPointsPerBlock,
-		tsdb.AggressiveMaxPointsPerBlock,
+		tsdb.DefaultMaxPointsPerBlock,
 		tsdb.AggressiveMaxPointsPerBlock,
 	}
 	err := fs.SetBlockCounts(blocks)
@@ -3244,14 +2891,13 @@ func TestDefaultPlanner_FullyCompacted_ManySingleGenLessThen2GBMaxAggrBlocksSmal
 
 	cgroup, cgLen, genLen := cp.PlanOptimize()
 	require.Equal(t, []tsm1.CompactionGroup(nil), cgroup, "compaction group")
-	require.Zero(t, cgLen, "compaction group length")
-	require.Zero(t, genLen, "generation count")
+	require.Equal(t, int64(0), cgLen, "compaction group length")
+	require.Equal(t, int64(0), genLen, "generation count")
 }
 
 // This test is added to account for a single generation that has a group size
-// over 2 GB and multiple files under 2 GB with multiple files under aggressive
-// max points per block. This should further compact.
-func TestDefaultPlanner_FullyCompacted_ManySingleGenLessThen2GBNotMaxAggrBlocks(t *testing.T) {
+// over 2 GB and multiple files under 2 GB all at max points per block for aggressive compaction.
+func TestDefaultPlanner_FullyCompacted_ManySingleGenLessThen2GBMaxAggrBlocks(t *testing.T) {
 	// > 2 GB total group size
 	// 100% of files are at aggressive max block size
 	data := []tsm1.FileStat{
@@ -3276,8 +2922,8 @@ func TestDefaultPlanner_FullyCompacted_ManySingleGenLessThen2GBNotMaxAggrBlocks(
 	}
 	blocks := []int{
 		tsdb.AggressiveMaxPointsPerBlock,
-		tsdb.DefaultMaxPointsPerBlock,
-		tsdb.DefaultMaxPointsPerBlock,
+		tsdb.AggressiveMaxPointsPerBlock,
+		tsdb.AggressiveMaxPointsPerBlock,
 	}
 	err := fs.SetBlockCounts(blocks)
 	require.NoError(t, err, "SetBlockCounts")
@@ -3285,8 +2931,9 @@ func TestDefaultPlanner_FullyCompacted_ManySingleGenLessThen2GBNotMaxAggrBlocks(
 	cp := tsm1.NewDefaultPlanner(fs, tsdb.DefaultCompactFullWriteColdDuration)
 
 	compacted, reason := cp.FullyCompacted()
-	require.Equal(t, reason, tsdb.SingleGenerationReasonText, "fullyCompacted reason")
-	require.False(t, compacted, "is fully compacted")
+	reasonExp := ""
+	require.Equal(t, reason, reasonExp, "fullyCompacted reason")
+	require.True(t, compacted, "is fully compacted")
 
 	_, cgLen := cp.PlanLevel(1)
 	require.Zero(t, cgLen, "compaction group length; PlanLevel(1)")
@@ -3299,66 +2946,28 @@ func TestDefaultPlanner_FullyCompacted_ManySingleGenLessThen2GBNotMaxAggrBlocks(
 	require.Zero(t, len(tsmP), "compaction group; Plan()")
 	require.Zero(t, pLenP, "compaction group length; Plan()")
 
-	_, cgLen, genLen := cp.PlanOptimize()
-	require.Equal(t, int64(1), cgLen, "compaction group length")
-	require.Equal(t, int64(1), genLen, "generation count")
+	cgroup, cgLen, genLen := cp.PlanOptimize()
+	require.Equal(t, []tsm1.CompactionGroup(nil), cgroup, "compaction group")
+	require.Zero(t, cgLen, "compaction group length")
+	require.Zero(t, genLen, "generation count")
 }
 
-// This test is added to account for multiple generations over level 4
-// compaction and over 2 GB group size, with a level 3 start generation
-// over 2 GB group size.
-func TestDefaultPlanner_FullyCompacted_ManySingleGen2GBLastLevel2(t *testing.T) {
+func TestDefaultPlanner_FullyCompacted_ManySingleGenLessThen2GBMaxAggrBlocksSmallFileFirst(t *testing.T) {
 	// > 2 GB total group size
 	// 100% of files are at aggressive max block size
 	data := []tsm1.FileStat{
 		{
-			Path: "01-05.tsm1",
-			Size: 2048 * 1024 * 1024,
-		},
-		{
-			Path: "01-06.tsm1",
-			Size: 2048 * 1024 * 1024,
-		},
-		{
-			Path: "01-07.tsm1",
-			Size: 2048 * 1024 * 1024,
-		},
-		{
-			Path: "01-08.tsm1",
-			Size: 1048 * 1024 * 1024,
-		},
-		{
-			Path: "02-05.tsm1",
-			Size: 2048 * 1024 * 1024,
-		},
-		{
-			Path: "02-06.tsm1",
-			Size: 2048 * 1024 * 1024,
-		},
-		{
-			Path: "02-07.tsm1",
-			Size: 2048 * 1024 * 1024,
-		},
-		{
-			Path: "02-08.tsm1",
-			Size: 1048 * 1024 * 1024,
+			Path: "01-13.tsm1",
+			Size: 450 * 1024 * 1024,
 		},
 		{
-			Path: "03-03.tsm1",
-			Size: 2048 * 1024 * 1024,
+			Path: "01-14.tsm1",
+			Size: 650 * 1024 * 1024,
 		},
 		{
-			Path: "03-04.tsm1",
+			Path: "01-15.tsm1",
 			Size: 2048 * 1024 * 1024,
 		},
-		{
-			Path: "03-04.tsm1",
-			Size: 600 * 1024 * 1024,
-		},
-		{
-			Path: "03-06.tsm1",
-			Size: 500 * 1024 * 1024,
-		},
 	}
 
 	fs := &fakeFileStore{
@@ -3366,13 +2975,20 @@ func TestDefaultPlanner_FullyCompacted_ManySingleGen2GBLastLevel2(t *testing.T)
 			return data
 		},
 	}
+	blocks := []int{
+		tsdb.AggressiveMaxPointsPerBlock,
+		tsdb.AggressiveMaxPointsPerBlock,
+		tsdb.AggressiveMaxPointsPerBlock,
+	}
+	err := fs.SetBlockCounts(blocks)
+	require.NoError(t, err, "SetBlockCounts")
 
 	cp := tsm1.NewDefaultPlanner(fs, tsdb.DefaultCompactFullWriteColdDuration)
 
-	expFiles := make([]tsm1.FileStat, 0)
-	for _, file := range data {
-		expFiles = append(expFiles, file)
-	}
+	compacted, reason := cp.FullyCompacted()
+	reasonExp := ""
+	require.Equal(t, reason, reasonExp, "fullyCompacted reason")
+	require.True(t, compacted, "is fully compacted")
 
 	_, cgLen := cp.PlanLevel(1)
 	require.Zero(t, cgLen, "compaction group length; PlanLevel(1)")
@@ -3385,10 +3001,10 @@ func TestDefaultPlanner_FullyCompacted_ManySingleGen2GBLastLevel2(t *testing.T)
 	require.Zero(t, len(tsmP), "compaction group; Plan()")
 	require.Zero(t, pLenP, "compaction group length; Plan()")
 
-	tsm, cgLen, genLen := cp.PlanOptimize()
-	require.Equal(t, int64(1), cgLen, "compaction group length")
-	require.Equal(t, int64(3), genLen, "generation count")
-	require.Equal(t, len(expFiles), len(tsm[0]), "tsm files in compaction group")
+	cgroup, cgLen, genLen := cp.PlanOptimize()
+	require.Equal(t, []tsm1.CompactionGroup(nil), cgroup, "compaction group")
+	require.Zero(t, cgLen, "compaction group length")
+	require.Zero(t, genLen, "generation count")
 }
 
 func TestDefaultPlanner_FullyCompacted_ManySingleGen2GBLastLevel2MixedSizes(t *testing.T) {
@@ -3475,175 +3091,6 @@ func TestDefaultPlanner_FullyCompacted_ManySingleGen2GBLastLevel2MixedSizes(t *t
 	require.Equal(t, len(expFiles), len(tsm[0]), "tsm files in compaction group")
 }
 
-// This test will check to ensure that any TSM generations planned with the default planner
-// using Plan() and PlanLevel() over default block size are skipped.
-func TestDefaultPlanner_PlanOverAggressiveBlocks(t *testing.T) {
-	data := []tsm1.FileStat{
-		{
-			Path: "01-02.tsm1",
-			Size: 251 * 1024 * 1024,
-		},
-		{
-			Path: "01-03.tsm1",
-			Size: 1 * 1024 * 1024,
-		},
-		{
-			Path: "02-02.tsm1",
-			Size: 251 * 1024 * 1024,
-		},
-		{
-			Path: "02-03.tsm1",
-			Size: 1 * 1024 * 1024,
-		},
-		{
-			Path: "03-02.tsm1",
-			Size: 251 * 1024 * 1024,
-		},
-		{
-			Path: "03-03.tsm1",
-			Size: 1 * 1024 * 1024,
-		},
-	}
-
-	fs := &fakeFileStore{
-		PathsFn: func() []tsm1.FileStat {
-			return data
-		},
-	}
-	blocks := []int{
-		tsdb.AggressiveMaxPointsPerBlock,
-		tsdb.AggressiveMaxPointsPerBlock,
-		tsdb.AggressiveMaxPointsPerBlock,
-		tsdb.AggressiveMaxPointsPerBlock,
-		tsdb.AggressiveMaxPointsPerBlock,
-		tsdb.AggressiveMaxPointsPerBlock,
-	}
-	err := fs.SetBlockCounts(blocks)
-	require.NoError(t, err, "SetBlockCounts")
-
-	cp := tsm1.NewDefaultPlanner(fs, tsdb.DefaultCompactFullWriteColdDuration)
-
-	_, cgLen := cp.PlanLevel(1)
-	require.Zero(t, cgLen, "compaction group length; PlanLevel(1)")
-	_, cgLen = cp.PlanLevel(2)
-	require.Zero(t, cgLen, "compaction group length; PlanLevel(2)")
-	_, cgLen = cp.PlanLevel(3)
-	require.Zero(t, cgLen, "compaction group length; PlanLevel(3)")
-
-	tsmP, pLenP := cp.Plan(time.Now().Add(-time.Second))
-	require.Zero(t, len(tsmP), "compaction group; Plan()")
-	require.Zero(t, pLenP, "compaction group length; Plan()")
-}
-
-// This test will mock a 'backfill' condition where we have a single
-// shard with many generations. The initial generation should be fully
-// compacted, but we have some new generations that are not. We need to ensure
-// the optimize planner will pick these up and compact everything together.
-func TestDefaultPlanner_BackfillMock(t *testing.T) {
-	// > 2 GB total group size
-	// 50% of files are at aggressive max block size
-	data := []tsm1.FileStat{
-		{
-			Path: "01-05.tsm1",
-			Size: 2048 * 1024 * 1024,
-		},
-		{
-			Path: "01-06.tsm1",
-			Size: 2048 * 1024 * 1024,
-		},
-		{
-			Path: "01-07.tsm1",
-			Size: 2048 * 1024 * 1024,
-		},
-		{
-			Path: "02-04.tsm1",
-			Size: 700 * 1024 * 1024,
-		},
-		{
-			Path: "02-05.tsm1",
-			Size: 500 * 1024 * 1024,
-		},
-		{
-			Path: "02-06.tsm1",
-			Size: 400 * 1024 * 1024,
-		},
-		{
-			Path: "03-02.tsm1",
-			Size: 700 * 1024 * 1024,
-		},
-		{
-			Path: "03-03.tsm1",
-			Size: 500 * 1024 * 1024,
-		},
-		{
-			Path: "03-04.tsm1",
-			Size: 400 * 1024 * 1024,
-		},
-		{
-			Path: "04-01.tsm1",
-			Size: 700 * 1024 * 1024,
-		},
-		{
-			Path: "04-02.tsm1",
-			Size: 500 * 1024 * 1024,
-		},
-		{
-			Path: "03-03.tsm1",
-			Size: 400 * 1024 * 1024,
-		},
-	}
-
-	fs := &fakeFileStore{
-		PathsFn: func() []tsm1.FileStat {
-			return data
-		},
-	}
-	blocks := []int{
-		tsdb.AggressiveMaxPointsPerBlock,
-		tsdb.AggressiveMaxPointsPerBlock,
-		tsdb.AggressiveMaxPointsPerBlock,
-
-		tsdb.AggressiveMaxPointsPerBlock,
-		tsdb.AggressiveMaxPointsPerBlock,
-		tsdb.DefaultMaxPointsPerBlock,
-
-		tsdb.AggressiveMaxPointsPerBlock,
-		tsdb.AggressiveMaxPointsPerBlock,
-		tsdb.DefaultMaxPointsPerBlock,
-
-		tsdb.DefaultMaxPointsPerBlock,
-		// Use some magic numbers but these are just small values for block counts
-		100,
-		10,
-	}
-	err := fs.SetBlockCounts(blocks)
-	require.NoError(t, err, "SetBlockCounts")
-
-	cp := tsm1.NewDefaultPlanner(fs, tsdb.DefaultCompactFullWriteColdDuration)
-
-	_, cgLen := cp.PlanLevel(1)
-	require.Zero(t, cgLen, "compaction group length; PlanLevel(1)")
-	_, cgLen = cp.PlanLevel(2)
-	require.Zero(t, cgLen, "compaction group length; PlanLevel(2)")
-	_, cgLen = cp.PlanLevel(3)
-	require.Zero(t, cgLen, "compaction group length; PlanLevel(3)")
-
-	tsmP, pLenP := cp.Plan(time.Now().Add(-time.Second))
-	require.Zero(t, len(tsmP), "compaction group; Plan()")
-	require.Zero(t, pLenP, "compaction group length; Plan()")
-
-	expFiles := make([]tsm1.FileStat, 0)
-	for _, file := range data {
-		expFiles = append(expFiles, file)
-	}
-
-	tsm, cgLen, genLen := cp.PlanOptimize()
-	require.Equal(t, len(expFiles), len(tsm[0]), "expected TSM files")
-	require.Equal(t, int64(1), cgLen, "compaction group length")
-	// Should pick up 4 generations for compaction
-	require.Equal(t, int64(4), genLen, "generation count")
-}
-
 func TestDefaultPlanner_PlanOptimize_Tombstones(t *testing.T) {
 	data := []tsm1.FileStat{
 		{

From 0799f00cea2c4e263e90896a017a6ac100a63f72 Mon Sep 17 00:00:00 2001
From: devanbenz <devandbenz@gmail.com>
Date: Mon, 13 Jan 2025 14:01:30 -0600
Subject: [PATCH 27/31] feat: create loop for tests where there should be no
 further compaction

---
 tsdb/engine/tsm1/compact_test.go | 531 ++++++-------------------------
 1 file changed, 103 insertions(+), 428 deletions(-)

diff --git a/tsdb/engine/tsm1/compact_test.go b/tsdb/engine/tsm1/compact_test.go
index 05e8022990a..94df33f5b72 100644
--- a/tsdb/engine/tsm1/compact_test.go
+++ b/tsdb/engine/tsm1/compact_test.go
@@ -2280,7 +2280,7 @@ func TestDefaultPlanner_PlanOptimize_Test(t *testing.T) {
 		generationCount         int64
 	}
 
-	fileStats := []PlanOptimizeTests{
+	furtherCompactedTests := []PlanOptimizeTests{
 		// Large multi generation group with files at and under 2GB
 		PlanOptimizeTests{
 			[]tsm1.FileStat{
@@ -2601,7 +2601,7 @@ func TestDefaultPlanner_PlanOptimize_Test(t *testing.T) {
 		},
 	}
 
-	expected := func(cp *tsm1.DefaultPlanner, reasonExp string, generationCountExp int64) {
+	expectedNotFullyCompacted := func(cp *tsm1.DefaultPlanner, reasonExp string, generationCountExp int64) {
 		compacted, reason := cp.FullyCompacted()
 		require.Equal(t, reason, reasonExp, "fullyCompacted reason")
 		require.False(t, compacted, "is fully compacted")
@@ -2623,7 +2623,7 @@ func TestDefaultPlanner_PlanOptimize_Test(t *testing.T) {
 
 	}
 
-	for _, fileStat := range fileStats {
+	for _, fileStat := range furtherCompactedTests {
 		ffs := &fakeFileStore{
 			PathsFn: func() []tsm1.FileStat {
 				return fileStat.fs
@@ -2636,459 +2636,134 @@ func TestDefaultPlanner_PlanOptimize_Test(t *testing.T) {
 		}
 
 		cp := tsm1.NewDefaultPlanner(ffs, tsdb.DefaultCompactFullWriteColdDuration)
-		expected(cp, fileStat.fullyCompactedReasonExp, fileStat.generationCount)
+		expectedNotFullyCompacted(cp, fileStat.fullyCompactedReasonExp, fileStat.generationCount)
 
 		// Reverse test files and re-run tests
 		slices.Reverse(fileStat.fs)
 		cp = tsm1.NewDefaultPlanner(ffs, tsdb.DefaultCompactFullWriteColdDuration)
-		expected(cp, fileStat.fullyCompactedReasonExp, fileStat.generationCount)
+		expectedNotFullyCompacted(cp, fileStat.fullyCompactedReasonExp, fileStat.generationCount)
 	}
 
-}
-
-// This test is added to account for halting state after
-// TestDefaultPlanner_FullyCompacted_SmallSingleGeneration
-// will need to ensure that once we have single TSM file under 2 GB we stop
-func TestDefaultPlanner_FullyCompacted_SmallSingleGeneration_Halt(t *testing.T) {
-	// ~650 MB total group size
-	data := []tsm1.FileStat{
+	areFullyCompactedTests := []PlanOptimizeTests{
 		{
-			Path: "01-09.tsm1",
-			Size: 650 * 1024 * 1024,
-		},
-	}
-
-	cp := tsm1.NewDefaultPlanner(
-		&fakeFileStore{
-			PathsFn: func() []tsm1.FileStat {
-				return data
+			// This test is added to account for halting state after
+			// TestDefaultPlanner_FullyCompacted_SmallSingleGeneration
+			// will need to ensure that once we have single TSM file under 2 GB we stop
+			[]tsm1.FileStat{
+				{
+					Path: "01-09.tsm1",
+					Size: 650 * 1024 * 1024,
+				},
 			},
-		}, tsdb.DefaultCompactFullWriteColdDuration,
-	)
-
-	compacted, reason := cp.FullyCompacted()
-	reasonExp := ""
-	require.Equal(t, reason, reasonExp, "fullyCompacted reason")
-	require.True(t, compacted, "is fully compacted")
-
-	_, cgLen := cp.PlanLevel(1)
-	require.Zero(t, cgLen, "compaction group length; PlanLevel(1)")
-	_, cgLen = cp.PlanLevel(2)
-	require.Zero(t, cgLen, "compaction group length; PlanLevel(2)")
-	_, cgLen = cp.PlanLevel(3)
-	require.Zero(t, cgLen, "compaction group length; PlanLevel(3)")
-
-	tsmP, pLenP := cp.Plan(time.Now().Add(-time.Second))
-	require.Zero(t, len(tsmP), "compaction group; Plan()")
-	require.Zero(t, pLenP, "compaction group length; Plan()")
-
-	cgroup, cgLen, genLen := cp.PlanOptimize()
-	require.Equal(t, []tsm1.CompactionGroup(nil), cgroup, "compaction group")
-	require.Zero(t, cgLen, "compaction group length")
-	require.Zero(t, genLen, "generation count")
-}
-
-// This test is added to account for a single generation that has a group size
-// over 2 GB with 1 file under 2 GB all at max points per block with aggressive compaction.
-// It should not compact any further.
-func TestDefaultPlanner_FullyCompacted_LargeSingleGenerationMaxAggressiveBlocks(t *testing.T) {
-	// > 2 GB total group size
-	// 100% of files are at aggressive max block size
-	data := []tsm1.FileStat{
-		{
-			Path: "01-13.tsm1",
-			Size: 2048 * 1024 * 1024,
-		},
-		{
-			Path: "01-14.tsm1",
-			Size: 691 * 1024 * 1024,
-		},
-	}
-
-	fs := &fakeFileStore{
-		PathsFn: func() []tsm1.FileStat {
-			return data
-		},
-	}
-	blocks := []int{
-		tsdb.AggressiveMaxPointsPerBlock,
-		tsdb.AggressiveMaxPointsPerBlock,
-	}
-	err := fs.SetBlockCounts(blocks)
-	require.NoError(t, err, "SetBlockCounts")
-
-	cp := tsm1.NewDefaultPlanner(fs, tsdb.DefaultCompactFullWriteColdDuration)
-
-	compacted, reason := cp.FullyCompacted()
-	reasonExp := ""
-	require.Equal(t, reason, reasonExp, "fullyCompacted reason")
-	require.True(t, compacted, "is fully compacted")
-
-	_, cgLen := cp.PlanLevel(1)
-	require.Zero(t, cgLen, "compaction group length; PlanLevel(1)")
-	_, cgLen = cp.PlanLevel(2)
-	require.Zero(t, cgLen, "compaction group length; PlanLevel(2)")
-	_, cgLen = cp.PlanLevel(3)
-	require.Zero(t, cgLen, "compaction group length; PlanLevel(3)")
-
-	tsmP, pLenP := cp.Plan(time.Now().Add(-time.Second))
-	require.Zero(t, len(tsmP), "compaction group; Plan()")
-	require.Zero(t, pLenP, "compaction group length; Plan()")
-
-	cgroup, cgLen, genLen := cp.PlanOptimize()
-	require.Equal(t, []tsm1.CompactionGroup(nil), cgroup, "compaction group")
-	require.Equal(t, int64(0), cgLen, "compaction group length")
-	require.Equal(t, int64(0), genLen, "generation count")
-}
-
-func TestDefaultPlanner_FullyCompacted_LargeSingleGenerationMaxAggressiveBlocksSmallFileFirst(t *testing.T) {
-	// > 2 GB total group size
-	// 100% of files are at aggressive max block size
-	data := []tsm1.FileStat{
-		{
-			Path: "01-13.tsm1",
-			Size: 691 * 1024 * 1024,
-		},
-		{
-			Path: "01-14.tsm1",
-			Size: 2048 * 1024 * 1024,
-		},
-	}
-
-	fs := &fakeFileStore{
-		PathsFn: func() []tsm1.FileStat {
-			return data
-		},
-	}
-	blocks := []int{
-		tsdb.AggressiveMaxPointsPerBlock,
-		tsdb.AggressiveMaxPointsPerBlock,
-	}
-	err := fs.SetBlockCounts(blocks)
-	require.NoError(t, err, "SetBlockCounts")
-
-	cp := tsm1.NewDefaultPlanner(fs, tsdb.DefaultCompactFullWriteColdDuration)
-
-	compacted, reason := cp.FullyCompacted()
-	reasonExp := ""
-	require.Equal(t, reason, reasonExp, "fullyCompacted reason")
-	require.True(t, compacted, "is fully compacted")
-
-	_, cgLen := cp.PlanLevel(1)
-	require.Zero(t, cgLen, "compaction group length; PlanLevel(1)")
-	_, cgLen = cp.PlanLevel(2)
-	require.Zero(t, cgLen, "compaction group length; PlanLevel(2)")
-	_, cgLen = cp.PlanLevel(3)
-	require.Zero(t, cgLen, "compaction group length; PlanLevel(3)")
-
-	tsmP, pLenP := cp.Plan(time.Now().Add(-time.Second))
-	require.Zero(t, len(tsmP), "compaction group; Plan()")
-	require.Zero(t, pLenP, "compaction group length; Plan()")
-
-	cgroup, cgLen, genLen := cp.PlanOptimize()
-	require.Equal(t, []tsm1.CompactionGroup(nil), cgroup, "compaction group")
-	require.Equal(t, int64(0), cgLen, "compaction group length")
-	require.Equal(t, int64(0), genLen, "generation count")
-}
-
-// This test is added to account for a single generation that has a group size
-// over 2 GB at max points per block with aggressive compaction, and, 1 file
-// under 2 GB at default max points per block.
-// It should not compact any further.
-func TestDefaultPlanner_FullyCompacted_LargeSingleGenerationNoMaxAggrBlocks(t *testing.T) {
-	// > 2 GB total group size
-	// 100% of files are at aggressive max block size
-	data := []tsm1.FileStat{
-		{
-			Path: "01-13.tsm1",
-			Size: 2048 * 1024 * 1024,
-		},
-		{
-			Path: "01-14.tsm1",
-			Size: 691 * 1024 * 1024,
-		},
-	}
-
-	fs := &fakeFileStore{
-		PathsFn: func() []tsm1.FileStat {
-			return data
-		},
-	}
-	blocks := []int{
-		tsdb.AggressiveMaxPointsPerBlock,
-		tsdb.DefaultMaxPointsPerBlock,
-	}
-	err := fs.SetBlockCounts(blocks)
-	require.NoError(t, err, "SetBlockCounts")
-
-	cp := tsm1.NewDefaultPlanner(fs, tsdb.DefaultCompactFullWriteColdDuration)
-
-	compacted, reason := cp.FullyCompacted()
-	reasonExp := ""
-	require.Equal(t, reason, reasonExp, "fullyCompacted reason")
-	require.True(t, compacted, "is fully compacted")
-
-	_, cgLen := cp.PlanLevel(1)
-	require.Zero(t, cgLen, "compaction group length; PlanLevel(1)")
-	_, cgLen = cp.PlanLevel(2)
-	require.Zero(t, cgLen, "compaction group length; PlanLevel(2)")
-	_, cgLen = cp.PlanLevel(3)
-	require.Zero(t, cgLen, "compaction group length; PlanLevel(3)")
-
-	tsmP, pLenP := cp.Plan(time.Now().Add(-time.Second))
-	require.Zero(t, len(tsmP), "compaction group; Plan()")
-	require.Zero(t, pLenP, "compaction group length; Plan()")
-
-	cgroup, cgLen, genLen := cp.PlanOptimize()
-	require.Equal(t, []tsm1.CompactionGroup(nil), cgroup, "compaction group")
-	require.Equal(t, int64(0), cgLen, "compaction group length")
-	require.Equal(t, int64(0), genLen, "generation count")
-}
-
-func TestDefaultPlanner_FullyCompacted_LargeSingleGenerationNoMaxAggrBlocksSmallFileFirst(t *testing.T) {
-	// > 2 GB total group size
-	// 100% of files are at aggressive max block size
-	data := []tsm1.FileStat{
-		{
-			Path: "01-13.tsm1",
-			Size: 691 * 1024 * 1024,
+			[]int{},
+			"", 0,
 		},
 		{
-			Path: "01-14.tsm1",
-			Size: 2048 * 1024 * 1024,
-		},
-	}
-
-	fs := &fakeFileStore{
-		PathsFn: func() []tsm1.FileStat {
-			return data
+			// This test is added to account for a single generation that has a group size
+			// over 2 GB with 1 file under 2 GB all at max points per block with aggressive compaction.
+			// It should not compact any further.
+			[]tsm1.FileStat{
+				{
+					Path: "01-13.tsm1",
+					Size: 2048 * 1024 * 1024,
+				},
+				{
+					Path: "01-14.tsm1",
+					Size: 691 * 1024 * 1024,
+				},
+			}, []int{
+			tsdb.AggressiveMaxPointsPerBlock,
+			tsdb.AggressiveMaxPointsPerBlock,
+		}, "", 0,
 		},
-	}
-	blocks := []int{
-		tsdb.DefaultMaxPointsPerBlock,
-		tsdb.AggressiveMaxPointsPerBlock,
-	}
-	err := fs.SetBlockCounts(blocks)
-	require.NoError(t, err, "SetBlockCounts")
-
-	cp := tsm1.NewDefaultPlanner(fs, tsdb.DefaultCompactFullWriteColdDuration)
-
-	compacted, reason := cp.FullyCompacted()
-	reasonExp := ""
-	require.Equal(t, reason, reasonExp, "fullyCompacted reason")
-	require.True(t, compacted, "is fully compacted")
-
-	_, cgLen := cp.PlanLevel(1)
-	require.Zero(t, cgLen, "compaction group length; PlanLevel(1)")
-	_, cgLen = cp.PlanLevel(2)
-	require.Zero(t, cgLen, "compaction group length; PlanLevel(2)")
-	_, cgLen = cp.PlanLevel(3)
-	require.Zero(t, cgLen, "compaction group length; PlanLevel(3)")
-
-	tsmP, pLenP := cp.Plan(time.Now().Add(-time.Second))
-	require.Zero(t, len(tsmP), "compaction group; Plan()")
-	require.Zero(t, pLenP, "compaction group length; Plan()")
-
-	cgroup, cgLen, genLen := cp.PlanOptimize()
-	require.Equal(t, []tsm1.CompactionGroup(nil), cgroup, "compaction group")
-	require.Equal(t, int64(0), cgLen, "compaction group length")
-	require.Equal(t, int64(0), genLen, "generation count")
-}
-
-// This test is added to account for a single generation that has a group size
-// over 2 GB and multiple files under 2 GB all at max points per block for aggressive compaction.
-func TestDefaultPlanner_FullyCompacted_ManySingleGenLessThen2GBMaxAggrBlocks(t *testing.T) {
-	// > 2 GB total group size
-	// 100% of files are at aggressive max block size
-	data := []tsm1.FileStat{
 		{
-			Path: "01-13.tsm1",
-			Size: 2048 * 1024 * 1024,
+			// This test is added to account for a single generation that has a group size
+			// over 2 GB at max points per block with aggressive compaction, and, 1 file
+			// under 2 GB at default max points per block.
+			// It should not compact any further.
+			[]tsm1.FileStat{
+				{
+					Path: "01-13.tsm1",
+					Size: 2048 * 1024 * 1024,
+				},
+				{
+					Path: "01-14.tsm1",
+					Size: 691 * 1024 * 1024,
+				},
+			}, []int{
+			tsdb.AggressiveMaxPointsPerBlock,
+			tsdb.DefaultMaxPointsPerBlock,
 		},
-		{
-			Path: "01-14.tsm1",
-			Size: 650 * 1024 * 1024,
+			"",
+			0,
 		},
 		{
-			Path: "01-15.tsm1",
-			Size: 450 * 1024 * 1024,
-		},
-	}
-
-	fs := &fakeFileStore{
-		PathsFn: func() []tsm1.FileStat {
-			return data
+			// This test is added to account for a single generation that has a group size
+			// over 2 GB and multiple files under 2 GB all at max points per block for aggressive compaction.
+			[]tsm1.FileStat{
+				{
+					Path: "01-13.tsm1",
+					Size: 2048 * 1024 * 1024,
+				},
+				{
+					Path: "01-14.tsm1",
+					Size: 650 * 1024 * 1024,
+				},
+				{
+					Path: "01-15.tsm1",
+					Size: 450 * 1024 * 1024,
+				},
+			}, []int{
+			tsdb.AggressiveMaxPointsPerBlock,
+			tsdb.AggressiveMaxPointsPerBlock,
+			tsdb.AggressiveMaxPointsPerBlock,
+		}, "", 0,
 		},
 	}
-	blocks := []int{
-		tsdb.AggressiveMaxPointsPerBlock,
-		tsdb.AggressiveMaxPointsPerBlock,
-		tsdb.AggressiveMaxPointsPerBlock,
-	}
-	err := fs.SetBlockCounts(blocks)
-	require.NoError(t, err, "SetBlockCounts")
-
-	cp := tsm1.NewDefaultPlanner(fs, tsdb.DefaultCompactFullWriteColdDuration)
 
-	compacted, reason := cp.FullyCompacted()
-	reasonExp := ""
-	require.Equal(t, reason, reasonExp, "fullyCompacted reason")
-	require.True(t, compacted, "is fully compacted")
-
-	_, cgLen := cp.PlanLevel(1)
-	require.Zero(t, cgLen, "compaction group length; PlanLevel(1)")
-	_, cgLen = cp.PlanLevel(2)
-	require.Zero(t, cgLen, "compaction group length; PlanLevel(2)")
-	_, cgLen = cp.PlanLevel(3)
-	require.Zero(t, cgLen, "compaction group length; PlanLevel(3)")
-
-	tsmP, pLenP := cp.Plan(time.Now().Add(-time.Second))
-	require.Zero(t, len(tsmP), "compaction group; Plan()")
-	require.Zero(t, pLenP, "compaction group length; Plan()")
+	expectedFullyCompacted := func(cp *tsm1.DefaultPlanner, reasonExp string) {
+		compacted, reason := cp.FullyCompacted()
+		require.Equal(t, reason, reasonExp, "fullyCompacted reason")
+		require.True(t, compacted, "is fully compacted")
 
-	cgroup, cgLen, genLen := cp.PlanOptimize()
-	require.Equal(t, []tsm1.CompactionGroup(nil), cgroup, "compaction group")
-	require.Zero(t, cgLen, "compaction group length")
-	require.Zero(t, genLen, "generation count")
-}
+		_, cgLen := cp.PlanLevel(1)
+		require.Zero(t, cgLen, "compaction group length; PlanLevel(1)")
+		_, cgLen = cp.PlanLevel(2)
+		require.Zero(t, cgLen, "compaction group length; PlanLevel(2)")
+		_, cgLen = cp.PlanLevel(3)
+		require.Zero(t, cgLen, "compaction group length; PlanLevel(3)")
 
-func TestDefaultPlanner_FullyCompacted_ManySingleGenLessThen2GBMaxAggrBlocksSmallFileFirst(t *testing.T) {
-	// > 2 GB total group size
-	// 100% of files are at aggressive max block size
-	data := []tsm1.FileStat{
-		{
-			Path: "01-13.tsm1",
-			Size: 450 * 1024 * 1024,
-		},
-		{
-			Path: "01-14.tsm1",
-			Size: 650 * 1024 * 1024,
-		},
-		{
-			Path: "01-15.tsm1",
-			Size: 2048 * 1024 * 1024,
-		},
-	}
+		tsmP, pLenP := cp.Plan(time.Now().Add(-time.Second))
+		require.Zero(t, len(tsmP), "compaction group; Plan()")
+		require.Zero(t, pLenP, "compaction group length; Plan()")
 
-	fs := &fakeFileStore{
-		PathsFn: func() []tsm1.FileStat {
-			return data
-		},
+		cgroup, cgLen, genLen := cp.PlanOptimize()
+		require.Equal(t, []tsm1.CompactionGroup(nil), cgroup, "compaction group")
+		require.Zero(t, cgLen, "compaction group length")
+		require.Zero(t, genLen, "generation count")
 	}
-	blocks := []int{
-		tsdb.AggressiveMaxPointsPerBlock,
-		tsdb.AggressiveMaxPointsPerBlock,
-		tsdb.AggressiveMaxPointsPerBlock,
-	}
-	err := fs.SetBlockCounts(blocks)
-	require.NoError(t, err, "SetBlockCounts")
-
-	cp := tsm1.NewDefaultPlanner(fs, tsdb.DefaultCompactFullWriteColdDuration)
-
-	compacted, reason := cp.FullyCompacted()
-	reasonExp := ""
-	require.Equal(t, reason, reasonExp, "fullyCompacted reason")
-	require.True(t, compacted, "is fully compacted")
-
-	_, cgLen := cp.PlanLevel(1)
-	require.Zero(t, cgLen, "compaction group length; PlanLevel(1)")
-	_, cgLen = cp.PlanLevel(2)
-	require.Zero(t, cgLen, "compaction group length; PlanLevel(2)")
-	_, cgLen = cp.PlanLevel(3)
-	require.Zero(t, cgLen, "compaction group length; PlanLevel(3)")
-
-	tsmP, pLenP := cp.Plan(time.Now().Add(-time.Second))
-	require.Zero(t, len(tsmP), "compaction group; Plan()")
-	require.Zero(t, pLenP, "compaction group length; Plan()")
-
-	cgroup, cgLen, genLen := cp.PlanOptimize()
-	require.Equal(t, []tsm1.CompactionGroup(nil), cgroup, "compaction group")
-	require.Zero(t, cgLen, "compaction group length")
-	require.Zero(t, genLen, "generation count")
-}
 
-func TestDefaultPlanner_FullyCompacted_ManySingleGen2GBLastLevel2MixedSizes(t *testing.T) {
-	// > 2 GB total group size
-	// 100% of files are at aggressive max block size
-	data := []tsm1.FileStat{
-		{
-			Path: "01-05.tsm1",
-			Size: 2048 * 1024 * 1024,
-		},
-		{
-			Path: "01-06.tsm1",
-			Size: 2048 * 1024 * 1024,
-		},
-		{
-			Path: "01-07.tsm1",
-			Size: 2048 * 1024 * 1024,
-		},
-		{
-			Path: "01-08.tsm1",
-			Size: 1048 * 1024 * 1024,
-		},
-		{
-			Path: "02-05.tsm1",
-			Size: 2048 * 1024 * 1024,
-		},
-		{
-			Path: "02-06.tsm1",
-			Size: 2048 * 1024 * 1024,
-		},
-		{
-			Path: "02-07.tsm1",
-			Size: 2048 * 1024 * 1024,
-		},
-		{
-			Path: "02-08.tsm1",
-			Size: 1048 * 1024 * 1024,
-		},
-		{
-			Path: "03-03.tsm1",
-			Size: 600 * 1024 * 1024,
-		},
-		{
-			Path: "03-04.tsm1",
-			Size: 500 * 1024 * 1024,
-		},
-		{
-			Path: "03-04.tsm1",
-			Size: 2048 * 1024 * 1024,
-		},
-		{
-			Path: "03-06.tsm1",
-			Size: 2048 * 1024 * 1024,
-		},
-	}
+	for _, fileStat := range areFullyCompactedTests {
+		ffs := &fakeFileStore{
+			PathsFn: func() []tsm1.FileStat {
+				return fileStat.fs
+			},
+		}
 
-	fs := &fakeFileStore{
-		PathsFn: func() []tsm1.FileStat {
-			return data
-		},
-	}
+		if len(fileStat.bc) > 0 {
+			err := ffs.SetBlockCounts(fileStat.bc)
+			require.NoError(t, err, "setting block counts")
+		}
 
-	cp := tsm1.NewDefaultPlanner(fs, tsdb.DefaultCompactFullWriteColdDuration)
+		cp := tsm1.NewDefaultPlanner(ffs, tsdb.DefaultCompactFullWriteColdDuration)
+		expectedFullyCompacted(cp, fileStat.fullyCompactedReasonExp)
 
-	expFiles := make([]tsm1.FileStat, 0)
-	for _, file := range data {
-		expFiles = append(expFiles, file)
+		// Reverse test files and re-run tests
+		slices.Reverse(fileStat.fs)
+		cp = tsm1.NewDefaultPlanner(ffs, tsdb.DefaultCompactFullWriteColdDuration)
+		expectedFullyCompacted(cp, fileStat.fullyCompactedReasonExp)
 	}
 
-	_, cgLen := cp.PlanLevel(1)
-	require.Zero(t, cgLen, "compaction group length; PlanLevel(1)")
-	_, cgLen = cp.PlanLevel(2)
-	require.Zero(t, cgLen, "compaction group length; PlanLevel(2)")
-	_, cgLen = cp.PlanLevel(3)
-	require.Zero(t, cgLen, "compaction group length; PlanLevel(3)")
-
-	tsmP, pLenP := cp.Plan(time.Now().Add(-time.Second))
-	require.Zero(t, len(tsmP), "compaction group; Plan()")
-	require.Zero(t, pLenP, "compaction group length; Plan()")
-
-	tsm, cgLen, genLen := cp.PlanOptimize()
-	require.Equal(t, int64(1), cgLen, "compaction group length")
-	require.Equal(t, int64(3), genLen, "generation count")
-	require.Equal(t, len(expFiles), len(tsm[0]), "tsm files in compaction group")
 }
 
 func TestDefaultPlanner_PlanOptimize_Tombstones(t *testing.T) {

From 3e69f2d002a7f2cd9547d0299ad5b787fb5b21c4 Mon Sep 17 00:00:00 2001
From: devanbenz <devandbenz@gmail.com>
Date: Mon, 13 Jan 2025 14:02:33 -0600
Subject: [PATCH 28/31] feat: cleanup

---
 tsdb/engine/tsm1/compact_test.go | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/tsdb/engine/tsm1/compact_test.go b/tsdb/engine/tsm1/compact_test.go
index 94df33f5b72..5b21a0f9c95 100644
--- a/tsdb/engine/tsm1/compact_test.go
+++ b/tsdb/engine/tsm1/compact_test.go
@@ -2282,7 +2282,7 @@ func TestDefaultPlanner_PlanOptimize_Test(t *testing.T) {
 
 	furtherCompactedTests := []PlanOptimizeTests{
 		// Large multi generation group with files at and under 2GB
-		PlanOptimizeTests{
+		{
 			[]tsm1.FileStat{
 				{
 					Path: "01-05.tsm1",
@@ -2330,7 +2330,7 @@ func TestDefaultPlanner_PlanOptimize_Test(t *testing.T) {
 			3,
 		},
 		// ~650mb group size
-		PlanOptimizeTests{
+		{
 			[]tsm1.FileStat{
 				{
 					Path: "01-05.tsm1",
@@ -2354,7 +2354,7 @@ func TestDefaultPlanner_PlanOptimize_Test(t *testing.T) {
 			1,
 		},
 		// ~650 MB total group size with generations under 4
-		PlanOptimizeTests{
+		{
 			[]tsm1.FileStat{
 				{
 					Path: "01-02.tsm1",

From 976291aa6b81540d077e8c44b66ce3d7c76b29df Mon Sep 17 00:00:00 2001
From: devanbenz <devandbenz@gmail.com>
Date: Mon, 13 Jan 2025 14:23:05 -0600
Subject: [PATCH 29/31] feat: Add test names to the testing struct

---
 tsdb/engine/tsm1/compact_test.go | 94 ++++++++++++++++++--------------
 1 file changed, 52 insertions(+), 42 deletions(-)

diff --git a/tsdb/engine/tsm1/compact_test.go b/tsdb/engine/tsm1/compact_test.go
index 5b21a0f9c95..86e10527200 100644
--- a/tsdb/engine/tsm1/compact_test.go
+++ b/tsdb/engine/tsm1/compact_test.go
@@ -2274,15 +2274,17 @@ func TestDefaultPlanner_PlanOptimize_Test(t *testing.T) {
 	// bc are the block counts for the tsm files
 	// bc can be an empty list
 	type PlanOptimizeTests struct {
-		fs                      []tsm1.FileStat
-		bc                      []int
-		fullyCompactedReasonExp string
-		generationCount         int64
+		name                            string
+		fs                              []tsm1.FileStat
+		bc                              []int
+		expectedFullyCompactedReasonExp string
+		expectedgenerationCount         int64
 	}
 
 	furtherCompactedTests := []PlanOptimizeTests{
 		// Large multi generation group with files at and under 2GB
 		{
+			"Large multi generation group with files at and under 2GB",
 			[]tsm1.FileStat{
 				{
 					Path: "01-05.tsm1",
@@ -2331,6 +2333,7 @@ func TestDefaultPlanner_PlanOptimize_Test(t *testing.T) {
 		},
 		// ~650mb group size
 		{
+			"Small group size with single generation",
 			[]tsm1.FileStat{
 				{
 					Path: "01-05.tsm1",
@@ -2355,6 +2358,7 @@ func TestDefaultPlanner_PlanOptimize_Test(t *testing.T) {
 		},
 		// ~650 MB total group size with generations under 4
 		{
+			"Small group size with single generation and levels under 4",
 			[]tsm1.FileStat{
 				{
 					Path: "01-02.tsm1",
@@ -2374,6 +2378,7 @@ func TestDefaultPlanner_PlanOptimize_Test(t *testing.T) {
 			1,
 		},
 		{
+			"Small group size with single generation all at DefaultMaxPointsPerBlock",
 			[]tsm1.FileStat{
 				{
 					Path: "01-05.tsm1",
@@ -2395,10 +2400,10 @@ func TestDefaultPlanner_PlanOptimize_Test(t *testing.T) {
 			tsdb.SingleGenerationReasonText,
 			1,
 		},
-
 		// > 2 GB total group size
 		// 50% of files are at aggressive max block size
 		{
+			"Small group size with single generation 50% at DefaultMaxPointsPerBlock and 50% at AggressiveMaxPointsPerBlock",
 			[]tsm1.FileStat{
 				{
 					Path: "01-05.tsm1",
@@ -2447,8 +2452,7 @@ func TestDefaultPlanner_PlanOptimize_Test(t *testing.T) {
 			1,
 		},
 		{
-			// > 2 GB total group size
-			// 100% of files are at aggressive max block size
+			"Group size over 2GB with single generation",
 			[]tsm1.FileStat{
 				{
 					Path: "01-13.tsm1",
@@ -2473,6 +2477,7 @@ func TestDefaultPlanner_PlanOptimize_Test(t *testing.T) {
 		{
 			// Last files are lower than first files generations
 			// Mix of less than 4 level files and > level 4 files
+			"Generations with files under level 4",
 			[]tsm1.FileStat{
 				{
 					Path: "01-05.tsm1",
@@ -2529,6 +2534,7 @@ func TestDefaultPlanner_PlanOptimize_Test(t *testing.T) {
 			// shard with many generations. The initial generation should be fully
 			// compacted, but we have some new generations that are not. We need to ensure
 			// the optimize planner will pick these up and compact everything together.
+			"Backfill mock condition",
 			[]tsm1.FileStat{
 				{
 					Path: "01-05.tsm1",
@@ -2601,47 +2607,47 @@ func TestDefaultPlanner_PlanOptimize_Test(t *testing.T) {
 		},
 	}
 
-	expectedNotFullyCompacted := func(cp *tsm1.DefaultPlanner, reasonExp string, generationCountExp int64) {
+	expectedNotFullyCompacted := func(cp *tsm1.DefaultPlanner, reasonExp string, generationCountExp int64, testName string) {
 		compacted, reason := cp.FullyCompacted()
-		require.Equal(t, reason, reasonExp, "fullyCompacted reason")
-		require.False(t, compacted, "is fully compacted")
+		require.Equal(t, reason, reasonExp, "fullyCompacted reason", testName)
+		require.False(t, compacted, "is fully compacted", testName)
 
 		_, cgLen := cp.PlanLevel(1)
-		require.Zero(t, cgLen, "compaction group length; PlanLevel(1)")
+		require.Zero(t, cgLen, "compaction group length; PlanLevel(1)", testName)
 		_, cgLen = cp.PlanLevel(2)
-		require.Zero(t, cgLen, "compaction group length; PlanLevel(2)")
+		require.Zero(t, cgLen, "compaction group length; PlanLevel(2)", testName)
 		_, cgLen = cp.PlanLevel(3)
-		require.Zero(t, cgLen, "compaction group length; PlanLevel(3)")
+		require.Zero(t, cgLen, "compaction group length; PlanLevel(3)", testName)
 
 		tsmP, pLenP := cp.Plan(time.Now().Add(-time.Second))
-		require.Zero(t, len(tsmP), "compaction group; Plan()")
-		require.Zero(t, pLenP, "compaction group length; Plan()")
+		require.Zero(t, len(tsmP), "compaction group; Plan()", testName)
+		require.Zero(t, pLenP, "compaction group length; Plan()", testName)
 
 		_, cgLen, genLen := cp.PlanOptimize()
-		require.Equal(t, int64(1), cgLen, "compaction group length")
-		require.Equal(t, generationCountExp, genLen, "generation count")
+		require.Equal(t, int64(1), cgLen, "compaction group length", testName)
+		require.Equal(t, generationCountExp, genLen, "generation count", testName)
 
 	}
 
-	for _, fileStat := range furtherCompactedTests {
+	for _, test := range furtherCompactedTests {
 		ffs := &fakeFileStore{
 			PathsFn: func() []tsm1.FileStat {
-				return fileStat.fs
+				return test.fs
 			},
 		}
 
-		if len(fileStat.bc) > 0 {
-			err := ffs.SetBlockCounts(fileStat.bc)
+		if len(test.bc) > 0 {
+			err := ffs.SetBlockCounts(test.bc)
 			require.NoError(t, err, "setting block counts")
 		}
 
 		cp := tsm1.NewDefaultPlanner(ffs, tsdb.DefaultCompactFullWriteColdDuration)
-		expectedNotFullyCompacted(cp, fileStat.fullyCompactedReasonExp, fileStat.generationCount)
+		expectedNotFullyCompacted(cp, test.expectedFullyCompactedReasonExp, test.expectedgenerationCount, test.name)
 
 		// Reverse test files and re-run tests
-		slices.Reverse(fileStat.fs)
+		slices.Reverse(test.fs)
 		cp = tsm1.NewDefaultPlanner(ffs, tsdb.DefaultCompactFullWriteColdDuration)
-		expectedNotFullyCompacted(cp, fileStat.fullyCompactedReasonExp, fileStat.generationCount)
+		expectedNotFullyCompacted(cp, test.expectedFullyCompactedReasonExp, test.expectedgenerationCount, test.name)
 	}
 
 	areFullyCompactedTests := []PlanOptimizeTests{
@@ -2649,6 +2655,7 @@ func TestDefaultPlanner_PlanOptimize_Test(t *testing.T) {
 			// This test is added to account for halting state after
 			// TestDefaultPlanner_FullyCompacted_SmallSingleGeneration
 			// will need to ensure that once we have single TSM file under 2 GB we stop
+			"Single TSM file",
 			[]tsm1.FileStat{
 				{
 					Path: "01-09.tsm1",
@@ -2662,6 +2669,7 @@ func TestDefaultPlanner_PlanOptimize_Test(t *testing.T) {
 			// This test is added to account for a single generation that has a group size
 			// over 2 GB with 1 file under 2 GB all at max points per block with aggressive compaction.
 			// It should not compact any further.
+			"TSM files at AggressiveMaxPointsPerBlock",
 			[]tsm1.FileStat{
 				{
 					Path: "01-13.tsm1",
@@ -2681,6 +2689,7 @@ func TestDefaultPlanner_PlanOptimize_Test(t *testing.T) {
 			// over 2 GB at max points per block with aggressive compaction, and, 1 file
 			// under 2 GB at default max points per block.
 			// It should not compact any further.
+			"TSM files cannot compact further, single file under 2G and at DefaultMaxPointsPerBlock",
 			[]tsm1.FileStat{
 				{
 					Path: "01-13.tsm1",
@@ -2700,6 +2709,7 @@ func TestDefaultPlanner_PlanOptimize_Test(t *testing.T) {
 		{
 			// This test is added to account for a single generation that has a group size
 			// over 2 GB and multiple files under 2 GB all at max points per block for aggressive compaction.
+			"Group size over 2 with multiple files under 2GB and at AggressiveMaxPointsPerBlock",
 			[]tsm1.FileStat{
 				{
 					Path: "01-13.tsm1",
@@ -2721,47 +2731,47 @@ func TestDefaultPlanner_PlanOptimize_Test(t *testing.T) {
 		},
 	}
 
-	expectedFullyCompacted := func(cp *tsm1.DefaultPlanner, reasonExp string) {
+	expectedFullyCompacted := func(cp *tsm1.DefaultPlanner, reasonExp string, testName string) {
 		compacted, reason := cp.FullyCompacted()
-		require.Equal(t, reason, reasonExp, "fullyCompacted reason")
-		require.True(t, compacted, "is fully compacted")
+		require.Equal(t, reason, reasonExp, "fullyCompacted reason", testName)
+		require.True(t, compacted, "is fully compacted", testName)
 
 		_, cgLen := cp.PlanLevel(1)
-		require.Zero(t, cgLen, "compaction group length; PlanLevel(1)")
+		require.Zero(t, cgLen, "compaction group length; PlanLevel(1)", testName)
 		_, cgLen = cp.PlanLevel(2)
-		require.Zero(t, cgLen, "compaction group length; PlanLevel(2)")
+		require.Zero(t, cgLen, "compaction group length; PlanLevel(2)", testName)
 		_, cgLen = cp.PlanLevel(3)
-		require.Zero(t, cgLen, "compaction group length; PlanLevel(3)")
+		require.Zero(t, cgLen, "compaction group length; PlanLevel(3)", testName)
 
 		tsmP, pLenP := cp.Plan(time.Now().Add(-time.Second))
-		require.Zero(t, len(tsmP), "compaction group; Plan()")
-		require.Zero(t, pLenP, "compaction group length; Plan()")
+		require.Zero(t, len(tsmP), "compaction group; Plan()", testName)
+		require.Zero(t, pLenP, "compaction group length; Plan()", testName)
 
 		cgroup, cgLen, genLen := cp.PlanOptimize()
-		require.Equal(t, []tsm1.CompactionGroup(nil), cgroup, "compaction group")
-		require.Zero(t, cgLen, "compaction group length")
-		require.Zero(t, genLen, "generation count")
+		require.Equal(t, []tsm1.CompactionGroup(nil), cgroup, "compaction group", testName)
+		require.Zero(t, cgLen, "compaction group length", testName)
+		require.Zero(t, genLen, "generation count", testName)
 	}
 
-	for _, fileStat := range areFullyCompactedTests {
+	for _, test := range areFullyCompactedTests {
 		ffs := &fakeFileStore{
 			PathsFn: func() []tsm1.FileStat {
-				return fileStat.fs
+				return test.fs
 			},
 		}
 
-		if len(fileStat.bc) > 0 {
-			err := ffs.SetBlockCounts(fileStat.bc)
+		if len(test.bc) > 0 {
+			err := ffs.SetBlockCounts(test.bc)
 			require.NoError(t, err, "setting block counts")
 		}
 
 		cp := tsm1.NewDefaultPlanner(ffs, tsdb.DefaultCompactFullWriteColdDuration)
-		expectedFullyCompacted(cp, fileStat.fullyCompactedReasonExp)
+		expectedFullyCompacted(cp, test.expectedFullyCompactedReasonExp, test.name)
 
 		// Reverse test files and re-run tests
-		slices.Reverse(fileStat.fs)
+		slices.Reverse(test.fs)
 		cp = tsm1.NewDefaultPlanner(ffs, tsdb.DefaultCompactFullWriteColdDuration)
-		expectedFullyCompacted(cp, fileStat.fullyCompactedReasonExp)
+		expectedFullyCompacted(cp, test.expectedFullyCompactedReasonExp, test.name)
 	}
 
 }

From 0a2ba1ec3ff204af2550bdc18ad2d53dbed6274b Mon Sep 17 00:00:00 2001
From: devanbenz <devandbenz@gmail.com>
Date: Tue, 14 Jan 2025 09:18:26 -0600
Subject: [PATCH 30/31] feat: Use t.Run instead of declaring the test name in
 the requires

---
 tsdb/engine/tsm1/compact_test.go | 109 ++++++++++++++++---------------
 1 file changed, 55 insertions(+), 54 deletions(-)

diff --git a/tsdb/engine/tsm1/compact_test.go b/tsdb/engine/tsm1/compact_test.go
index 86e10527200..e73edfb418f 100644
--- a/tsdb/engine/tsm1/compact_test.go
+++ b/tsdb/engine/tsm1/compact_test.go
@@ -2271,8 +2271,6 @@ func TestDefaultPlanner_PlanOptimize_NoLevel4(t *testing.T) {
 }
 
 func TestDefaultPlanner_PlanOptimize_Test(t *testing.T) {
-	// bc are the block counts for the tsm files
-	// bc can be an empty list
 	type PlanOptimizeTests struct {
 		name                            string
 		fs                              []tsm1.FileStat
@@ -2607,47 +2605,49 @@ func TestDefaultPlanner_PlanOptimize_Test(t *testing.T) {
 		},
 	}
 
-	expectedNotFullyCompacted := func(cp *tsm1.DefaultPlanner, reasonExp string, generationCountExp int64, testName string) {
+	expectedNotFullyCompacted := func(cp *tsm1.DefaultPlanner, reasonExp string, generationCountExp int64) {
 		compacted, reason := cp.FullyCompacted()
-		require.Equal(t, reason, reasonExp, "fullyCompacted reason", testName)
-		require.False(t, compacted, "is fully compacted", testName)
+		require.Equal(t, reason, reasonExp, "fullyCompacted reason")
+		require.False(t, compacted, "is fully compacted")
 
 		_, cgLen := cp.PlanLevel(1)
-		require.Zero(t, cgLen, "compaction group length; PlanLevel(1)", testName)
+		require.Zero(t, cgLen, "compaction group length; PlanLevel(1)")
 		_, cgLen = cp.PlanLevel(2)
-		require.Zero(t, cgLen, "compaction group length; PlanLevel(2)", testName)
+		require.Zero(t, cgLen, "compaction group length; PlanLevel(2)")
 		_, cgLen = cp.PlanLevel(3)
-		require.Zero(t, cgLen, "compaction group length; PlanLevel(3)", testName)
+		require.Zero(t, cgLen, "compaction group length; PlanLevel(3)")
 
 		tsmP, pLenP := cp.Plan(time.Now().Add(-time.Second))
-		require.Zero(t, len(tsmP), "compaction group; Plan()", testName)
-		require.Zero(t, pLenP, "compaction group length; Plan()", testName)
+		require.Zero(t, len(tsmP), "compaction group; Plan()")
+		require.Zero(t, pLenP, "compaction group length; Plan()")
 
 		_, cgLen, genLen := cp.PlanOptimize()
-		require.Equal(t, int64(1), cgLen, "compaction group length", testName)
-		require.Equal(t, generationCountExp, genLen, "generation count", testName)
+		require.Equal(t, int64(1), cgLen, "compaction group length")
+		require.Equal(t, generationCountExp, genLen, "generation count")
 
 	}
 
 	for _, test := range furtherCompactedTests {
-		ffs := &fakeFileStore{
-			PathsFn: func() []tsm1.FileStat {
-				return test.fs
-			},
-		}
+		t.Run(test.name, func(t *testing.T) {
+			ffs := &fakeFileStore{
+				PathsFn: func() []tsm1.FileStat {
+					return test.fs
+				},
+			}
 
-		if len(test.bc) > 0 {
-			err := ffs.SetBlockCounts(test.bc)
-			require.NoError(t, err, "setting block counts")
-		}
+			if len(test.bc) > 0 {
+				err := ffs.SetBlockCounts(test.bc)
+				require.NoError(t, err, "setting block counts")
+			}
 
-		cp := tsm1.NewDefaultPlanner(ffs, tsdb.DefaultCompactFullWriteColdDuration)
-		expectedNotFullyCompacted(cp, test.expectedFullyCompactedReasonExp, test.expectedgenerationCount, test.name)
+			cp := tsm1.NewDefaultPlanner(ffs, tsdb.DefaultCompactFullWriteColdDuration)
+			expectedNotFullyCompacted(cp, test.expectedFullyCompactedReasonExp, test.expectedgenerationCount)
 
-		// Reverse test files and re-run tests
-		slices.Reverse(test.fs)
-		cp = tsm1.NewDefaultPlanner(ffs, tsdb.DefaultCompactFullWriteColdDuration)
-		expectedNotFullyCompacted(cp, test.expectedFullyCompactedReasonExp, test.expectedgenerationCount, test.name)
+			// Reverse test files and re-run tests
+			slices.Reverse(test.fs)
+			cp = tsm1.NewDefaultPlanner(ffs, tsdb.DefaultCompactFullWriteColdDuration)
+			expectedNotFullyCompacted(cp, test.expectedFullyCompactedReasonExp, test.expectedgenerationCount)
+		})
 	}
 
 	areFullyCompactedTests := []PlanOptimizeTests{
@@ -2731,49 +2731,50 @@ func TestDefaultPlanner_PlanOptimize_Test(t *testing.T) {
 		},
 	}
 
-	expectedFullyCompacted := func(cp *tsm1.DefaultPlanner, reasonExp string, testName string) {
+	expectedFullyCompacted := func(cp *tsm1.DefaultPlanner, reasonExp string) {
 		compacted, reason := cp.FullyCompacted()
-		require.Equal(t, reason, reasonExp, "fullyCompacted reason", testName)
-		require.True(t, compacted, "is fully compacted", testName)
+		require.Equal(t, reason, reasonExp, "fullyCompacted reason")
+		require.True(t, compacted, "is fully compacted")
 
 		_, cgLen := cp.PlanLevel(1)
-		require.Zero(t, cgLen, "compaction group length; PlanLevel(1)", testName)
+		require.Zero(t, cgLen, "compaction group length; PlanLevel(1)")
 		_, cgLen = cp.PlanLevel(2)
-		require.Zero(t, cgLen, "compaction group length; PlanLevel(2)", testName)
+		require.Zero(t, cgLen, "compaction group length; PlanLevel(2)")
 		_, cgLen = cp.PlanLevel(3)
-		require.Zero(t, cgLen, "compaction group length; PlanLevel(3)", testName)
+		require.Zero(t, cgLen, "compaction group length; PlanLevel(3)")
 
 		tsmP, pLenP := cp.Plan(time.Now().Add(-time.Second))
-		require.Zero(t, len(tsmP), "compaction group; Plan()", testName)
-		require.Zero(t, pLenP, "compaction group length; Plan()", testName)
+		require.Zero(t, len(tsmP), "compaction group; Plan()")
+		require.Zero(t, pLenP, "compaction group length; Plan()")
 
 		cgroup, cgLen, genLen := cp.PlanOptimize()
-		require.Equal(t, []tsm1.CompactionGroup(nil), cgroup, "compaction group", testName)
-		require.Zero(t, cgLen, "compaction group length", testName)
-		require.Zero(t, genLen, "generation count", testName)
+		require.Equal(t, []tsm1.CompactionGroup(nil), cgroup, "compaction group")
+		require.Zero(t, cgLen, "compaction group length")
+		require.Zero(t, genLen, "generation count")
 	}
 
 	for _, test := range areFullyCompactedTests {
-		ffs := &fakeFileStore{
-			PathsFn: func() []tsm1.FileStat {
-				return test.fs
-			},
-		}
+		t.Run(test.name, func(t *testing.T) {
+			ffs := &fakeFileStore{
+				PathsFn: func() []tsm1.FileStat {
+					return test.fs
+				},
+			}
 
-		if len(test.bc) > 0 {
-			err := ffs.SetBlockCounts(test.bc)
-			require.NoError(t, err, "setting block counts")
-		}
+			if len(test.bc) > 0 {
+				err := ffs.SetBlockCounts(test.bc)
+				require.NoError(t, err, "setting block counts")
+			}
 
-		cp := tsm1.NewDefaultPlanner(ffs, tsdb.DefaultCompactFullWriteColdDuration)
-		expectedFullyCompacted(cp, test.expectedFullyCompactedReasonExp, test.name)
+			cp := tsm1.NewDefaultPlanner(ffs, tsdb.DefaultCompactFullWriteColdDuration)
+			expectedFullyCompacted(cp, test.expectedFullyCompactedReasonExp)
 
-		// Reverse test files and re-run tests
-		slices.Reverse(test.fs)
-		cp = tsm1.NewDefaultPlanner(ffs, tsdb.DefaultCompactFullWriteColdDuration)
-		expectedFullyCompacted(cp, test.expectedFullyCompactedReasonExp, test.name)
+			// Reverse test files and re-run tests
+			slices.Reverse(test.fs)
+			cp = tsm1.NewDefaultPlanner(ffs, tsdb.DefaultCompactFullWriteColdDuration)
+			expectedFullyCompacted(cp, test.expectedFullyCompactedReasonExp)
+		})
 	}
-
 }
 
 func TestDefaultPlanner_PlanOptimize_Tombstones(t *testing.T) {

From 8c908c547b9812e31ab8cf3fa5923ef8ede27631 Mon Sep 17 00:00:00 2001
From: devanbenz <devandbenz@gmail.com>
Date: Tue, 14 Jan 2025 12:36:14 -0600
Subject: [PATCH 31/31] feat: Reverse block counts

---
 tsdb/engine/tsm1/compact_test.go | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/tsdb/engine/tsm1/compact_test.go b/tsdb/engine/tsm1/compact_test.go
index e73edfb418f..40bc8cbdf59 100644
--- a/tsdb/engine/tsm1/compact_test.go
+++ b/tsdb/engine/tsm1/compact_test.go
@@ -2645,6 +2645,12 @@ func TestDefaultPlanner_PlanOptimize_Test(t *testing.T) {
 
 			// Reverse test files and re-run tests
 			slices.Reverse(test.fs)
+			if len(test.bc) > 0 {
+				slices.Reverse(test.bc)
+				err := ffs.SetBlockCounts(test.bc)
+				require.NoError(t, err, "setting reverse block counts")
+			}
+
 			cp = tsm1.NewDefaultPlanner(ffs, tsdb.DefaultCompactFullWriteColdDuration)
 			expectedNotFullyCompacted(cp, test.expectedFullyCompactedReasonExp, test.expectedgenerationCount)
 		})
@@ -2771,6 +2777,12 @@ func TestDefaultPlanner_PlanOptimize_Test(t *testing.T) {
 
 			// Reverse test files and re-run tests
 			slices.Reverse(test.fs)
+			if len(test.bc) > 0 {
+				slices.Reverse(test.bc)
+				err := ffs.SetBlockCounts(test.bc)
+				require.NoError(t, err, "setting reverse block counts")
+			}
+
 			cp = tsm1.NewDefaultPlanner(ffs, tsdb.DefaultCompactFullWriteColdDuration)
 			expectedFullyCompacted(cp, test.expectedFullyCompactedReasonExp)
 		})