Allow controlling max manifests per content with env var

repo/manifest/committed_manifest_manager.go

@@ -9,6 +9,7 @@ import (
 	"math/rand"
 	"os"
 	"sort"
+	"strconv"
 	"sync"
 
 	"github.com/pkg/errors"
@@ -18,7 +19,10 @@ import (
 	"github.com/kopia/kopia/repo/content/index"
 )
 
-const maxManifestsPerContent = 1000000
+const (
+	defaultMaxManifestsPerContent = 1000000
+	maxManifestsPerContentEnvKey  = "KOPIA_MAX_MANIFESTS_PER_CONTENT_COUNT"
+)
 
 // committedManifestManager manages committed manifest entries stored in 'm' contents.
 type committedManifestManager struct {
@@ -110,6 +114,18 @@ func (m *committedManifestManager) writeContentChunk(
 	return contentID, nil
 }
 
+func getMaxManifestsPerContent() int {
+	retVal := defaultMaxManifestsPerContent
+
+	if v := os.Getenv(maxManifestsPerContentEnvKey); v != "" {
+		if vint, err := strconv.Atoi(v); err == nil {
+			retVal = vint
+		}
+	}
+
+	return retVal
+}
+
 // writeEntriesLocked writes entries in the provided map as manifest contents
 // and removes all entries from the map when complete and returns the set of content IDs written
 // (typically one).
@@ -140,7 +156,12 @@ func (m *committedManifestManager) writeEntriesLocked(
 
 		// Still write all entries out in a single content piece if we're not
 		// compacting things.
-		if !isCompaction || len(man.Entries) < maxManifestsPerContent {
+		//
+		// Even if the result of getMaxManifestsPer content is 0 or negative, we'll
+		// still make progress because this is a less-than check. At worst, if
+		// there's a configuration that asks for a negative or zero manifests per
+		// content piece then we'll write out each manifest individually.
+		if !isCompaction || len(man.Entries) < getMaxManifestsPerContent() {
 			continue
 		}
 

repo/manifest/manifest_manager_test.go

@@ -501,7 +501,7 @@ func TestWriteManyManifests(t *testing.T) {
 	data := blobtesting.DataMap{}
 	item1 := map[string]int{"foo": 1, "bar": 2}
 	labels1 := map[string]string{"type": "item", "color": "red"}
-	numManifests := maxManifestsPerContent + 5
+	numManifests := defaultMaxManifestsPerContent + 5
 
 	mgr := newManagerForTesting(ctx, t, data, ManagerOptions{})
 
@@ -522,44 +522,87 @@ func TestWriteManyManifests(t *testing.T) {
 }
 
 func TestCompactManyManifests(t *testing.T) {
-	ctx := testlogging.Context(t)
-	data := blobtesting.DataMap{}
-	item1 := map[string]int{"foo": 1, "bar": 2}
-	labels1 := map[string]string{"type": "item", "color": "red"}
+	table := []struct {
+		name             string
+		envFlag          string
+		initialManifests int
+		otherManifests   int
+		expectContents   int
+	}{
+		{
+			name:             "DefaultValue",
+			envFlag:          "",
+			initialManifests: defaultMaxManifestsPerContent - 1,
+			otherManifests:   6,
+			expectContents:   2,
+		},
+		{
+			name:             "SmallerEnvValue",
+			envFlag:          "100",
+			initialManifests: 99,
+			otherManifests:   6,
+			expectContents:   2,
+		},
+		{
+			name:             "ZeroEnvValue",
+			envFlag:          "0",
+			initialManifests: 3,
+			otherManifests:   3,
+			expectContents:   6,
+		},
+		{
+			name:             "NegativeEnvValue",
+			envFlag:          "-42",
+			initialManifests: 3,
+			otherManifests:   3,
+			expectContents:   6,
+		},
+	}
 
-	mgr := newManagerForTesting(ctx, t, data, ManagerOptions{})
+	for _, test := range table {
+		t.Run(test.name, func(t *testing.T) {
+			ctx := testlogging.Context(t)
+			data := blobtesting.DataMap{}
+			item1 := map[string]int{"foo": 1, "bar": 2}
+			labels1 := map[string]string{"type": "item", "color": "red"}
 
-	for i := 0; i < maxManifestsPerContent-1; i++ {
-		addAndVerify(ctx, t, mgr, labels1, item1)
-	}
+			t.Setenv(maxManifestsPerContentEnvKey, test.envFlag)
 
-	require.NoError(t, mgr.Flush(ctx))
-	require.NoError(t, mgr.b.Flush(ctx))
+			mgr := newManagerForTesting(ctx, t, data, ManagerOptions{})
 
-	// Should only have a single content piece since this wasn't compaction.
-	foundContents := getManifestContentCount(ctx, t, mgr)
-	assert.Equal(t, 1, foundContents)
+			for i := 0; i < test.initialManifests; i++ {
+				addAndVerify(ctx, t, mgr, labels1, item1)
+			}
 
-	// Add individually so we can tell that compaction deleted the old content
-	// pieces.
-	for i := 0; i < 6; i++ {
-		addAndVerify(ctx, t, mgr, labels1, item1)
+			require.NoError(t, mgr.Flush(ctx))
+			require.NoError(t, mgr.b.Flush(ctx))
 
-		require.NoError(t, mgr.Flush(ctx))
-		require.NoError(t, mgr.b.Flush(ctx))
-	}
+			// Should only have a single content piece since this wasn't compaction.
+			foundContents := getManifestContentCount(ctx, t, mgr)
+			assert.Equal(t, 1, foundContents)
 
-	foundContents = getManifestContentCount(ctx, t, mgr)
-	assert.Equal(t, 7, foundContents)
+			// Add individually so we can tell that compaction deleted the old content
+			// pieces.
+			for i := 0; i < test.otherManifests; i++ {
+				addAndVerify(ctx, t, mgr, labels1, item1)
 
-	// Run compaction which should result in multiple content pieces.
-	err := mgr.Compact(ctx)
-	require.NoError(t, err, "compacting manifests")
+				require.NoError(t, mgr.Flush(ctx))
+				require.NoError(t, mgr.b.Flush(ctx))
+			}
 
-	foundContents = getManifestContentCount(ctx, t, mgr)
-	assert.Equal(t, 2, foundContents)
+			foundContents = getManifestContentCount(ctx, t, mgr)
+			assert.Equal(t, test.otherManifests+1, foundContents)
 
-	mans, err := mgr.Find(ctx, map[string]string{"color": "red"})
-	assert.NoError(t, err)
-	assert.Len(t, mans, maxManifestsPerContent+5)
+			// Run compaction which should result in multiple content pieces.
+			err := mgr.Compact(ctx)
+			require.NoError(t, err, "compacting manifests")
+
+			foundContents = getManifestContentCount(ctx, t, mgr)
+			assert.Equal(t, test.expectContents, foundContents)
+
+			mans, err := mgr.Find(ctx, map[string]string{"color": "red"})
+			assert.NoError(t, err)
+			assert.Len(t, mans, test.initialManifests+test.otherManifests)
+		})
+	}
 }