From b2b41f3a097c8a308582abfb252622c1a4c0759c Mon Sep 17 00:00:00 2001 From: Dipack P Panjabi Date: Wed, 3 Jul 2019 11:32:04 -0400 Subject: [PATCH] Compute WAL size and use it during retention size checks Signed-off-by: Dipack P Panjabi --- block_test.go | 10 +++++++--- db.go | 6 +++++- db_test.go | 7 +++++-- fileutil/dir.go | 33 +++++++++++++++++++++++++++++++++ wal/wal.go | 6 ++++++ wal/wal_test.go | 7 +++++-- 6 files changed, 61 insertions(+), 8 deletions(-) create mode 100644 fileutil/dir.go diff --git a/block_test.go b/block_test.go index 3f39a899..adf8eec0 100644 --- a/block_test.go +++ b/block_test.go @@ -25,6 +25,8 @@ import ( "strconv" "testing" + "github.com/prometheus/tsdb/fileutil" + "github.com/go-kit/kit/log" "github.com/prometheus/tsdb/chunks" "github.com/prometheus/tsdb/labels" @@ -175,7 +177,8 @@ func TestBlockSize(t *testing.T) { testutil.Ok(t, blockInit.Close()) }() expSizeInit = blockInit.Size() - actSizeInit := testutil.DirSize(t, blockInit.Dir()) + actSizeInit, err := fileutil.DirSize(blockInit.Dir()) + testutil.Ok(t, err) testutil.Equals(t, expSizeInit, actSizeInit) } @@ -184,7 +187,7 @@ func TestBlockSize(t *testing.T) { testutil.Ok(t, blockInit.Delete(1, 10, labels.NewMustRegexpMatcher("", ".*"))) expAfterDelete := blockInit.Size() testutil.Assert(t, expAfterDelete > expSizeInit, "after a delete the block size should be bigger as the tombstone file should grow %v > %v", expAfterDelete, expSizeInit) - actAfterDelete := testutil.DirSize(t, blockDirInit) + actAfterDelete, err := fileutil.DirSize(blockDirInit) testutil.Ok(t, err) testutil.Equals(t, expAfterDelete, actAfterDelete, "after a delete reported block size doesn't match actual disk size") @@ -198,7 +201,8 @@ func TestBlockSize(t *testing.T) { testutil.Ok(t, blockAfterCompact.Close()) }() expAfterCompact := blockAfterCompact.Size() - actAfterCompact := testutil.DirSize(t, blockAfterCompact.Dir()) + actAfterCompact, err := fileutil.DirSize(blockAfterCompact.Dir()) + testutil.Ok(t, err) testutil.Assert(t, actAfterDelete > actAfterCompact, "after a delete and compaction the block size should be smaller %v,%v", actAfterDelete, actAfterCompact) testutil.Equals(t, expAfterCompact, actAfterCompact, "after a delete and compaction reported block size doesn't match actual disk size") } diff --git a/db.go b/db.go index ec830027..85e38683 100644 --- a/db.go +++ b/db.go @@ -890,7 +890,11 @@ func (db *DB) beyondSizeRetention(blocks []*Block) (deleteable map[ulid.ULID]*Bl } deleteable = make(map[ulid.ULID]*Block) - blocksSize := int64(0) + walSize, _ := db.Head().wal.Size() + + // Initializing the size counter with the WAL size, + // as that is part of the retention strategy as well. + blocksSize := walSize for i, block := range blocks { blocksSize += block.Size() if blocksSize > db.opts.MaxBytes { diff --git a/db_test.go b/db_test.go index 25fb8a7e..a2cec062 100644 --- a/db_test.go +++ b/db_test.go @@ -31,6 +31,7 @@ import ( "github.com/prometheus/client_golang/prometheus" prom_testutil "github.com/prometheus/client_golang/prometheus/testutil" "github.com/prometheus/tsdb/chunks" + "github.com/prometheus/tsdb/fileutil" "github.com/prometheus/tsdb/index" "github.com/prometheus/tsdb/labels" "github.com/prometheus/tsdb/testutil" @@ -1113,7 +1114,8 @@ func TestSizeRetention(t *testing.T) { testutil.Ok(t, db.reload()) // Reload the db to register the new db size. testutil.Equals(t, len(blocks), len(db.Blocks())) // Ensure all blocks are registered. expSize := int64(prom_testutil.ToFloat64(db.metrics.blocksBytes)) // Use the the actual internal metrics. - actSize := testutil.DirSize(t, db.Dir()) + actSize, err := fileutil.DirSize(db.Dir()) + testutil.Ok(t, err) testutil.Equals(t, expSize, actSize, "registered size doesn't match actual disk size") // Decrease the max bytes limit so that a delete is triggered. @@ -1127,7 +1129,8 @@ func TestSizeRetention(t *testing.T) { actBlocks := db.Blocks() expSize = int64(prom_testutil.ToFloat64(db.metrics.blocksBytes)) actRetentCount := int(prom_testutil.ToFloat64(db.metrics.sizeRetentionCount)) - actSize = testutil.DirSize(t, db.Dir()) + actSize, err = fileutil.DirSize(db.Dir()) + testutil.Ok(t, err) testutil.Equals(t, 1, actRetentCount, "metric retention count mismatch") testutil.Equals(t, actSize, expSize, "metric db size doesn't match actual disk size") diff --git a/fileutil/dir.go b/fileutil/dir.go new file mode 100644 index 00000000..e6ac4ec9 --- /dev/null +++ b/fileutil/dir.go @@ -0,0 +1,33 @@ +// Copyright 2019 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package fileutil + +import ( + "os" + "path/filepath" +) + +func DirSize(dir string) (int64, error) { + var size int64 + err := filepath.Walk(dir, func(filePath string, info os.FileInfo, err error) error { + if err != nil { + return err + } + if !info.IsDir() { + size += info.Size() + } + return nil + }) + return size, err +} diff --git a/wal/wal.go b/wal/wal.go index 75d6dc3d..98e26064 100644 --- a/wal/wal.go +++ b/wal/wal.go @@ -854,3 +854,9 @@ func (r *segmentBufReader) Read(b []byte) (n int, err error) { r.buf.Reset(r.segs[r.cur]) return n, nil } + +// Computing size of the WAL. +// We do this by adding the sizes of all the files under the WAL dir. +func (w *WAL) Size() (int64, error) { + return fileutil.DirSize(w.Dir()) +} diff --git a/wal/wal_test.go b/wal/wal_test.go index 12fe1a2d..e8646000 100644 --- a/wal/wal_test.go +++ b/wal/wal_test.go @@ -24,6 +24,7 @@ import ( "testing" client_testutil "github.com/prometheus/client_golang/prometheus/testutil" + "github.com/prometheus/tsdb/fileutil" "github.com/prometheus/tsdb/testutil" ) @@ -408,8 +409,10 @@ func TestCompression(t *testing.T) { testutil.Ok(t, os.RemoveAll(dirUnCompressed)) }() - uncompressedSize := testutil.DirSize(t, dirUnCompressed) - compressedSize := testutil.DirSize(t, dirCompressed) + uncompressedSize, err := fileutil.DirSize(dirUnCompressed) + testutil.Ok(t, err) + compressedSize, err := fileutil.DirSize(dirCompressed) + testutil.Ok(t, err) testutil.Assert(t, float64(uncompressedSize)*0.75 > float64(compressedSize), "Compressing zeroes should save at least 25%% space - uncompressedSize: %d, compressedSize: %d", uncompressedSize, compressedSize) }