From d9da1932d24ad07bcf9634bc2afe768622802d0e Mon Sep 17 00:00:00 2001 From: t_max <1172915550@qq.com> Date: Tue, 27 Aug 2024 16:26:39 +0800 Subject: [PATCH] feat: add compress and reserved disk size --- .github/workflows/ci.yml | 34 +-- .gitignore | 1 + example_test.go | 3 +- go.mod | 21 +- go.sum | 37 ++- interface.go | 40 ++- internal/fileutil/fileutil.go | 15 +- lock.go | 17 ++ lock_windows.go | 20 ++ options.go | 36 ++- rotatelogs.go | 544 +++++++++++++++++++++++++++------- rotatelogs_test.go | 76 ++++- space.go | 19 ++ space_windows.go | 23 ++ 14 files changed, 694 insertions(+), 192 deletions(-) create mode 100644 lock.go create mode 100644 lock_windows.go create mode 100644 space.go create mode 100644 space_windows.go diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 4a1e853..ed6d98c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -6,33 +6,17 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - go: [ '1.15', '1.14' ] + go: [ '1.17', 'stable' ] name: Go ${{ matrix.go }} test steps: - - name: Checkout repository - uses: actions/checkout@v2 - - name: Install Go stable version - if: matrix.go != 'tip' - uses: actions/setup-go@v2 + - name: checkout TDengine by pr + uses: actions/checkout@v4 + + - uses: actions/setup-go@v5 with: go-version: ${{ matrix.go }} - - name: Install Go tip - if: matrix.go == 'tip' - run: | - git clone --depth=1 https://go.googlesource.com/go $HOME/gotip - cd $HOME/gotip/src - ./make.bash - echo "::set-env name=GOROOT::$HOME/gotip" - echo "::add-path::$HOME/gotip/bin" - echo "::add-path::$(go env GOPATH)/bin" - - name: Install GolangCI-Lint - run: curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(go env GOPATH)/bin v1.27.0 - - name: Test with coverage - run: make cover - - name: Upload code coverage to codecov - if: matrix.go == 'tip' - uses: codecov/codecov-action@v1 - with: - file: ./coverage.out - - run: make lint + + - name: test + id: test + run: sudo go test -v --count=1 diff --git a/.gitignore b/.gitignore index 04fcde8..f3863e2 100644 --- a/.gitignore +++ b/.gitignore @@ -24,3 +24,4 @@ _testmain.go /.idea/file-rotatelogs.iml /.idea/modules.xml /.idea/vcs.xml +/.idea/remote-targets.xml diff --git a/example_test.go b/example_test.go index 850703d..77800e5 100644 --- a/example_test.go +++ b/example_test.go @@ -4,6 +4,7 @@ import ( "fmt" "io/ioutil" "os" + "path/filepath" rotatelogs "github.com/taosdata/file-rotatelogs/v2" ) @@ -15,7 +16,7 @@ func ExampleForceNewFile() { return } - logPath := fmt.Sprintf("%s/test.log", logDir) + logPath := filepath.Join(logDir, "test.log") for i := 0; i < 2; i++ { writer, err := rotatelogs.New(logPath, diff --git a/go.mod b/go.mod index 55c82cf..a104583 100644 --- a/go.mod +++ b/go.mod @@ -1,14 +1,17 @@ module github.com/taosdata/file-rotatelogs/v2 -go 1.12 +go 1.17 require ( - github.com/fastly/go-utils v0.0.0-20180712184237-d95a45783239 // indirect - github.com/jehiah/go-strftime v0.0.0-20171201141054-1d33003b3869 // indirect - github.com/jonboulle/clockwork v0.1.0 - github.com/lestrrat-go/envload v0.0.0-20180220234015-a3eb8ddeffcc // indirect - github.com/lestrrat-go/strftime v0.0.0-20180821113735-8b31f9c59b0f - github.com/pkg/errors v0.8.1 - github.com/stretchr/testify v1.3.0 - github.com/tebeka/strftime v0.1.3 // indirect + github.com/jonboulle/clockwork v0.4.0 + github.com/lestrrat-go/strftime v1.0.6 + github.com/pkg/errors v0.9.1 + github.com/stretchr/testify v1.9.0 + golang.org/x/sys v0.24.0 +) + +require ( + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index f38e69e..864ec45 100644 --- a/go.sum +++ b/go.sum @@ -1,21 +1,30 @@ -github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/fastly/go-utils v0.0.0-20180712184237-d95a45783239 h1:Ghm4eQYC0nEPnSJdVkTrXpu9KtoVCSo1hg7mtI7G9KU= -github.com/fastly/go-utils v0.0.0-20180712184237-d95a45783239/go.mod h1:Gdwt2ce0yfBxPvZrHkprdPPTTS3N5rwmLE8T22KBXlw= -github.com/jehiah/go-strftime v0.0.0-20171201141054-1d33003b3869 h1:IPJ3dvxmJ4uczJe5YQdrYB16oTJlGSC/OyZDqUk9xX4= -github.com/jehiah/go-strftime v0.0.0-20171201141054-1d33003b3869/go.mod h1:cJ6Cj7dQo+O6GJNiMx+Pa94qKj+TG8ONdKHgMNIyyag= -github.com/jonboulle/clockwork v0.1.0 h1:VKV+ZcuP6l3yW9doeqz6ziZGgcynBVQO+obU0+0hcPo= -github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/jonboulle/clockwork v0.4.0 h1:p4Cf1aMWXnXAUh8lVfewRBx1zaTSYKrKMF2g3ST4RZ4= +github.com/jonboulle/clockwork v0.4.0/go.mod h1:xgRqUGwRcjKCO1vbZUEtSLrqKoPSsUpK7fnezOII0kc= github.com/lestrrat-go/envload v0.0.0-20180220234015-a3eb8ddeffcc h1:RKf14vYWi2ttpEmkA4aQ3j4u9dStX2t4M8UM6qqNsG8= github.com/lestrrat-go/envload v0.0.0-20180220234015-a3eb8ddeffcc/go.mod h1:kopuH9ugFRkIXf3YoqHKyrJ9YfUFsckUU9S7B+XP+is= -github.com/lestrrat-go/strftime v0.0.0-20180821113735-8b31f9c59b0f h1:/o/LRlB6dBTBNViFglNdGfsDHBjdL8Yvfm7qQE4ZUh0= -github.com/lestrrat-go/strftime v0.0.0-20180821113735-8b31f9c59b0f/go.mod h1:RMlXygAD3c48Psmr06d2G75L4E4xxzxkIe/+ppX9eAU= -github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= -github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/lestrrat-go/strftime v1.0.6 h1:CFGsDEt1pOpFNU+TJB0nhz9jl+K0hZSLE205AhTIGQQ= +github.com/lestrrat-go/strftime v1.0.6/go.mod h1:f7jQKgV5nnJpYgdEasS+/y7EsTb8ykN2z68n3TtcTaw= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= -github.com/tebeka/strftime v0.1.3 h1:5HQXOqWKYRFfNyBMNVc9z5+QzuBtIXy03psIhtdJYto= -github.com/tebeka/strftime v0.1.3/go.mod h1:7wJm3dZlpr4l/oVK0t1HYIc4rMzQ2XJlOMIUJUJH6XQ= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +golang.org/x/sys v0.24.0 h1:Twjiwq9dn6R1fQcyiK+wQyHWfaz/BJB+YIpzU/Cv3Xg= +golang.org/x/sys v0.24.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/interface.go b/interface.go index 9a04ab4..8b865cc 100644 --- a/interface.go +++ b/interface.go @@ -33,21 +33,31 @@ type FileRotatedEvent struct { // RotateLogs represents a log file that gets // automatically rotated as you write to it. type RotateLogs struct { - clock Clock - curFn string - curBaseFn string - globPattern string - generation int - linkName string - maxAge time.Duration - mutex sync.RWMutex - eventHandler Handler - outFh *os.File - pattern *strftime.Strftime - rotationTime time.Duration - rotationSize int64 - rotationCount uint - forceNewFile bool + clock Clock + curFn string + curBaseFn string + globPattern string + generation int + linkName string + lockFilename string + maxAge time.Duration + mutex sync.RWMutex + eventHandler Handler + outFh *os.File + lockFh *os.File + pattern *strftime.Strftime + rotationTime time.Duration + rotationSize int64 + rotationCount uint + forceNewFile bool + logPath string + reservedDiskSize int64 + availDiskSize int64 + compress bool + rotateCleanChan chan struct{} + exitChan chan struct{} + done chan struct{} + cleanLockName string } // Clock is the interface used by the RotateLogs diff --git a/internal/fileutil/fileutil.go b/internal/fileutil/fileutil.go index 346f4fb..4134f44 100644 --- a/internal/fileutil/fileutil.go +++ b/internal/fileutil/fileutil.go @@ -3,6 +3,7 @@ package fileutil import ( "os" "path/filepath" + "runtime" "time" "github.com/lestrrat-go/strftime" @@ -41,7 +42,7 @@ func GenerateFn(pattern *strftime.Strftime, clock interface{ Now() time.Time }, // CreateFile creates a new file in the given path, creating parent directories // as necessary -func CreateFile(filename string) (*os.File, error) { +func CreateFile(filename string, appendFile bool) (*os.File, error) { // make sure the dir is existed, eg: // ./foo/bar/baz/hello.log must make sure ./foo/bar/baz is existed dirname := filepath.Dir(filename) @@ -49,7 +50,17 @@ func CreateFile(filename string) (*os.File, error) { return nil, errors.Wrapf(err, "failed to create directory %s", dirname) } // if we got here, then we need to create a file - fh, err := os.OpenFile(filename, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0644) + var fh *os.File + var err error + if appendFile { + if runtime.GOOS == "windows" { + fh, err = os.OpenFile(filename, os.O_CREATE|os.O_APPEND, 0644) + } else { + fh, err = os.OpenFile(filename, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0644) + } + } else { + fh, err = os.OpenFile(filename, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0644) + } if err != nil { return nil, errors.Errorf("failed to open file %s: %s", filename, err) } diff --git a/lock.go b/lock.go new file mode 100644 index 0000000..971d61b --- /dev/null +++ b/lock.go @@ -0,0 +1,17 @@ +//go:build !windows +// +build !windows + +package rotatelogs + +import ( + "os" + "syscall" +) + +func LockFile(file *os.File) error { + return syscall.Flock(int(file.Fd()), syscall.LOCK_EX|syscall.LOCK_NB) +} + +func UnlockFile(file *os.File) error { + return syscall.Flock(int(file.Fd()), syscall.LOCK_UN|syscall.LOCK_NB) +} diff --git a/lock_windows.go b/lock_windows.go new file mode 100644 index 0000000..7971e9b --- /dev/null +++ b/lock_windows.go @@ -0,0 +1,20 @@ +//go:build windows +// +build windows + +package rotatelogs + +import ( + "os" + + "golang.org/x/sys/windows" +) + +func LockFile(file *os.File) error { + ol := new(windows.Overlapped) + return windows.LockFileEx(windows.Handle(file.Fd()), uint32(windows.LOCKFILE_EXCLUSIVE_LOCK|windows.LOCKFILE_FAIL_IMMEDIATELY), 0, ^uint32(0), ^uint32(0), ol) +} + +func UnlockFile(file *os.File) error { + ol := new(windows.Overlapped) + return windows.UnlockFileEx(windows.Handle(file.Fd()), 0, ^uint32(0), ^uint32(0), ol) +} diff --git a/options.go b/options.go index 9599cb5..13e0b6d 100644 --- a/options.go +++ b/options.go @@ -7,14 +7,18 @@ import ( ) const ( - optkeyClock = "clock" - optkeyHandler = "handler" - optkeyLinkName = "link-name" - optkeyMaxAge = "max-age" - optkeyRotationTime = "rotation-time" - optkeyRotationSize = "rotation-size" - optkeyRotationCount = "rotation-count" - optkeyForceNewFile = "force-new-file" + optkeyClock = "clock" + optkeyHandler = "handler" + optkeyLinkName = "link-name" + optkeyMaxAge = "max-age" + optkeyRotationTime = "rotation-time" + optkeyRotationSize = "rotation-size" + optkeyRotationCount = "rotation-count" + optkeyForceNewFile = "force-new-file" + optKeyReservedDiskSize = "reserved-disk-size" + optKeyGlobPattern = "glob-pattern" + optKeyCompress = "compress" + optKeyCleanLockFile = "clean-lock-file" ) // WithClock creates a new Option that sets a clock @@ -87,3 +91,19 @@ func WithHandler(h Handler) Option { func ForceNewFile() Option { return option.New(optkeyForceNewFile, true) } + +func WithReservedDiskSize(size int64) Option { + return option.New(optKeyReservedDiskSize, size) +} + +func WithRotateGlobPattern(pattern string) Option { + return option.New(optKeyGlobPattern, pattern) +} + +func WithCompress(compress bool) Option { + return option.New(optKeyCompress, compress) +} + +func WithCleanLockFile(lockName string) Option { + return option.New(optKeyCleanLockFile, lockName) +} diff --git a/rotatelogs.go b/rotatelogs.go index b7801bc..aed93ad 100644 --- a/rotatelogs.go +++ b/rotatelogs.go @@ -5,21 +5,26 @@ package rotatelogs import ( + "compress/gzip" + "context" "fmt" "io" "os" "path/filepath" "regexp" "sort" + "strconv" "strings" - "sync" + "sync/atomic" "time" - strftime "github.com/lestrrat-go/strftime" + "github.com/lestrrat-go/strftime" "github.com/pkg/errors" "github.com/taosdata/file-rotatelogs/v2/internal/fileutil" ) +const CheckDiskSizeInterval = 25 * time.Second + func (c clockFn) Now() time.Time { return c() } @@ -27,6 +32,11 @@ func (c clockFn) Now() time.Time { // New creates a new RotateLogs object. A log filename pattern // must be passed. Optional `Option` parameters may be passed func New(p string, options ...Option) (*RotateLogs, error) { + logPath := filepath.Dir(p) + isRootDir := isRoot(logPath) + if isRootDir { + return nil, errors.New("log path can't be root directory") + } globPattern := p + "*" for _, re := range patternConversionRegexps { globPattern = re.ReplaceAllString(globPattern, "*") @@ -36,7 +46,6 @@ func New(p string, options ...Option) (*RotateLogs, error) { if err != nil { return nil, errors.Wrap(err, `invalid strftime pattern`) } - var clock Clock = Local rotationTime := 24 * time.Hour var rotationSize int64 @@ -45,6 +54,9 @@ func New(p string, options ...Option) (*RotateLogs, error) { var maxAge time.Duration var handler Handler var forceNewFile bool + var reservedDiskSize int64 + var compress bool + var cleanLockName = filepath.Join(logPath, ".rotate_clean_lock") for _, o := range options { switch o.Name() { @@ -73,6 +85,14 @@ func New(p string, options ...Option) (*RotateLogs, error) { handler = o.Value().(Handler) case optkeyForceNewFile: forceNewFile = true + case optKeyReservedDiskSize: + reservedDiskSize = o.Value().(int64) + case optKeyGlobPattern: + globPattern = o.Value().(string) + case optKeyCompress: + compress = o.Value().(bool) + case optKeyCleanLockFile: + cleanLockName = o.Value().(string) } } @@ -85,18 +105,64 @@ func New(p string, options ...Option) (*RotateLogs, error) { maxAge = 7 * 24 * time.Hour } - return &RotateLogs{ - clock: clock, - eventHandler: handler, - globPattern: globPattern, - linkName: linkName, - maxAge: maxAge, - pattern: pattern, - rotationTime: rotationTime, - rotationSize: rotationSize, - rotationCount: rotationCount, - forceNewFile: forceNewFile, - }, nil + rl := &RotateLogs{ + clock: clock, + eventHandler: handler, + globPattern: globPattern, + linkName: linkName, + maxAge: maxAge, + pattern: pattern, + rotationTime: rotationTime, + rotationSize: rotationSize, + rotationCount: rotationCount, + forceNewFile: forceNewFile, + logPath: logPath, + reservedDiskSize: reservedDiskSize, + compress: compress, + rotateCleanChan: make(chan struct{}, 5), + exitChan: make(chan struct{}), + done: make(chan struct{}), + cleanLockName: cleanLockName, + } + if reservedDiskSize > 0 { + err = os.MkdirAll(rl.logPath, 0755) + if err != nil { + return nil, errors.Wrap(err, fmt.Sprintf("make dir for %s fail", rl.logPath)) + } + _, avail, err := GetDiskSize(rl.logPath) + if err != nil { + return nil, errors.Wrap(err, `failed to get disk size`) + } + rl.availDiskSize = int64(avail) + } + go func() { rl.waitSignal() }() + return rl, nil +} + +func isRoot(path string) bool { + cleanPath := filepath.Clean(path) + return cleanPath == "/" || cleanPath == "C:\\" +} + +func (rl *RotateLogs) waitSignal() { + ticker := time.NewTicker(CheckDiskSizeInterval) + go func() { + for { + select { + case <-ticker.C: + _, avail, err := GetDiskSize(rl.logPath) + if err == nil { + atomic.StoreInt64(&rl.availDiskSize, int64(avail)) + } + case <-rl.rotateCleanChan: + rl.rotateClean() + case <-rl.exitChan: + ticker.Stop() + close(rl.done) + return + } + } + }() } // Write satisfies the io.Writer interface. It writes to the @@ -107,7 +173,9 @@ func (rl *RotateLogs) Write(p []byte) (n int, err error) { // Guard against concurrent writes rl.mutex.Lock() defer rl.mutex.Unlock() - + if !rl.diskSpaceEnough() { + return 0, errors.New("disk space is not enough") + } out, err := rl.getWriterNolock(false, false) if err != nil { return 0, errors.Wrap(err, `failed to acquite target io.Writer`) @@ -116,31 +184,65 @@ func (rl *RotateLogs) Write(p []byte) (n int, err error) { return out.Write(p) } +func (rl *RotateLogs) diskSpaceEnough() bool { + if rl.reservedDiskSize <= 0 { + return true + } + return atomic.LoadInt64(&rl.availDiskSize) > rl.reservedDiskSize +} + // must be locked during this operation -func (rl *RotateLogs) getWriterNolock(bailOnRotateFail, useGenerationalNames bool) (io.Writer, error) { +func (rl *RotateLogs) getWriterNolock(bailOnRotateFail, useGenerationalNames bool) (*os.File, error) { generation := rl.generation previousFn := rl.curFn // This filename contains the name of the "NEW" filename // to log to, which may be newer than rl.currentFilename baseFn := fileutil.GenerateFn(rl.pattern, rl.clock, rl.rotationTime) - filename := baseFn var forceNewFile bool - + filename := baseFn fi, err := os.Stat(rl.curFn) sizeRotation := false if err == nil && rl.rotationSize > 0 && rl.rotationSize <= fi.Size() { forceNewFile = true sizeRotation = true } + if rl.curBaseFn == "" { + // first start, check the generation number + var latestLogFile os.DirEntry + generation, latestLogFile, err = getLatestLogFile(baseFn) + if latestLogFile != nil { + // end with gz means it is compressed, need to force new file + if strings.HasSuffix(latestLogFile.Name(), ".gz") { + forceNewFile = true + generation += 1 + } else { + if rl.forceNewFile { + forceNewFile = true + generation += 1 + } else { + // check if the latest log file is full + info, err := latestLogFile.Info() + if err == nil && rl.rotationSize > 0 && rl.rotationSize > info.Size() { + // use latestLogFile as current file + filename = filepath.Join(rl.logPath, latestLogFile.Name()) + } else { + // ignore error, just force new file + forceNewFile = true + generation += 1 + } + } + } + } else { + // no log file, just use baseFn + generation = 0 + filename = baseFn + } - if baseFn != rl.curBaseFn { + } else if baseFn != rl.curBaseFn { generation = 0 - // even though this is the first write after calling New(), - // check if a new file needs to be created - if rl.forceNewFile { - forceNewFile = true - } + // The base filename has changed. This means we need to create a new file + forceNewFile = true } else { if !useGenerationalNames && !sizeRotation { // nothing to do @@ -149,56 +251,68 @@ func (rl *RotateLogs) getWriterNolock(bailOnRotateFail, useGenerationalNames boo forceNewFile = true generation++ } + + var fh *os.File + var lockFh *os.File + lockfileName := filename + `_lock` + if !forceNewFile { + // Try to open the file and lock it, if we can't, then we need to create a new file + fh, lockFh, err = createLogFileAndLock(filename, lockfileName, true) + if err != nil { + forceNewFile = true + generation += 1 + } + } if forceNewFile { // A new file has been requested. Instead of just using the // regular strftime pattern, we create a new file name using // generational names such as "foo.1", "foo.2", "foo.3", etc var name string + tryTimes := 0 for { + if tryTimes > 100 { + return nil, errors.New("try create new files too many times") + } if generation == 0 { - name = filename + name = baseFn } else { - name = fmt.Sprintf("%s.%d", filename, generation) + name = fmt.Sprintf("%s.%d", baseFn, generation) } - if _, err := os.Stat(name); err != nil { - filename = name - - break + lockfileName = name + `_lock` + fh, lockFh, err = createLogFileAndLock(name, lockfileName, false) + if err != nil { + tryTimes += 1 + generation += 1 + continue } - generation++ + filename = name + break } } - - fh, err := fileutil.CreateFile(filename) - if err != nil { - return nil, errors.Wrapf(err, `failed to create a new file %v`, filename) + if rl.lockFh != nil { + UnlockFile(rl.lockFh) + rl.lockFh.Close() + rl.removeFile(rl.lockFilename) } - + if rl.outFh != nil { + UnlockFile(rl.outFh) + rl.outFh.Close() + } + rl.lockFh = lockFh + rl.outFh = fh + rl.curBaseFn = baseFn + rl.curFn = filename + rl.generation = generation + rl.lockFilename = lockfileName + // make link and rotate if err := rl.rotateNolock(filename); err != nil { err = errors.Wrap(err, "failed to rotate") if bailOnRotateFail { - // Failure to rotate is a problem, but it's really not a great - // idea to stop your application just because you couldn't rename - // your log. - // - // We only return this error when explicitly needed (as specified by bailOnRotateFail) - // - // However, we *NEED* to close `fh` here - if fh != nil { // probably can't happen, but being paranoid - fh.Close() - } - return nil, err } fmt.Fprintf(os.Stderr, "%s\n", err.Error()) } - rl.outFh.Close() - rl.outFh = fh - rl.curBaseFn = baseFn - rl.curFn = filename - rl.generation = generation - if h := rl.eventHandler; h != nil { go h.Handle(&FileRotatedEvent{ prev: previousFn, @@ -209,6 +323,70 @@ func (rl *RotateLogs) getWriterNolock(bailOnRotateFail, useGenerationalNames boo return fh, nil } +func createLogFileAndLock(filename string, lockFilename string, appendFile bool) (*os.File, *os.File, error) { + // Try to open the file and lock it, if we can't, then we need to create a new file + logFileHandle, err := fileutil.CreateFile(filename, appendFile) + if err != nil { + return nil, nil, err + } + // check file in use + err = LockFile(logFileHandle) + if err != nil { + UnlockFile(logFileHandle) + logFileHandle.Close() + return nil, nil, err + } + // create lock file and lock it + lockHandle, err := os.OpenFile(lockFilename, os.O_CREATE, 0644) + if err != nil { + UnlockFile(logFileHandle) + logFileHandle.Close() + return nil, nil, err + } + err = LockFile(lockHandle) + if err != nil { + // lock lockfile failed + UnlockFile(logFileHandle) + logFileHandle.Close() + lockHandle.Close() + return nil, nil, err + } + // unlock log file + UnlockFile(logFileHandle) + return logFileHandle, lockHandle, nil +} + +func getLatestLogFile(baseFn string) (int, os.DirEntry, error) { + dir, file := filepath.Split(baseFn) + pattern := regexp.QuoteMeta(file) + `.(\d+)` + re := regexp.MustCompile(pattern) + files, err := os.ReadDir(dir) + if err != nil { + return 0, nil, errors.Wrap(err, `failed to read directory`) + } + generation := 0 + var latestLogFile os.DirEntry + for _, f := range files { + fName := f.Name() + if strings.HasSuffix(fName, "_lock") || strings.HasSuffix(fName, "_symlink") || strings.HasSuffix(fName, ".tmp") { + continue + } + if fName == file { + latestLogFile = f + continue + } + matches := re.FindStringSubmatch(fName) + if len(matches) > 1 { + num, err := strconv.Atoi(matches[1]) + if err == nil && num > generation { + generation = num + latestLogFile = f + } + } + } + return generation, latestLogFile, nil +} + // CurrentFileName returns the current file name that // the RotateLogs object is writing to func (rl *RotateLogs) CurrentFileName() string { @@ -223,22 +401,6 @@ var patternConversionRegexps = []*regexp.Regexp{ regexp.MustCompile(`\*+`), } -type cleanupGuard struct { - enable bool - fn func() - mutex sync.Mutex -} - -func (g *cleanupGuard) Enable() { - g.mutex.Lock() - defer g.mutex.Unlock() - g.enable = true -} - -func (g *cleanupGuard) Run() { - g.fn() -} - // Rotate forcefully rotates the log files. If the generated file name // clash because file already exists, a numeric suffix of the form // ".1", ".2", ".3" and so forth are appended to the end of the log file @@ -255,20 +417,6 @@ func (rl *RotateLogs) Rotate() error { } func (rl *RotateLogs) rotateNolock(filename string) error { - lockfn := filename + `_lock` - fh, err := os.OpenFile(lockfn, os.O_CREATE|os.O_EXCL, 0644) - if err != nil { - // Can't lock, just return - return err - } - - var guard cleanupGuard - guard.fn = func() { - fh.Close() - os.Remove(lockfn) - } - defer guard.Run() - if rl.linkName != "" { tmpLinkName := filename + `_symlink` @@ -309,32 +457,64 @@ func (rl *RotateLogs) rotateNolock(filename string) error { if rl.maxAge <= 0 && rl.rotationCount <= 0 { return errors.New("panic: maxAge and rotationCount are both set") } + select { + case rl.rotateCleanChan <- struct{}{}: + default: + } + return nil +} + +func (rl *RotateLogs) rotateClean() error { + // open lock file + cleanLockFile, err := os.OpenFile(rl.cleanLockName, os.O_CREATE, 0644) + if err != nil { + return errors.Wrap(err, `failed to acquire clean lock`) + } + getLocked := false + defer func() { + cleanLockFile.Close() + // if you get locked, remove lock file after clean + if getLocked { + rl.removeFile(rl.cleanLockName) + } + }() + // lock file + err = LockFile(cleanLockFile) + if err != nil { + return errors.Wrap(err, `failed to lock clean lock file`) + } + getLocked = true + // get log files with glob pattern matches, err := filepath.Glob(rl.globPattern) if err != nil { return err } cutoff := rl.clock.Now().Add(-1 * rl.maxAge) - - toUnlinkFl := make([]os.FileInfo, 0, len(matches)) + var toCompress []string + toCleanFl := make([]os.FileInfo, 0, len(matches)) m := make(map[os.FileInfo]string) - for _, path := range matches { - // Ignore lock files - if strings.HasSuffix(path, "_lock") || strings.HasSuffix(path, "_symlink") { + for _, p := range matches { + // Ignore lock files and tmp files + if strings.HasSuffix(p, "_lock") || strings.HasSuffix(p, "_symlink") || strings.HasSuffix(p, ".tmp") { + continue + } + // Ignore files that are not in the same directory as the log file + if filepath.Dir(p) != rl.logPath { continue } - fi, err := os.Stat(path) + fi, err := os.Stat(p) if err != nil { continue } - fl, err := os.Lstat(path) + fl, err := os.Lstat(p) if err != nil { continue } - + toCompress = append(toCompress, p) if rl.maxAge > 0 && fi.ModTime().After(cutoff) { continue } @@ -342,38 +522,159 @@ func (rl *RotateLogs) rotateNolock(filename string) error { if rl.rotationCount > 0 && fl.Mode()&os.ModeSymlink == os.ModeSymlink { continue } - m[fl] = path - toUnlinkFl = append(toUnlinkFl, fl) + m[fl] = p + toCleanFl = append(toCleanFl, fl) } - sort.Slice(toUnlinkFl, func(i, j int) bool { - return toUnlinkFl[i].ModTime().Before(toUnlinkFl[j].ModTime()) + // sort by mod time + sort.Slice(toCleanFl, func(i, j int) bool { + return toCleanFl[i].ModTime().Before(toCleanFl[j].ModTime()) }) - toUnlink := make([]string, 0, len(toUnlinkFl)) - for _, info := range toUnlinkFl { - toUnlink = append(toUnlink, m[info]) + toClean := make([]string, 0, len(toCleanFl)) + for _, info := range toCleanFl { + toClean = append(toClean, m[info]) } - if rl.rotationCount > 0 { + + var toRemove []string + if rl.rotationCount > 0 && uint(len(toClean)) > rl.rotationCount { // Only delete if we have more than rotationCount - if rl.rotationCount >= uint(len(toUnlink)) { - return nil - } + toRemove = toClean[:len(toClean)-int(rl.rotationCount)] + } - toUnlink = toUnlink[:len(toUnlink)-int(rl.rotationCount)] + if rl.maxAge > 0 { + toRemove = toClean } - if len(toUnlink) <= 0 { - return nil + for _, path := range toRemove { + if rl.checkLogInuse(path) { + continue + } + rl.removeFile(path) } - guard.Enable() - go func() { - // unlink files on a separate goroutine - for _, path := range toUnlink { - os.Remove(path) + if rl.compress { + tmp := make(map[string]struct{}, len(toRemove)) + for _, item := range toRemove { + tmp[item] = struct{}{} + } + currentFileName := rl.CurrentFileName() + var tempFile string + for i := 0; i < len(toCompress); i++ { + // ignore compressed file and current file + if strings.HasSuffix(toCompress[i], ".gz") || toCompress[i] == currentFileName { + continue + } + if _, ok := tmp[toCompress[i]]; ok { + continue + } + if rl.checkLogInuse(toCompress[i]) { + continue + } + // compress file to .gz.tmp + tempFile = toCompress[i] + ".gz.tmp" + err = compressFile(toCompress[i], tempFile) + if err != nil { + rl.removeFile(tempFile) + continue + } + // rename temp file to .gz + err = os.Rename(tempFile, toCompress[i]+".gz") + if err != nil { + rl.removeFile(tempFile) + continue + } + // remove original file + rl.removeFile(toCompress[i]) + } + } + return nil +} + +func (rl *RotateLogs) checkLogInuse(filename string) bool { + fileHandle, err := os.Open(filename) + if err != nil { + // if open file failed, return in use + return true + } + defer fileHandle.Close() + err = LockFile(fileHandle) + if err != nil { + // if lock file failed, return in use + return true + } + UnlockFile(fileHandle) + lockFile := filename + `_lock` + lockHandle, err := os.Open(lockFile) + if err != nil { + // open lock file failed, return not in use + return false + } + cleanLockFile := false + defer func() { + if cleanLockFile { + rl.removeFile(lockFile) } }() + defer lockHandle.Close() + stat, err := lockHandle.Stat() + if err != nil { + // get stat failed, return in use, maybe lock file is broken + return true + } + if time.Now().Sub(stat.ModTime()) < time.Second*1 { + // lock file modify time is less than 1 second, return in use, maybe lock file is not locked + return true + } + err = LockFile(lockHandle) + if err != nil { + // lock failed, return in use + return true + } + // lock success means lockfile not in use, return not in use, and remove lock file + UnlockFile(lockHandle) + cleanLockFile = true + return false +} - return nil +func compressFile(fileName, gzTempFile string) error { + // open file and lock it + f, err := os.Open(fileName) + if err != nil { + return errors.Wrap(err, fmt.Sprintf("failed to open file %s", fileName)) + } + defer f.Close() + err = LockFile(f) + if err != nil { + return errors.Wrap(err, "failed to lock file") + } + tempFile, err := os.OpenFile(gzTempFile, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0644) + if err != nil { + return err + } + defer tempFile.Close() + w := gzip.NewWriter(tempFile) + defer w.Close() + _, err = io.Copy(w, f) + if err != nil { + return err + } + w.Flush() + return err +} + +func (rl *RotateLogs) removeFile(fileName string) error { + // ensure the file is in the log path and not a directory + dir := filepath.Dir(fileName) + if dir != rl.logPath { + return errors.Errorf("remove file error, %s is not in log path", fileName) + } + info, err := os.Stat(fileName) + if err != nil { + return errors.Wrap(err, fmt.Sprintf("failed to stat file %s", fileName)) + } + if info.IsDir() { + return fmt.Errorf("remove file error, %s is a directory", fileName) + } + return os.Remove(fileName) } // Close satisfies the io.Closer interface. You must @@ -386,9 +687,22 @@ func (rl *RotateLogs) Close() error { if rl.outFh == nil { return nil } - + close(rl.exitChan) + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + var err error + select { + case <-rl.done: + case <-ctx.Done(): + err = errors.New("close timeout") + } rl.outFh.Close() rl.outFh = nil - - return nil + if rl.lockFh != nil { + UnlockFile(rl.lockFh) + rl.lockFh.Close() + rl.lockFh = nil + rl.removeFile(rl.lockFilename) + } + return err } diff --git a/rotatelogs_test.go b/rotatelogs_test.go index 07a008a..ae12b9b 100644 --- a/rotatelogs_test.go +++ b/rotatelogs_test.go @@ -252,7 +252,13 @@ func TestLogRotationCount(t *testing.T) { } time.Sleep(time.Second) files, _ := filepath.Glob(filepath.Join(dir, "log*")) - if !assert.Equal(t, 1, len(files), "Only latest log is kept") { + var filteredFiles []string + for _, file := range files { + if !strings.HasSuffix(file, "_lock") { + filteredFiles = append(filteredFiles, file) + } + } + if !assert.Equal(t, 1, len(filteredFiles), "Only latest log is kept") { return } }) @@ -279,7 +285,13 @@ func TestLogRotationCount(t *testing.T) { } time.Sleep(time.Second) files, _ := filepath.Glob(filepath.Join(dir, "log*")) - if !assert.Equal(t, 2, len(files), "One file is kept") { + var filteredFiles []string + for _, file := range files { + if !strings.HasSuffix(file, "_lock") { + filteredFiles = append(filteredFiles, file) + } + } + if !assert.Equal(t, 2, len(filteredFiles), "One file is kept") { return } }) @@ -334,7 +346,7 @@ func TestGHIssue16(t *testing.T) { rl, err := rotatelogs.New( filepath.Join(dir, "log%Y%m%d%H%M%S"), - rotatelogs.WithLinkName("./test.log"), + rotatelogs.WithLinkName(filepath.Join(dir, "./test.log")), rotatelogs.WithRotationTime(10*time.Second), rotatelogs.WithRotationCount(3), rotatelogs.WithMaxAge(-1), @@ -580,3 +592,61 @@ func TestForceNewFile(t *testing.T) { } }) } + +func TestCompress(t *testing.T) { + dir, err := ioutil.TempDir("", "file-rotatelogs-compress") + if !assert.NoError(t, err, `creating temporary directory should succeed`) { + return + } + defer os.RemoveAll(dir) + rl, err := rotatelogs.New( + filepath.Join(dir, "compress.log"), + rotatelogs.ForceNewFile(), + rotatelogs.WithCompress(true), + rotatelogs.WithRotateGlobPattern(filepath.Join(dir, "compress.log*")), + rotatelogs.WithCleanLockFile(filepath.Join(dir, "clean.lock")), + ) + if !assert.NoError(t, err, "rotatelogs.New should succeed") { + return + } + _, err = rl.Write([]byte("Hello, World!")) + if !assert.NoError(t, err, "rl.Write should succeed") { + return + } + err = rl.Rotate() + if !assert.NoError(t, err, "rl.Rotate should succeed") { + return + } + rl.Write([]byte("Hello, World!")) + time.Sleep(time.Second) + files, _ := filepath.Glob(filepath.Join(dir, "compress.log.gz")) + if !assert.Len(t, files, 1, "only one compressed file should exist") { + return + } + rl.Close() +} + +func TestReservedDiskSize(t *testing.T) { + dir, err := ioutil.TempDir("", "file-rotatelogs-reserved") + if !assert.NoError(t, err, `creating temporary directory should succeed`) { + return + } + defer os.RemoveAll(dir) + total, avail, err := rotatelogs.GetDiskSize(dir) + if !assert.NoError(t, err, "GetDiskSize should succeed") { + return + } + t.Log("total disk size:", total, "available disk size:", avail) + rl, err := rotatelogs.New( + filepath.Join(dir, "reserved.log"), + rotatelogs.WithReservedDiskSize(int64(total)), + ) + if !assert.NoError(t, err, "rotatelogs.New should succeed") { + return + } + _, err = rl.Write([]byte("Hello, World!")) + if !assert.Error(t, err, "rl.Write should fail") { + return + } + rl.Close() +} diff --git a/space.go b/space.go new file mode 100644 index 0000000..f7b0d17 --- /dev/null +++ b/space.go @@ -0,0 +1,19 @@ +//go:build !windows +// +build !windows + +package rotatelogs + +import ( + "golang.org/x/sys/unix" +) + +func GetDiskSize(dir string) (total uint64, avail uint64, err error) { + fs := unix.Statfs_t{} + err = unix.Statfs(dir, &fs) + if err != nil { + return 0, 0, err + } + avail = fs.Bavail * uint64(fs.Frsize) + total = fs.Blocks * uint64(fs.Frsize) + return total, avail, nil +} diff --git a/space_windows.go b/space_windows.go new file mode 100644 index 0000000..cc6ab98 --- /dev/null +++ b/space_windows.go @@ -0,0 +1,23 @@ +//go:build windows +// +build windows + +package rotatelogs + +import ( + "fmt" + + "golang.org/x/sys/windows" +) + +func GetDiskSize(dir string) (total uint64, avail uint64, err error) { + d, err := windows.UTF16PtrFromString(dir) + if err != nil { + return 0, 0, fmt.Errorf(`failed to call UTF16PtrFromString, d:%s, err: %s`, dir, err) + } + var freeBytesAvailableToCaller, totalNumberOfBytes, totalNumberOfFreeBytes uint64 + err = windows.GetDiskFreeSpaceEx(d, &freeBytesAvailableToCaller, &totalNumberOfBytes, &totalNumberOfFreeBytes) + if err != nil { + return 0, 0, err + } + return totalNumberOfBytes, freeBytesAvailableToCaller, nil +}