diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml new file mode 100644 index 0000000..8c4bec0 --- /dev/null +++ b/.github/workflows/build_and_test.yml @@ -0,0 +1,46 @@ +# This workflow will build a golang project +# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-go + +name: Build And Test + +on: + push: + tags: + branches: + pull_request: + branches: + +jobs: + + build-and-test: + runs-on: ubuntu-latest + strategy: + matrix: + go-version: [ '1.21' ] + + steps: + - uses: actions/checkout@v3 + + - name: Set up Go ${{ matrix.go-version }} + uses: actions/setup-go@v4 + with: + go-version: ${{ matrix.go-version }} + + - name: Display Go version + run: go version + + - name: Install dependencies + run: | + go mod download + go get -t -u golang.org/x/tools/cmd/cover + + - name: Build + run: go build -v ./... + + - name: Test + run: go test -v -race -coverprofile=coverage.out -covermode=atomic + + - name: Upload coverage reports to Codecov + uses: codecov/codecov-action@v3 + env: + CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} diff --git a/README.md b/README.md index cf42af8..7303430 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,13 @@ # slog-dedup +[![tag](https://img.shields.io/github/tag/veqryn/slog-dedup.svg)](https://github.com/veqryn/slog-dedup/releases) +![Go Version](https://img.shields.io/badge/Go-%3E%3D%201.21-%23007d9c) +[![GoDoc](https://godoc.org/github.com/veqryn/slog-dedup?status.svg)](https://pkg.go.dev/github.com/veqryn/slog-dedup) +![Build Status](https://github.com/veqryn/slog-dedup/actions/workflows/build_and_test.yml/badge.svg) +[![Go report](https://goreportcard.com/badge/github.com/veqryn/slog-dedup)](https://goreportcard.com/report/github.com/veqryn/slog-dedup) +[![Coverage](https://img.shields.io/codecov/c/github/veqryn/slog-dedup)](https://codecov.io/gh/veqryn/slog-dedup) +[![Contributors](https://img.shields.io/github/contributors/veqryn/slog-dedup)](https://github.com/veqryn/slog-dedup/graphs/contributors) +[![License](https://img.shields.io/github/license/veqryn/slog-dedup)](./LICENSE) + Golang structured logging (slog) deduplication for use with json logging (or any other format where duplicates are not appreciated). The slog handlers in this module are "middleware" handlers. When creating them, you must pass in another handler, which will be called after this handler has finished handling a log record. Because of this, these handlers can be chained with other middlewares, and can be used with many different final handlers, whether from the stdlib or third-party, such as json, protobuf, text, or data sinks. @@ -7,6 +16,7 @@ The main impetus behind this package is because most JSON tools do not like dupl Unfortunately the default behavior of the stdlib slog handlers is to allow duplicate keys: ```go +// This make json tools unhappy :( slog.SetDefault(slog.New(slog.NewJSONHandler(os.Stdout, nil))) slog.Info("this is the stdlib json handler by itself", slog.String("duplicated", "zero"), @@ -27,6 +37,10 @@ Outputs: ``` With this in mind, this repo was created with several different ways of deduplicating the keys. +## Install +`go get github.com/veqryn/slog-dedup` + +## Usage ### Overwrite Older Duplicates Handler ```go logger := slog.New(dedup.NewOverwriteHandler(slog.NewJSONHandler(os.Stdout, nil), nil)) diff --git a/append_handler_test.go b/append_handler_test.go index c2f0b77..26c37ba 100644 --- a/append_handler_test.go +++ b/append_handler_test.go @@ -1,6 +1,7 @@ package dedup import ( + "log/slog" "strings" "testing" ) @@ -110,3 +111,40 @@ func TestAppendHandler(t *testing.T) { checkRecordForDuplicates(t, tester.Record) } + +/* + { + "time": "2023-09-29T13:00:59Z", + "level": "INFO", + "msg": "case insenstive, keep builtin conflict", + "arg1": ["val1","val2"], + "msg":"builtin-conflict" + } +*/ +func TestAppendHandler_CaseInsensitiveKeepIfBuiltinConflict(t *testing.T) { + t.Parallel() + + tester := &testHandler{} + h := NewAppendHandler(tester, &AppendHandlerOptions{ + KeyCompare: CaseInsensitiveCmp, + ResolveBuiltinKeyConflict: KeepIfBuiltinKeyConflict, + }) + + log := slog.New(h) + log.Info("case insenstive, keep builtin conflict", "arg1", "val1", "ARG1", "val2", slog.MessageKey, "builtin-conflict") + + jBytes, err := tester.MarshalJSON() + if err != nil { + t.Errorf("Unable to marshal json: %v", err) + } + jStr := strings.TrimSpace(string(jBytes)) + + expected := `{"time":"2023-09-29T13:00:59Z","level":"INFO","msg":"case insenstive, keep builtin conflict","arg1":["val1","val2"],"msg":"builtin-conflict"}` + if jStr != expected { + t.Errorf("Expected:\n%s\nGot:\n%s", expected, jStr) + } + + // Uncomment to see the results + // t.Error(jStr) + // t.Error(tester.String()) +} diff --git a/helpers_test.go b/helpers_test.go index aaff114..e5c5f3b 100644 --- a/helpers_test.go +++ b/helpers_test.go @@ -77,7 +77,7 @@ func logComplex(t *testing.T, handler slog.Handler) { log = log.With("with1", "arg0", "arg1", "with1arg1", "arg2", "with1arg2", "arg3", "with1arg3", slog.SourceKey, "with1source", slog.TimeKey, "with1time", slog.Group("emptyGroup"), "typed", "overwritten", slog.Int("typed", 3)) log = log.With("with2", "arg0", "arg1", "with2arg1", "arg3", "with2arg3", "arg4", "with2arg4", "msg#01", "prexisting01", "msg#01a", "seekbug01a", "msg#02", "seekbug02", slog.MessageKey, "with2msg", slog.MessageKey, "with2msg2", slog.LevelKey, "with2level", "group1", "with2group1", slog.Bool("typed", true)) - log = log.WithGroup("group1") + log = log.WithGroup("group1").With(slog.Attr{}) log = log.With("with3", "arg0", "arg1", "group1with3arg1", "arg2", "group1with3arg2", "arg3", "group1with3arg3", slog.Group("overwrittenGroup", "arg", "arg"), slog.Group("separateGroup2", "group2", "group2arg0", "arg1", "group2arg1", "arg2", "group2arg2"), slog.SourceKey, "with3source", slog.TimeKey, "with3time") log = log.WithGroup("").WithGroup("") log = log.With("with4", "arg0", "arg1", "group1with4arg1", "arg3", "group1with4arg3", "arg4", "group1with4arg4", slog.Group("", "arg5", "with4inlinedGroupArg5"), slog.String("overwrittenGroup", "with4overwrittenGroup"), slog.MessageKey, "with4msg", slog.LevelKey, "with4overwritten") diff --git a/overwrite_handler_test.go b/overwrite_handler_test.go index be240fd..e3494af 100644 --- a/overwrite_handler_test.go +++ b/overwrite_handler_test.go @@ -1,6 +1,7 @@ package dedup import ( + "log/slog" "strings" "testing" ) @@ -74,3 +75,41 @@ func TestOverwriteHandler(t *testing.T) { checkRecordForDuplicates(t, tester.Record) } + +/* + { + "time": "2023-09-29T13:00:59Z", + "level": "INFO", + "msg": "case insenstive, drop builtin conflict", + "ARG1": "val2" + } +*/ +func TestOverwriteHandler_CaseInsensitiveDropBuiltinConflicts(t *testing.T) { + t.Parallel() + + tester := &testHandler{} + h := NewOverwriteHandler(tester, &OverwriteHandlerOptions{ + KeyCompare: CaseInsensitiveCmp, + ResolveBuiltinKeyConflict: DropIfBuiltinKeyConflict, + }) + + log := slog.New(h) + log.Info("case insenstive, drop builtin conflict", "arg1", "val1", "ARG1", "val2", slog.MessageKey, "builtin-conflict") + + jBytes, err := tester.MarshalJSON() + if err != nil { + t.Errorf("Unable to marshal json: %v", err) + } + jStr := strings.TrimSpace(string(jBytes)) + + expected := `{"time":"2023-09-29T13:00:59Z","level":"INFO","msg":"case insenstive, drop builtin conflict","ARG1":"val2"}` + if jStr != expected { + t.Errorf("Expected:\n%s\nGot:\n%s", expected, jStr) + } + + // Uncomment to see the results + // t.Error(jStr) + // t.Error(tester.String()) + + checkRecordForDuplicates(t, tester.Record) +}