diff --git a/.cspell.json b/.cspell.json index f495a2e5e1b..3eccf2d3f1e 100644 --- a/.cspell.json +++ b/.cspell.json @@ -74,6 +74,7 @@ "GOPATH", "Gource", "handlebargh", + "hashvalue", "HEALTHCHECK", "healthz", "Hetzner", diff --git a/agent/logger.go b/agent/logger.go index 7e3469fd65a..21f04753cd8 100644 --- a/agent/logger.go +++ b/agent/logger.go @@ -15,9 +15,11 @@ package agent import ( + "crypto/sha256" "io" "sync" + hashvalue_replacer "github.com/6543/go-hashvalue-replacer" "github.com/rs/zerolog" "go.woodpecker-ci.org/woodpecker/v2/pipeline" @@ -35,22 +37,31 @@ func (r *Runner) createLogger(_logger zerolog.Logger, uploads *sync.WaitGroup, w Logger() uploads.Add(1) - - var secrets []string - for _, secret := range workflow.Config.Secrets { - secrets = append(secrets, secret.Value) - } + defer uploads.Done() logger.Debug().Msg("log stream opened") - logStream := log.NewLineWriter(r.client, step.UUID, secrets...) - if err := log.CopyLineByLine(logStream, rc, pipeline.MaxLogLineLength); err != nil { + // mask secrets from reader + maskedReader, err := hashvalue_replacer.NewReader(rc, workflow.Config.SecretMask.Salt, workflow.Config.SecretMask.Hashes, workflow.Config.SecretMask.Lengths, hashvalue_replacer.Options{ + Mask: "********", + Hash: func(salt, data []byte) []byte { + h := sha256.New() + h.Write(salt) + h.Write([]byte(data)) + return h.Sum(nil) + }, + }) + if err != nil { + logger.Error().Err(err).Msg("could not create masked reader") + return nil + } + + logStream := log.NewLineWriter(r.client, step.UUID) + if err := log.CopyLineByLine(logStream, maskedReader, pipeline.MaxLogLineLength); err != nil { logger.Error().Err(err).Msg("copy limited logStream part") } logger.Debug().Msg("log stream copied, close ...") - uploads.Done() - return nil } } diff --git a/go.mod b/go.mod index ba4bf407e31..6e27e18a920 100644 --- a/go.mod +++ b/go.mod @@ -1,8 +1,6 @@ module go.woodpecker-ci.org/woodpecker/v2 -go 1.22.0 - -toolchain go1.23.2 +go 1.23.2 require ( al.essio.dev/pkg/shellescape v1.5.1 @@ -10,6 +8,7 @@ require ( codeberg.org/6543/go-yaml2json v1.0.0 codeberg.org/6543/xyaml v1.1.0 codeberg.org/mvdkleijn/forgejo-sdk/forgejo v1.2.0 + github.com/6543/go-hashvalue-replacer v0.0.0-20241116014433-da29dad32109 github.com/6543/logfile-open v1.2.1 github.com/adrg/xdg v0.5.2 github.com/bmatcuk/doublestar/v4 v4.7.1 diff --git a/go.sum b/go.sum index e9c3764ccd7..0cad55649a4 100644 --- a/go.sum +++ b/go.sum @@ -15,6 +15,8 @@ gitea.com/xorm/sqlfiddle v0.0.0-20180821085327-62ce714f951a/go.mod h1:EXuID2Zs0p gitee.com/travelliu/dm v1.8.11192/go.mod h1:DHTzyhCrM843x9VdKVbZ+GKXGRbKM2sJ4LxihRxShkE= github.com/42wim/httpsig v1.2.2 h1:ofAYoHUNs/MJOLqQ8hIxeyz2QxOz8qdSVvp3PX/oPgA= github.com/42wim/httpsig v1.2.2/go.mod h1:P/UYo7ytNBFwc+dg35IubuAUIs8zj5zzFIgUCEl55WY= +github.com/6543/go-hashvalue-replacer v0.0.0-20241116014433-da29dad32109 h1:5DPvI79163nINaU7cj3/6XGDYtnh49hOIOJbPB97/Lk= +github.com/6543/go-hashvalue-replacer v0.0.0-20241116014433-da29dad32109/go.mod h1:+fCz/+h1AIDEtSgeZG6wVPXOwag1WgfosmIz7KaeHDM= github.com/6543/logfile-open v1.2.1 h1:az+TtNHclTAKaHfFCTSbuduMllANox1gM9qLQr7LV5I= github.com/6543/logfile-open v1.2.1/go.mod h1:ZoEy7pW2mexmQxiZIqPCeh8vUxVuiHYXmSZNbvEb51g= github.com/Azure/azure-sdk-for-go/sdk/azcore v0.19.0/go.mod h1:h6H6c8enJmmocHUbLiiGY6sx7f9i+X3m1CHdd5c6Rdw= diff --git a/pipeline/backend/types/config.go b/pipeline/backend/types/config.go index bf07e9e7a3d..dc39e5beb6f 100644 --- a/pipeline/backend/types/config.go +++ b/pipeline/backend/types/config.go @@ -16,10 +16,10 @@ package types // Config defines the runtime configuration of a workflow. type Config struct { - Stages []*Stage `json:"pipeline"` // workflow stages - Networks []*Network `json:"networks"` // network definitions - Volumes []*Volume `json:"volumes"` // volume definitions - Secrets []*Secret `json:"secrets"` // secret definitions + Stages []*Stage `json:"pipeline"` // workflow stages + Networks []*Network `json:"networks"` // network definitions + Volumes []*Volume `json:"volumes"` // volume definitions + SecretMask SecretMask `json:"secret_mask"` } // CliCommand is the context key to pass cli context to backends if needed. diff --git a/pipeline/backend/types/secret.go b/pipeline/backend/types/secret.go index 8287fe78962..388b304ace4 100644 --- a/pipeline/backend/types/secret.go +++ b/pipeline/backend/types/secret.go @@ -15,7 +15,8 @@ package types // Secret defines a runtime secret. -type Secret struct { - Name string `json:"name,omitempty"` - Value string `json:"value,omitempty"` +type SecretMask struct { + Salt []byte `json:"salt,omitempty"` + Hashes [][]byte `json:"value,omitempty"` + Lengths []int `json:"lengths,omitempty"` } diff --git a/pipeline/frontend/yaml/compiler/compiler.go b/pipeline/frontend/yaml/compiler/compiler.go index be2b59cbc27..3c7ead7eaf0 100644 --- a/pipeline/frontend/yaml/compiler/compiler.go +++ b/pipeline/frontend/yaml/compiler/compiler.go @@ -15,9 +15,13 @@ package compiler import ( + "crypto/rand" + "crypto/sha256" "fmt" "path" + hashvalue_replacer "github.com/6543/go-hashvalue-replacer" + backend_types "go.woodpecker-ci.org/woodpecker/v2/pipeline/backend/types" "go.woodpecker-ci.org/woodpecker/v2/pipeline/frontend/metadata" yaml_types "go.woodpecker-ci.org/woodpecker/v2/pipeline/frontend/yaml/types" @@ -139,13 +143,23 @@ func (c *Compiler) Compile(conf *yaml_types.Workflow) (*backend_types.Config, er Name: fmt.Sprintf("%s_default", c.prefix), }) - // create secrets for mask + // create mask for secrets + secretValues := make([]string, len(c.secrets)) for _, sec := range c.secrets { - config.Secrets = append(config.Secrets, &backend_types.Secret{ - Name: sec.Name, - Value: sec.Value, - }) + secretValues = append(secretValues, sec.Value) + } + salt := make([]byte, 64) + _, err := rand.Read(salt) + if err != nil { + return nil, fmt.Errorf("could not generate salt for secret masker: %w", err) } + config.SecretMask.Salt = salt + config.SecretMask.Hashes, config.SecretMask.Lengths = hashvalue_replacer.ValuesToArgs(func(salt, data []byte) []byte { + h := sha256.New() + h.Write(salt) + h.Write([]byte(data)) + return h.Sum(nil) + }, salt, secretValues) // overrides the default workspace paths when specified // in the YAML file. diff --git a/pipeline/log/line_writer.go b/pipeline/log/line_writer.go index ff8706f366d..cd2c539f54e 100644 --- a/pipeline/log/line_writer.go +++ b/pipeline/log/line_writer.go @@ -24,7 +24,6 @@ import ( "github.com/rs/zerolog/log" "go.woodpecker-ci.org/woodpecker/v2/pipeline/rpc" - "go.woodpecker-ci.org/woodpecker/v2/pipeline/shared" ) // LineWriter sends logs to the client. @@ -35,25 +34,20 @@ type LineWriter struct { stepUUID string num int startTime time.Time - replacer *strings.Replacer } // NewLineWriter returns a new line reader. -func NewLineWriter(peer rpc.Peer, stepUUID string, secret ...string) io.Writer { +func NewLineWriter(peer rpc.Peer, stepUUID string) io.Writer { lw := &LineWriter{ peer: peer, stepUUID: stepUUID, startTime: time.Now().UTC(), - replacer: shared.NewSecretsReplacer(secret), } return lw } func (w *LineWriter) Write(p []byte) (n int, err error) { data := string(p) - if w.replacer != nil { - data = w.replacer.Replace(data) - } log.Trace().Str("step-uuid", w.stepUUID).Msgf("grpc write line: %s", data) line := &rpc.LogEntry{ diff --git a/pipeline/log/line_writer_test.go b/pipeline/log/line_writer_test.go index 8a2d6b12037..b91ae1ebcfa 100644 --- a/pipeline/log/line_writer_test.go +++ b/pipeline/log/line_writer_test.go @@ -29,8 +29,7 @@ func TestLineWriter(t *testing.T) { peer := mocks.NewPeer(t) peer.On("EnqueueLog", mock.Anything) - secrets := []string{"world"} - lw := log.NewLineWriter(peer, "e9ea76a5-44a1-4059-9c4a-6956c478b26d", secrets...) + lw := log.NewLineWriter(peer, "e9ea76a5-44a1-4059-9c4a-6956c478b26d") _, err := lw.Write([]byte("hello world\n")) assert.NoError(t, err) @@ -42,7 +41,7 @@ func TestLineWriter(t *testing.T) { Time: 0, Type: rpc.LogEntryStdout, Line: 0, - Data: []byte("hello ********"), + Data: []byte("hello world"), }) peer.AssertCalled(t, "EnqueueLog", &rpc.LogEntry{ diff --git a/pipeline/rpc/proto/version.go b/pipeline/rpc/proto/version.go index 9305354c4d7..c479ef92dd5 100644 --- a/pipeline/rpc/proto/version.go +++ b/pipeline/rpc/proto/version.go @@ -16,4 +16,4 @@ package proto // Version is the version of the woodpecker.proto file, // IMPORTANT: increased by 1 each time it get changed. -const Version int32 = 11 +const Version int32 = 12 diff --git a/pipeline/shared/replace_secrets_test.go b/pipeline/shared/replace_secrets_test.go index bcf40999da1..c389d3c9058 100644 --- a/pipeline/shared/replace_secrets_test.go +++ b/pipeline/shared/replace_secrets_test.go @@ -15,6 +15,7 @@ package shared import ( + "bytes" "testing" "github.com/stretchr/testify/assert" @@ -27,7 +28,7 @@ func TestNewSecretsReplacer(t *testing.T) { secrets []string expect string }{{ - name: "dont replace secrets with less than 3 chars", + name: "dont replace secrets with less than 4 chars", log: "start log\ndone", secrets: []string{"", "d", "art"}, expect: "start log\ndone", @@ -61,3 +62,78 @@ func TestNewSecretsReplacer(t *testing.T) { }) } } + +func BenchmarkReader(b *testing.B) { + testCases := []struct { + name string + log string + secrets []string + }{ + { + name: "single line", + log: "this is a log with secret password and more text", + secrets: []string{"password"}, + }, + { + name: "multi line", + log: "log start\nthis is a multi\nline secret\nlog end", + secrets: []string{"multi\nline secret"}, + }, + { + name: "many secrets", + log: "log with many secrets: secret1 secret2 secret3 secret4 secret5", + secrets: []string{"secret1", "secret2", "secret3", "secret4", "secret5"}, + }, + { + name: "large log", + log: "start " + string(bytes.Repeat([]byte("test secret test "), 1000)) + " end", + secrets: []string{"secret"}, + }, + { + name: "large log no match", + log: "start " + string(bytes.Repeat([]byte("test secret test "), 1000)) + " end", + secrets: []string{"XXXXXXX"}, + }, + } + + for _, tc := range testCases { + b.Run(tc.name, func(b *testing.B) { + rep := NewSecretsReplacer(tc.secrets) + b.ResetTimer() + b.SetBytes(int64(len(tc.log))) + for i := 0; i < b.N; i++ { + _ = rep.Replace(tc.log) + } + }) + } +} + +// go test -benchmem -run='^$' -tags test -bench '^BenchmarkReader$' -benchtime=100000x go.woodpecker-ci.org/woodpecker/v2/pipeline/shared +// +// cpu: AMD Ryzen 9 7940HS (16-Core) +// BenchmarkReader/single_line-16 100000 55.13 ns/op 870.70 MB/s 48 B/op 1 allocs/op +// BenchmarkReader/multi_line-16 100000 149.0 ns/op 302.06 MB/s 120 B/op 3 allocs/op +// BenchmarkReader/many_secrets-16 100000 273.0 ns/op 227.10 MB/s 296 B/op 4 allocs/op +// BenchmarkReader/large_log-16 100000 19544 ns/op 870.33 MB/s 40520 B/op 9 allocs/op +// BenchmarkReader/large_log_no_match-16 100000 5080 ns/op 3348.63 MB/s 0 B/op 0 allocs/op +// +// cpu: AMD Ryzen 9 3900XT (12-Core) +// BenchmarkReader/single_line-24 100000 90.87 ns/op 528.23 MB/s 48 B/op 1 allocs/op +// BenchmarkReader/multi_line-24 100000 276.2 ns/op 162.94 MB/s 120 B/op 3 allocs/op +// BenchmarkReader/many_secrets-24 100000 433.7 ns/op 142.97 MB/s 296 B/op 4 allocs/op +// BenchmarkReader/large_log-24 100000 26542 ns/op 640.88 MB/s 40520 B/op 9 allocs/op +// BenchmarkReader/large_log_no_match-24 100000 6212 ns/op 2738.45 MB/s 0 B/op 0 allocs/op +// +// cpu: Ampere Altra (2 vCPUs) +// BenchmarkReader/single_line-2 100000 105.1 ns/op 456.89 MB/s 48 B/op 1 allocs/op +// BenchmarkReader/multi_line-2 100000 441.7 ns/op 101.88 MB/s 120 B/op 3 allocs/op +// BenchmarkReader/many_secrets-2 100000 868.7 ns/op 71.37 MB/s 296 B/op 4 allocs/op +// BenchmarkReader/large_log-2 100000 48947 ns/op 347.52 MB/s 40520 B/op 9 allocs/op +// BenchmarkReader/large_log_no_match-2 100000 9156 ns/op 1857.79 MB/s 0 B/op 0 allocs/op +// +// cpu: Intel Xeon Processor (Skylake, IBRS, no TSX) (2 vCPUs) +// BenchmarkReader/single_line-2 100000 167.7 ns/op 286.25 MB/s 48 B/op 1 allocs/op +// BenchmarkReader/multi_line-2 100000 640.7 ns/op 70.24 MB/s 120 B/op 3 allocs/op +// BenchmarkReader/many_secrets-2 100000 1044 ns/op 59.38 MB/s 296 B/op 4 allocs/op +// BenchmarkReader/large_log-2 100000 45271 ns/op 375.73 MB/s 40520 B/op 9 allocs/op +// BenchmarkReader/large_log_no_match-2 100000 11240 ns/op 1513.37 MB/s 0 B/op 0 allocs/op