diff --git a/cmd/prometheus/main.go b/cmd/prometheus/main.go index 766f912b54..961f5bd24d 100644 --- a/cmd/prometheus/main.go +++ b/cmd/prometheus/main.go @@ -35,10 +35,6 @@ import ( "syscall" "time" - pphandler "github.com/prometheus/prometheus/pp-pkg/handler" - rwprocessor "github.com/prometheus/prometheus/pp-pkg/handler/processor" - pptsdb "github.com/prometheus/prometheus/pp-pkg/tsdb" - "github.com/KimMachineGun/automemlimit/memlimit" "github.com/alecthomas/kingpin/v2" "github.com/alecthomas/units" @@ -61,15 +57,21 @@ import ( "k8s.io/klog" klogv2 "k8s.io/klog/v2" - "github.com/prometheus/prometheus/pp-pkg/receiver" // PP_CHANGES.md: rebuild on cpp - "github.com/prometheus/prometheus/pp-pkg/remote" // PP_CHANGES.md: rebuild on cpp - "github.com/prometheus/prometheus/pp-pkg/scrape" // PP_CHANGES.md: rebuild on cpp - pp_pkg_storage "github.com/prometheus/prometheus/pp-pkg/storage" // PP_CHANGES.md: rebuild on cpp - "github.com/prometheus/prometheus/pp/go/relabeler/appender" // PP_CHANGES.md: rebuild on cpp - "github.com/prometheus/prometheus/pp/go/relabeler/head" // PP_CHANGES.md: rebuild on cpp - "github.com/prometheus/prometheus/pp/go/relabeler/head/catalog" // PP_CHANGES.md: rebuild on cpp - "github.com/prometheus/prometheus/pp/go/relabeler/head/ready" // PP_CHANGES.md: rebuild on cpp - "github.com/prometheus/prometheus/pp/go/relabeler/remotewriter" // PP_CHANGES.md: rebuild on cpp + pp_pkg_handler "github.com/prometheus/prometheus/pp-pkg/handler" // PP_CHANGES.md: rebuild on cpp + rwprocessor "github.com/prometheus/prometheus/pp-pkg/handler/processor" // PP_CHANGES.md: rebuild on cpp + pp_pkg_logger "github.com/prometheus/prometheus/pp-pkg/logger" // PP_CHANGES.md: rebuild on cpp + "github.com/prometheus/prometheus/pp-pkg/remote" // PP_CHANGES.md: rebuild on cpp + "github.com/prometheus/prometheus/pp-pkg/scrape" // PP_CHANGES.md: rebuild on cpp + pp_pkg_storage "github.com/prometheus/prometheus/pp-pkg/storage" // PP_CHANGES.md: rebuild on cpp + pp_pkg_tsdb "github.com/prometheus/prometheus/pp-pkg/tsdb" // PP_CHANGES.md: rebuild on cpp + + pp_storage "github.com/prometheus/prometheus/pp/go/storage" // PP_CHANGES.md: rebuild on cpp + "github.com/prometheus/prometheus/pp/go/storage/catalog" // PP_CHANGES.md: rebuild on cpp + "github.com/prometheus/prometheus/pp/go/storage/head/head" // PP_CHANGES.md: rebuild on cpp + "github.com/prometheus/prometheus/pp/go/storage/head/services" // PP_CHANGES.md: rebuild on cpp + "github.com/prometheus/prometheus/pp/go/storage/querier" // PP_CHANGES.md: rebuild on cpp + "github.com/prometheus/prometheus/pp/go/storage/ready" // PP_CHANGES.md: rebuild on cpp + "github.com/prometheus/prometheus/pp/go/storage/remotewriter" // PP_CHANGES.md: rebuild on cpp "github.com/prometheus/prometheus/config" "github.com/prometheus/prometheus/discovery" @@ -726,9 +728,10 @@ func main() { os.Exit(1) } - reloadBlocksTriggerNotifier := receiver.NewReloadBlocksTriggerNotifier() + pp_pkg_logger.InitLogHandler(log.With(logger, "component", "pp")) + + reloadBlocksTriggerNotifier := pp_storage.NewTriggerNotifier() cfg.tsdb.ReloadBlocksExternalTrigger = reloadBlocksTriggerNotifier - ctxReceiver, cancelReceiver := context.WithCancel(context.Background()) dataDir, err := filepath.Abs(localStoragePath) if err != nil { @@ -748,44 +751,59 @@ func main() { } clock := clockwork.NewRealClock() - headCatalog, err := catalog.New(clock, fileLog, catalog.DefaultIDGenerator{}, int(catalogMaxLogFileSize), prometheus.DefaultRegisterer) + headCatalog, err := catalog.New( + clock, + fileLog, + catalog.DefaultIDGenerator{}, + int(catalogMaxLogFileSize), + prometheus.DefaultRegisterer, + ) if err != nil { level.Error(logger).Log("msg", "failed to create head catalog", "err", err) os.Exit(1) } - receiverReadyNotifier := ready.NewNotifiableNotifier() - // create receiver - receiver, err := receiver.NewReceiver( - ctxReceiver, - log.With(logger, "component", "receiver"), - prometheus.DefaultRegisterer, - receiverConfig, - localStoragePath, - cfgFile.RemoteWriteConfigs, - localStoragePath, - receiver.RotationInfo{ - BlockDuration: time.Duration(cfg.tsdb.MinBlockDuration), - Seed: cfgFile.GlobalConfig.ExternalLabels.Hash(), + removedHeadTriggerNotifier := pp_storage.NewTriggerNotifier() + hManagerReadyNotifier := ready.NewNotifiableNotifier() + hManager, err := pp_storage.NewManager( + &pp_storage.Options{ + Seed: cfgFile.GlobalConfig.ExternalLabels.Hash(), + BlockDuration: time.Duration(cfg.tsdb.MinBlockDuration), + CommitInterval: time.Duration(cfg.WalCommitInterval), + MaxRetentionPeriod: time.Duration(cfg.tsdb.RetentionDuration), + HeadRetentionPeriod: time.Duration(cfg.HeadRetentionTimeout), + KeeperCapacity: 2, + DataDir: localStoragePath, + MaxSegmentSize: cfg.WalMaxSamplesPerSegment, + NumberOfShards: receiverConfig.NumberOfShards, }, + clock, headCatalog, reloadBlocksTriggerNotifier, - receiverReadyNotifier, - time.Duration(cfg.WalCommitInterval), - time.Duration(cfg.tsdb.RetentionDuration), - time.Duration(cfg.HeadRetentionTimeout), - // x3 ScrapeInterval timeout for write block - time.Duration(cfgFile.GlobalConfig.ScrapeInterval*3), - cfg.WalMaxSamplesPerSegment, - appender.UnloadDataStorage, + removedHeadTriggerNotifier, + hManagerReadyNotifier, + prometheus.DefaultRegisterer, ) if err != nil { - level.Error(logger).Log("msg", "failed to create a receiver", "err", err) + level.Error(logger).Log("msg", "failed to create a head manager", "err", err) os.Exit(1) } remoteWriterReadyNotifier := ready.NewNotifiableNotifier() - remoteWriter := remotewriter.New(dataDir, headCatalog, clock, remoteWriterReadyNotifier, prometheus.DefaultRegisterer) + remoteWriter := remotewriter.New( + dataDir, + headCatalog, + clock, + remoteWriterReadyNotifier, + prometheus.DefaultRegisterer, + ) + + adapter := pp_pkg_storage.NewAdapter( + clock, + hManager.Proxy(), + hManager.MergeOutOfOrderChunks, + prometheus.DefaultRegisterer, + ) // PP_CHANGES.md: rebuild on cpp end @@ -796,14 +814,11 @@ func main() { // PP_CHANGES.md: rebuild on cpp start remoteRead = pp_pkg_storage.NewRemoteRead( log.With(logger, "component", "remote"), - prometheus.DefaultRegisterer, localStorage.StartTime, - localStoragePath, - time.Duration(cfg.RemoteFlushDeadline), ) fanoutStorage = storage.NewFanout( logger, - pp_pkg_storage.NewQueryableStorage(receiver), + adapter, localStorage, remoteRead, ) @@ -881,7 +896,7 @@ func main() { scrapeManager, err := scrape.NewManager( &cfg.scrape, log.With(logger, "component", "scrape manager"), - receiver, + adapter, prometheus.DefaultRegisterer, ) if err != nil { @@ -940,8 +955,8 @@ func main() { queryEngine = promql.NewEngine(opts) ruleManager = rules.NewManager(&rules.ManagerOptions{ - Appendable: receiver, // PP_CHANGES.md: rebuild on cpp - Queryable: receiver, // PP_CHANGES.md: rebuild on cpp + Appendable: adapter, // PP_CHANGES.md: rebuild on cpp + Queryable: adapter, // PP_CHANGES.md: rebuild on cpp QueryFunc: rules.EngineQueryFunc(queryEngine, fanoutStorage), NotifyFunc: rules.SendAlerts(notifierManager, cfg.web.ExternalURL.String()), Context: ctxRule, @@ -998,7 +1013,7 @@ func main() { } // Depends on cfg.web.ScrapeManager so needs to be after cfg.web.ScrapeManager = scrapeManager. - webHandler := web.New(log.With(logger, "component", "web"), &cfg.web, receiver) // PP_CHANGES.md: rebuild on cpp + webHandler := web.New(log.With(logger, "component", "web"), &cfg.web, adapter) // PP_CHANGES.md: rebuild on cpp // Monitor outgoing connections on default transport with conntrack. http.DefaultTransport.(*http.Transport).DialContext = conntrack.NewDialContextFunc( @@ -1010,8 +1025,8 @@ func main() { reloaders := []reloader{ { // PP_CHANGES.md: rebuild on cpp start - name: "receiver", - reloader: receiver.ApplyConfig, + name: "head_manager", + reloader: hManager.ApplyConfig, }, { // PP_CHANGES.md: rebuild on cpp end name: "db_storage", reloader: localStorage.ApplyConfig, @@ -1147,8 +1162,19 @@ func main() { os.Exit(1) } - multiNotifiable := ready.New().With(receiverReadyNotifier).With(remoteWriterReadyNotifier).Build() - opGC := catalog.NewGC(dataDir, headCatalog, multiNotifiable) + multiNotifiable := ready.NewMultiNotifiableBuilder().Add( + hManagerReadyNotifier, + ).Add( + remoteWriterReadyNotifier, + ).Build() + opGC := catalog.NewGC( + dataDir, + headCatalog, + clock, + multiNotifiable, + removedHeadTriggerNotifier, + time.Duration(cfg.tsdb.RetentionDuration), + ) var g run.Group { @@ -1346,7 +1372,7 @@ func main() { return fmt.Errorf("opening storage failed: %w", err) } - tsdb.DBSetBlocksToDelete(db, pptsdb.PPBlocksToDelete(db, dataDir, headCatalog)) + tsdb.DBSetBlocksToDelete(db, pp_pkg_tsdb.PPBlocksToDelete(db, dataDir, headCatalog)) switch fsType := prom_runtime.Statfs(localStoragePath); fsType { case "NFS_SUPER_MAGIC": level.Warn(logger).Log("fs_type", fsType, "msg", "This filesystem is not supported and may lead to data corruption and data loss. Please carefully read https://prometheus.io/docs/prometheus/latest/storage/ to learn more about supported filesystems.") @@ -1396,7 +1422,7 @@ func main() { db, err := agent.Open( logger, prometheus.DefaultRegisterer, - receiver, // PP_CHANGES.md: rebuild on cpp + adapter, // PP_CHANGES.md: rebuild on cpp localStoragePath, &opts, ) @@ -1451,7 +1477,7 @@ func main() { ) } { // PP_CHANGES.md: rebuild on cpp start - // run receiver. + // run head manager. cancel := make(chan struct{}) g.Add( func() error { @@ -1462,37 +1488,31 @@ func main() { return nil } - return receiver.Run(ctxReceiver) + return hManager.Run() }, func(err error) { - receiverCancelCtx, receiverCancelCtxCancel := context.WithCancel(ctxReceiver) - defer receiverCancelCtxCancel() - - level.Info(logger).Log("msg", "Stopping Receiver...") + level.Info(logger).Log("msg", "Stopping head manager...", "reason", err) close(cancel) - - if err := receiver.Shutdown(receiverCancelCtx); err != nil { - level.Error(logger).Log("msg", "Receiver shutdown failed", "err", err) + if err := hManager.Shutdown(context.Background()); err != nil { + level.Error(logger).Log("msg", "Head manager shutdown failed", "err", err) } - - level.Info(logger).Log("msg", "Receiver stopped...") - cancelReceiver() + level.Info(logger).Log("msg", "Head manager stopped.") }, ) } // PP_CHANGES.md: rebuild on cpp end { // PP_CHANGES.md: rebuild on cpp start g.Add( - func() error { return <-head.UnrecoverableErrorChan }, + func() error { + return <-querier.UnrecoverableErrorChan + }, func(err error) { - select { - case head.UnrecoverableErrorChan <- nil: - // stop execute func if need - default: - } + // stop execute func if need + querier.SendUnrecoverableError(nil) - if errors.Is(err, head.UnrecoverableError{}) { + if errors.Is(err, querier.UnrecoverableError{}) { level.Error(logger).Log("msg", "Received unrecoverable error", "err", err) } + level.Info(logger).Log("msg", "Unrecoverable Error Handler stopped.") }, ) } // PP_CHANGES.md: rebuild on cpp end @@ -1544,6 +1564,10 @@ func main() { } // PP_CHANGES.md: rebuild on cpp start the engine is really no longer in use before calling this to avoid + if err := fileLog.Close(); err != nil { + level.Error(logger).Log("msg", "failed to close file log", "err", err) + } + if err := queryEngine.Close(); err != nil { level.Warn(logger).Log("msg", "Closing query engine failed", "err", err) } @@ -1989,6 +2013,7 @@ func (opts tsdbOptions) ToTSDBOptions() tsdb.Options { OutOfOrderTimeWindow: opts.OutOfOrderTimeWindow, EnableDelayedCompaction: opts.EnableDelayedCompaction, EnableOverlappingCompaction: opts.EnableOverlappingCompaction, + ReloadBlocksExternalTrigger: opts.ReloadBlocksExternalTrigger, } } @@ -2076,7 +2101,7 @@ func readPromPPFeatures(logger log.Logger) { fname, fvalue, _ := strings.Cut(feature, "=") switch strings.TrimSpace(fname) { case "head_copy_series_on_rotate": - appender.CopySeriesOnRotate = true + services.CopySeriesOnRotate = true level.Info(logger).Log( "msg", "[FEATURE] Copying active series from current head to new head during rotation is enabled.", @@ -2096,7 +2121,7 @@ func readPromPPFeatures(logger log.Logger) { } } - head.ExtraReadConcurrency = v + head.ExtraWorkers = v level.Info(logger).Log( "msg", "[FEATURE] Concurrency reading is enabled.", @@ -2109,7 +2134,7 @@ func readPromPPFeatures(logger log.Logger) { if fvalue == "" { level.Error(logger).Log( "msg", "[FEATURE] The default number of shards is empty, no changes.", - "default_number_of_shards", receiver.DefaultNumberOfShards, + "default_number_of_shards", pp_storage.DefaultNumberOfShards, ) continue @@ -2120,36 +2145,36 @@ func readPromPPFeatures(logger log.Logger) { case err != nil: level.Error(logger).Log( "msg", "[FEATURE] Error parsing head_numbehead_default_number_of_shardsr_of_shards value", - "default_number_of_shards", receiver.DefaultNumberOfShards, + "default_number_of_shards", pp_storage.DefaultNumberOfShards, "err", err, ) case v > math.MaxUint16: level.Error(logger).Log( "msg", "[FEATURE] The default number of shards is overflow(max 65535), no changes.", - "default_number_of_shards", receiver.DefaultNumberOfShards, + "default_number_of_shards", pp_storage.DefaultNumberOfShards, ) case v < 1: level.Error(logger).Log( "msg", "[FEATURE] The default number of shards is incorrect(min 1), no changes.", - "default_number_of_shards", receiver.DefaultNumberOfShards, + "default_number_of_shards", pp_storage.DefaultNumberOfShards, ) default: - receiver.DefaultNumberOfShards = uint16(v) + pp_storage.DefaultNumberOfShards = uint16(v) level.Info(logger).Log( "msg", "[FEATURE] Changed default number of shards.", - "default_number_of_shards", receiver.DefaultNumberOfShards, + "default_number_of_shards", pp_storage.DefaultNumberOfShards, ) } case "disable_commits_on_remote_write": rwprocessor.AlwaysCommit = false - pphandler.OTLPAlwaysCommit = false + pp_pkg_handler.OTLPAlwaysCommit = false case "unload_data_storage": - appender.UnloadDataStorage = true + pp_storage.UnloadDataStorage = true _ = level.Info(logger).Log("msg", "[FEATURE] Data storage unloading is enabled.") } } diff --git a/cmd/prompptool/main.go b/cmd/prompptool/main.go index 9d380ae4f9..532a7c4441 100644 --- a/cmd/prompptool/main.go +++ b/cmd/prompptool/main.go @@ -14,7 +14,7 @@ import ( "github.com/go-kit/log" "github.com/go-kit/log/level" - rlogger "github.com/prometheus/prometheus/pp/go/relabeler/logger" + pplogger "github.com/prometheus/prometheus/pp/go/logger" ) func main() { @@ -81,16 +81,16 @@ func initLogger(verbose bool) log.Logger { func initLogHandler(logger log.Logger) { logger = log.With(logger, "op_caller", log.Caller(4)) - rlogger.Debugf = func(template string, args ...interface{}) { + pplogger.Debugf = func(template string, args ...interface{}) { level.Debug(logger).Log("msg", fmt.Sprintf(template, args...)) } - rlogger.Infof = func(template string, args ...interface{}) { + pplogger.Infof = func(template string, args ...interface{}) { level.Info(logger).Log("msg", fmt.Sprintf(template, args...)) } - rlogger.Warnf = func(template string, args ...interface{}) { + pplogger.Warnf = func(template string, args ...interface{}) { level.Warn(logger).Log("msg", fmt.Sprintf(template, args...)) } - rlogger.Errorf = func(template string, args ...interface{}) { + pplogger.Errorf = func(template string, args ...interface{}) { level.Error(logger).Log("msg", fmt.Sprintf(template, args...)) } } diff --git a/cmd/prompptool/walpp.go b/cmd/prompptool/walpp.go index e857aa2d4b..87708baefd 100644 --- a/cmd/prompptool/walpp.go +++ b/cmd/prompptool/walpp.go @@ -14,11 +14,10 @@ import ( "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/common/model" - "github.com/prometheus/prometheus/pp/go/relabeler" - "github.com/prometheus/prometheus/pp/go/relabeler/block" - "github.com/prometheus/prometheus/pp/go/relabeler/config" - "github.com/prometheus/prometheus/pp/go/relabeler/head" - "github.com/prometheus/prometheus/pp/go/relabeler/head/catalog" + "github.com/prometheus/prometheus/pp/go/storage" + "github.com/prometheus/prometheus/pp/go/storage/block" + "github.com/prometheus/prometheus/pp/go/storage/catalog" + "github.com/prometheus/prometheus/pp/go/storage/head/shard" ) type cmdWALPPToBlock struct { @@ -69,7 +68,7 @@ func (cmd *cmdWALPPToBlock) Do( if err != nil { return fmt.Errorf("failed init head catalog: %w", err) } - headRecords, err := headCatalog.List( + headRecords := headCatalog.List( func(record *catalog.Record) bool { return record.DeletedAt() == 0 && (record.Status() == catalog.StatusNew || record.Status() == catalog.StatusActive || record.Status() == catalog.StatusRotated) @@ -78,53 +77,40 @@ func (cmd *cmdWALPPToBlock) Do( return lhs.CreatedAt() < rhs.CreatedAt() }, ) - if err != nil { - return fmt.Errorf("failed listed head catalog: %w", err) - } + level.Debug(logger).Log("msg", "catalog records", "len", len(headRecords)) - var inputRelabelerConfig []*config.InputRelabelerConfig - bw := block.NewWriter(workingDir, block.DefaultChunkSegmentSize, time.Duration(cmd.blockDuration), registerer) + bw := block.NewWriter[*shard.Shard]( + workingDir, + block.DefaultChunkSegmentSize, + time.Duration(cmd.blockDuration), + registerer, + ) + + loader := storage.NewLoader(workingDir, 0, registerer, time.Duration(0)) + for _, headRecord := range headRecords { if err := ctx.Err(); err != nil { return err } level.Debug(logger).Log("msg", "load head", "id", headRecord.ID(), "dir", headRecord.Dir()) - h, _, _, err := head.Load( - headRecord.ID(), - 0, - filepath.Join(workingDir, headRecord.Dir()), - inputRelabelerConfig, - headRecord.NumberOfShards(), - 0, - head.NoOpLastAppendedSegmentIDSetter{}, - registerer, - time.Duration(0), - ) - if err != nil { - level.Error(logger).Log( - "msg", "failed to load head", - "id", headRecord.ID(), - "dir", headRecord.Dir(), - "err", err, - ) - return err + h, corrupted := loader.Load(headRecord, 0) + if corrupted { + level.Warn(logger).Log("msg", "corrupted head", "id", headRecord.ID(), "dir", headRecord.Dir()) } - h.Stop() level.Debug(logger).Log("msg", "write block", "id", headRecord.ID(), "dir", headRecord.Dir()) - tBlockWrite := h.CreateTask( - relabeler.BlockWrite, - func(shard relabeler.Shard) error { - _, err := bw.Write(shard) - return err - }, - relabeler.ForLSSTask, - ) - h.Enqueue(tBlockWrite) - if err = tBlockWrite.Wait(); err != nil { - return fmt.Errorf("failed to write tsdb block [id: %s, dir: %s]: %w", headRecord.ID(), headRecord.Dir(), err) + for shard := range h.RangeShards() { + if _, err := bw.Write(shard); err != nil { + _ = h.Close() + return fmt.Errorf( + "failed to write tsdb block [id: %s, dir: %s]: %w", + headRecord.ID(), + headRecord.Dir(), + err, + ) + } } if cmd.updateCatalog { @@ -169,7 +155,7 @@ func (cmd *cmdWALPPToBlock) clearing( level.Debug(logger).Log("msg", "catalog clearing: ended") }() - records, err := headCatalog.List( + records := headCatalog.List( func(record *catalog.Record) bool { return record.DeletedAt() == 0 && record.Status() == catalog.StatusPersisted }, @@ -177,9 +163,6 @@ func (cmd *cmdWALPPToBlock) clearing( return lhs.CreatedAt() < rhs.CreatedAt() }, ) - if err != nil { - return fmt.Errorf("failed listed head catalog: %w", err) - } for _, record := range records { if err := ctx.Err(); err != nil { @@ -199,7 +182,7 @@ func (cmd *cmdWALPPToBlock) clearing( ) } - if err = os.RemoveAll(filepath.Join(workingDir, record.Dir())); err != nil { + if err := os.RemoveAll(filepath.Join(workingDir, record.Dir())); err != nil { level.Error(logger).Log( "msg", "failed to delete head directory", "id", record.ID(), @@ -209,7 +192,7 @@ func (cmd *cmdWALPPToBlock) clearing( continue } - if err = headCatalog.Delete(record.ID()); err != nil { + if err := headCatalog.Delete(record.ID()); err != nil { level.Error(logger).Log( "msg", "failed to delete head record", "id", record.ID(), diff --git a/config/pp_remote_write_config.go b/config/pp_remote_write_config.go index 08ed9d597e..e37ba8b7b8 100644 --- a/config/pp_remote_write_config.go +++ b/config/pp_remote_write_config.go @@ -61,6 +61,16 @@ func (c *Config) GetReceiverConfig() (*pp_pkg_config.RemoteWriteReceiverConfig, return rcCfg, nil } +// PPNumberOfShards returns number of shards for manager. +func (c *Config) PPNumberOfShards() uint16 { + return c.ReceiverConfig.NumberOfShards +} + +// RemoteWriteReceiverConfig returns configs for RemoteWriteReceiver. +func (c *Config) RemoteWriteReceiverConfig() *pp_pkg_config.RemoteWriteReceiverConfig { + return c.ReceiverConfig.Copy() +} + func convertingRelabelConfigs(rCfgs []*relabel.Config) ([]*cppbridge.RelabelConfig, error) { var oprCfgs []*cppbridge.RelabelConfig raw, err := yaml.Marshal(rCfgs) diff --git a/config/pp_scrape_config.go b/config/pp_scrape_config.go new file mode 100644 index 0000000000..c3de0253d4 --- /dev/null +++ b/config/pp_scrape_config.go @@ -0,0 +1,17 @@ +package config + +import "github.com/prometheus/prometheus/pp/go/cppbridge" + +// +// ScrapeConfig +// + +// PPMetricRelabelConfigs returns slice the converted [relabel.Config] to the [cppbridge.RelabelConfig]'s. +func (c *ScrapeConfig) PPMetricRelabelConfigs() ([]*cppbridge.RelabelConfig, error) { + cfgs, err := convertingRelabelConfigs(c.MetricRelabelConfigs) + if err != nil { + return nil, err + } + + return cfgs, nil +} diff --git a/pp-pkg/.golangci.yml b/pp-pkg/.golangci.yml new file mode 100644 index 0000000000..83c92b26b4 --- /dev/null +++ b/pp-pkg/.golangci.yml @@ -0,0 +1,245 @@ +--- +# This file contains all available configuration options +# with their default values. + +# options for analysis running +run: + # default concurrency is a available CPU number + concurrency: 4 + + # exit code when at least one issue was found, default is 1 + issues-exit-code: 1 + + # include test files or not, default is true + tests: true +# output configuration options +output: + # print lines of code with issue, default is true + print-issued-lines: true + + # print linter name in the end of issue text, default is true + print-linter-name: true + +# all available settings of specific linters +linters-settings: + govet: + # settings per analyzer + settings: + printf: # analyzer name, run `go tool vet help` to see all analyzers + funcs: # run `go tool vet help printf` to see available settings for `printf` analyzer + - (github.com/golangci/golangci-lint/pkg/logutils.Log).Infof + - (github.com/golangci/golangci-lint/pkg/logutils.Log).Warnf + - (github.com/golangci/golangci-lint/pkg/logutils.Log).Errorf + - (github.com/golangci/golangci-lint/pkg/logutils.Log).Fatalf + shadow: + # Whether to be strict about shadowing; can be noisy. + # Default: false + strict: true + gofmt: + # simplify code: gofmt with `-s` option, true by default + simplify: true + goimports: + # put imports beginning with prefix after 3rd-party packages; + # it's a comma-separated list of prefixes + local-prefixes: fox.flant.com + dupl: + # tokens count to trigger issue, 150 by default + threshold: 100 + goconst: + # minimal length of string constant, 3 by default + min-len: 3 + # minimal occurrences count to trigger, 3 by default + min-occurrences: 3 + misspell: + # Correct spellings using locale preferences for US or UK. + # Default is to use a neutral variety of English. + # Setting locale to US will correct the British spelling of 'colour' to 'color'. + locale: US + ignore-words: + - someword + # unused: + # # treat code as a program (not a library) and report unused exported identifiers; default is false. + # # XXX: if you enable this setting, unused will report a lot of false-positives in text editors: + # # if it's called for subdir of a project it can't find funcs usages. All text editor integrations + # # with golangci-lint call it on a directory with the changed file. + # check-exported: false + # # Mark all exported fields as used. + # # default: true + # exported-fields-are-used: false + nakedret: + # make an issue if func has more lines of code than this setting and it has naked returns; default is 30 + max-func-lines: 30 + gocritic: + # Enable multiple checks by tags, run `GL_DEBUG=gocritic golangci-lint` run to see all tags and checks. + # Empty list by default. See https://github.com/go-critic/go-critic#usage -> section "Tags". + enabled-tags: + - style + - performance + settings: # settings passed to gocritic + captLocal: # must be valid enabled check name + paramsOnly: true + revive: + ignore-generated-header: true + rules: + - name: context-keys-type + - name: time-naming + - name: var-declaration + - name: unexported-return + - name: errorf + - name: blank-imports + - name: context-as-argument + - name: dot-imports + - name: error-return + - name: error-strings + - name: error-naming + - name: exported + - name: if-return + - name: increment-decrement + - name: range + - name: receiver-naming + - name: indent-error-flow + - name: argument-limit + arguments: + - 10 + - name: cyclomatic + arguments: + - 10 + - name: empty-block + - name: superfluous-else + - name: confusing-naming + - name: get-return + - name: modifies-parameter + - name: confusing-results + - name: deep-exit + - name: unused-parameter + - name: unreachable-code + - name: add-constant + arguments: + - allowInts: "0,1" + allowStrs: '"","/"' + - name: flag-parameter + - name: unnecessary-stmt + - name: struct-tag + - name: modifies-value-receiver + - name: constant-logical-expr + - name: bool-literal-in-expr + - name: redefines-builtin-id + - name: function-result-limit + arguments: + - 3 + - name: imports-blocklist # imports-blacklist + arguments: + - github.com/davecgh/go-spew/spew + - name: range-val-in-closure + - name: range-val-address + - name: waitgroup-by-value + - name: atomic + - name: empty-lines + - name: line-length-limit + arguments: + - 120 + - name: duplicated-imports + - name: import-shadowing + - name: bare-return + - name: unused-receiver + - name: unhandled-error + - name: cognitive-complexity + arguments: + - 20 + - name: string-of-int + - name: early-return + - name: unconditional-recursion + - name: identical-branches + - name: function-length + arguments: + - 30 + - 50 + testifylint: + disable: + - float-compare + - go-require + enable: + - bool-compare + - compares + - empty + - error-is-as + - error-nil + - expected-actual + - len + - require-error + - suite-dont-use-pkg + - suite-extra-assert-call + +linters: + disable-all: true + enable: + - bodyclose # checks whether HTTP response body is closed successfully [fast: false, auto-fix: false] + - dupl # Tool for code clone detection [fast: true, auto-fix: false] + - goconst # Finds repeated strings that could be replaced by a constant [fast: true, auto-fix: false] + - gocritic # The most opinionated Go source code linter [fast: true, auto-fix: false] + - godox # Tool for detection of FIXME, TODO and other comment keywords [fast: true, auto-fix: false] + - gofmt + - goimports # Goimports does everything that gofmt does. Additionally it checks unused imports [fast: true, auto-fix: true] + - gosec # Inspects source code for security problems [fast: true, auto-fix: false] + - gosimple # Linter for Go source code that specializes in simplifying a code [fast: false, auto-fix: false] + - govet # Vet examines Go source code and reports suspicious constructs, such as Printf calls whose arguments do not align with the format string [fast: false, auto-fix: false] + - ineffassign # Detects when assignments to existing variables are not used [fast: true, auto-fix: false] + - misspell # Finds commonly misspelled English words in comments [fast: true, auto-fix: true] + - nakedret # Finds naked returns in functions greater than a specified function length [fast: true, auto-fix: false] + - staticcheck # Staticcheck is a go vet on steroids, applying a ton of static analysis checks [fast: false, auto-fix: false] + - typecheck # Like the front-end of a Go compiler, parses and type-checks Go code [fast: true, auto-fix: false] + - unconvert # Remove unnecessary type conversions [fast: true, auto-fix: false] + - unused # Checks Go code for unused constants, variables, functions and types [fast: false, auto-fix: false] + - revive + - testifylint # Checks usage of github.com/stretchr/testify. + fast: false + +issues: + # List of regexps of issue texts to exclude, empty list by default. + # But independently from this option we use default exclude patterns, + # it can be disabled by `exclude-use-default: false`. To list all + # excluded by default patterns execute `golangci-lint run --help` + exclude: + - abcdef + + # Excluding configuration per-path, per-linter, per-text and per-source + exclude-rules: + # Exclude some linters from running on tests files. + - path: _test\.go + linters: + - gocyclo + - errcheck + - dupl + - gosec + - scopelint + + - path: _test\.go + linters: + - revive + text: "add-constant" + + - path: _test\.go + linters: + - revive + text: "line-length-limit" + + - path: _test\.go + linters: + - revive + text: "function-length" + + # Exclude some staticcheck messages + - linters: + - staticcheck + text: "SA9003:" # Empty body in an if or else branch + + # Exclude some stylecheck messages + - linters: + - stylecheck + text: "ST1000:" # Incorrect or missing package comment + + # Independently from option `exclude` we use default exclude patterns, + # it can be disabled by this option. To list all + # excluded by default patterns execute `golangci-lint run --help`. + # Default value for this option is true. + exclude-use-default: false diff --git a/pp-pkg/configadapter/remotewriter.go b/pp-pkg/configadapter/remotewriter.go new file mode 100644 index 0000000000..6beb74a5db --- /dev/null +++ b/pp-pkg/configadapter/remotewriter.go @@ -0,0 +1,54 @@ +package configadapter + +import ( + "crypto/md5" // #nosec G501 // cryptographic strength is not required + "encoding/hex" + "time" + + "gopkg.in/yaml.v2" + + "github.com/prometheus/prometheus/config" + "github.com/prometheus/prometheus/pp/go/relabeler/remotewriter" +) + +// RemoteWriterApplyConfig returns func-adapter for apply config on [remotewriter.RemoteWriter]. +func RemoteWriterApplyConfig(remoteWriter *remotewriter.RemoteWriter) func(promConfig *config.Config) error { + return func(promConfig *config.Config) error { + destinationConfigs := make([]remotewriter.DestinationConfig, 0, len(promConfig.RemoteWriteConfigs)) + for _, rwc := range promConfig.RemoteWriteConfigs { + if !rwc.IsPrometheusProtocol() { + continue + } + + if rwc.Name == "" { + rwcHash, err := toHash(rwc) + if err != nil { + return err + } + rwc.Name = rwcHash + } + + if rwc.QueueConfig.SampleAgeLimit == 0 { + rwc.QueueConfig.SampleAgeLimit = remotewriter.DefaultSampleAgeLimit + } + + destinationConfigs = append(destinationConfigs, remotewriter.DestinationConfig{ + RemoteWriteConfig: rwc.RemoteWriteConfig, + ExternalLabels: promConfig.GlobalConfig.ExternalLabels, + ReadTimeout: time.Duration(promConfig.GlobalConfig.ScrapeInterval), + }, + ) + } + return remoteWriter.ApplyConfig(destinationConfigs...) + } +} + +// toHash used for hashing configs and diff'ing hashes in ApplyConfig. +func toHash(data interface{}) (string, error) { + bytes, err := yaml.Marshal(data) + if err != nil { + return "", err + } + hash := md5.Sum(bytes) // #nosec G401 // cryptographic strength is not required + return hex.EncodeToString(hash[:]), nil +} diff --git a/pp-pkg/handler/interface.go b/pp-pkg/handler/interface.go index 7f5c6db92f..b6c3b4f100 100644 --- a/pp-pkg/handler/interface.go +++ b/pp-pkg/handler/interface.go @@ -4,28 +4,46 @@ import ( "context" "github.com/prometheus/prometheus/pp-pkg/handler/processor" + "github.com/prometheus/prometheus/pp-pkg/model" "github.com/prometheus/prometheus/pp/go/cppbridge" - "github.com/prometheus/prometheus/pp/go/relabeler" + "github.com/prometheus/prometheus/pp/go/storage/querier" "github.com/prometheus/prometheus/storage" ) -// Receiver interface. -type Receiver interface { - Appender(ctx context.Context) storage.Appender - AppendSnappyProtobuf(ctx context.Context, compressedData relabeler.ProtobufData, relabelerID string, commitToWal bool) error - AppendHashdex(ctx context.Context, hashdex cppbridge.ShardedData, relabelerID string, commitToWal bool) error +// Adapter for implementing the [Queryable] interface and append data. +type Adapter interface { + // AppendHashdex append incoming [cppbridge.HashdexContent] to [Head]. + AppendHashdex( + ctx context.Context, + hashdex cppbridge.ShardedData, + state *cppbridge.StateV2, + commitToWal bool, + ) error + + // AppendTimeSeries append TimeSeries data to [Head]. AppendTimeSeries( ctx context.Context, - data relabeler.TimeSeriesData, - state *cppbridge.State, - relabelerID string, + data model.TimeSeriesBatch, + state *cppbridge.StateV2, commitToWal bool, ) (cppbridge.RelabelerStats, error) - RelabelerIDIsExist(relabelerID string) bool - HeadQueryable() storage.Queryable - HeadStatus(ctx context.Context, limit int) relabeler.HeadStatus - // MergeOutOfOrderChunks merge chunks with out of order data chunks. - MergeOutOfOrderChunks(ctx context.Context) + + // AppendSnappyProtobuf append compressed via snappy Protobuf data to [Head]. + AppendSnappyProtobuf( + ctx context.Context, + compressedData model.ProtobufData, + state *cppbridge.StateV2, + commitToWal bool, + ) error + + // HeadQuerier returns [storage.Querier] from active head. + HeadQuerier(mint, maxt int64) (storage.Querier, error) + + // HeadStatus returns stats of Head. + HeadStatus(ctx context.Context, limit int) (*querier.HeadStatus, error) + + // MergeOutOfOrderChunks send signal to merge chunks with out of order data chunks. + MergeOutOfOrderChunks() } // StreamProcessor interface. diff --git a/pp-pkg/handler/otlp_handler.go b/pp-pkg/handler/otlp_handler.go index e5b4d24711..80c146c1a3 100644 --- a/pp-pkg/handler/otlp_handler.go +++ b/pp-pkg/handler/otlp_handler.go @@ -22,6 +22,7 @@ import ( conventions "go.opentelemetry.io/collector/semconv/v1.6.1" "go.uber.org/multierr" + "github.com/prometheus/prometheus/config" prom_config "github.com/prometheus/prometheus/config" "github.com/prometheus/prometheus/model/value" ppmodel "github.com/prometheus/prometheus/pp/go/model" @@ -49,19 +50,33 @@ var OTLPAlwaysCommit = true // OTLPWriteHandler handler for otlp data via remote write. type OTLPWriteHandler struct { - logger log.Logger - receiver Receiver + logger log.Logger + adapter Adapter + states *StatesStorage } -func NewOTLPWriteHandler(logger log.Logger, receiver Receiver) *OTLPWriteHandler { +func NewOTLPWriteHandler(logger log.Logger, adapter Adapter) *OTLPWriteHandler { return &OTLPWriteHandler{ - logger: logger, - receiver: receiver, + logger: logger, + adapter: adapter, + states: NewStatesStorage(), } } +// ApplyConfig updates the configs for [StatesStorage]. +func (h *OTLPWriteHandler) ApplyConfig(conf *config.Config) error { + return h.states.ApplyConfig(conf) +} + // ServeHTTP implementation http.Handler. func (h *OTLPWriteHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { + state, ok := h.states.GetStateByID(prom_config.TransparentRelabeler) + if !ok { + level.Error(h.logger).Log("msg", "failed get state", "err", "unknown relabler id") + w.WriteHeader(http.StatusServiceUnavailable) + return + } + req, err := DecodeOTLPWriteRequest(r) if err != nil { level.Error(h.logger).Log("msg", "Error decoding remote write request", "err", err.Error()) @@ -74,11 +89,10 @@ func (h *OTLPWriteHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { level.Warn(h.logger).Log("msg", "Error translating OTLP metrics to Prometheus write request", "err", err) } - stats, err := h.receiver.AppendTimeSeries( + stats, err := h.adapter.AppendTimeSeries( r.Context(), converter.TimeSeries(), - nil, - prom_config.TransparentRelabeler, + state, OTLPAlwaysCommit, ) diff --git a/pp-pkg/handler/pp_handler.go b/pp-pkg/handler/pp_handler.go index 1641e2c7d9..25787c9c38 100644 --- a/pp-pkg/handler/pp_handler.go +++ b/pp-pkg/handler/pp_handler.go @@ -11,6 +11,7 @@ import ( "go.uber.org/atomic" "golang.org/x/net/websocket" + "github.com/prometheus/prometheus/config" "github.com/prometheus/prometheus/pp-pkg/handler/adapter" "github.com/prometheus/prometheus/pp-pkg/handler/decoder/ppcore" "github.com/prometheus/prometheus/pp-pkg/handler/middleware" @@ -25,7 +26,8 @@ const ppLocalStoragePath = "ppdata/" // PPHandler service for remote write via pp-protocol. type PPHandler struct { - receiver Receiver + adapter Adapter + states *StatesStorage logger log.Logger stream StreamProcessor refill RefillProcessor @@ -39,24 +41,27 @@ type PPHandler struct { // NewPPHandler init new PPHandler. func NewPPHandler( workDir string, - receiver Receiver, + ar Adapter, logger log.Logger, registerer prometheus.Registerer, ) *PPHandler { - buffers := pool.New(8, 1e6, 2, func(sz int) interface{} { return make([]byte, 0, sz) }) + buffers := pool.New(8, 1e6, 2, func(sz int) any { return make([]byte, 0, sz) }) ppBlockStorage := block.NewStorage(filepath.Join(workDir, ppLocalStoragePath), buffers) + states := NewStatesStorage() factory := util.NewUnconflictRegisterer(registerer) h := &PPHandler{ - receiver: receiver, - logger: log.With(logger, "component", "pp_handler"), - stream: processor.NewStreamProcessor(ppcore.NewBuilder(ppBlockStorage), receiver, registerer), + adapter: ar, + states: states, + logger: log.With(logger, "component", "pp_handler"), + stream: processor.NewStreamProcessor(ppcore.NewBuilder(ppBlockStorage), ar, states, registerer), refill: processor.NewRefillProcessor( ppcore.NewReplayDecoderBuilder(ppBlockStorage), - receiver, + ar, + states, logger, registerer, ), - remoteWrite: processor.NewRemoteWriteProcessor(receiver, registerer), + remoteWrite: processor.NewRemoteWriteProcessor(ar, states, registerer), buffers: buffers, stop: new(atomic.Bool), // stats @@ -74,6 +79,11 @@ func NewPPHandler( return h } +// ApplyConfig updates the configs for [StatesStorage]. +func (h *PPHandler) ApplyConfig(conf *config.Config) error { + return h.states.ApplyConfig(conf) +} + // Websocket handler for websocket stream. func (h *PPHandler) Websocket(middlewares ...middleware.Middleware) http.HandlerFunc { hf := h.metadataValidator(websocket.Handler(h.websocketHandler).ServeHTTP) @@ -120,7 +130,7 @@ func (h *PPHandler) measure(next http.Handler, typeHandler string) http.HandlerF func (h *PPHandler) metadataValidator(next http.HandlerFunc) http.HandlerFunc { return func(rw http.ResponseWriter, r *http.Request) { metadata := middleware.MetadataFromContext(r.Context()) - if ok := h.receiver.RelabelerIDIsExist(metadata.RelabelerID); !ok { + if _, ok := h.states.GetStateByID(metadata.RelabelerID); !ok { level.Error(h.logger).Log("msg", "relabeler id not found", "relabeler_id", metadata.RelabelerID) rw.WriteHeader(http.StatusPreconditionFailed) return diff --git a/pp-pkg/handler/processor/adapter_moq_test.go b/pp-pkg/handler/processor/adapter_moq_test.go new file mode 100644 index 0000000000..e6dfa82767 --- /dev/null +++ b/pp-pkg/handler/processor/adapter_moq_test.go @@ -0,0 +1,252 @@ +// Code generated by moq; DO NOT EDIT. +// github.com/matryer/moq + +package processor_test + +import ( + "context" + pp_pkg_model "github.com/prometheus/prometheus/pp-pkg/model" + "github.com/prometheus/prometheus/pp/go/cppbridge" + "sync" +) + +// AdapterMock is a mock implementation of processor.Adapter. +// +// func TestSomethingThatUsesAdapter(t *testing.T) { +// +// // make and configure a mocked processor.Adapter +// mockedAdapter := &AdapterMock{ +// AppendHashdexFunc: func(ctx context.Context, hashdex cppbridge.ShardedData, state *cppbridge.StateV2, commitToWal bool) error { +// panic("mock out the AppendHashdex method") +// }, +// AppendSnappyProtobufFunc: func(ctx context.Context, compressedData pp_pkg_model.ProtobufData, state *cppbridge.StateV2, commitToWal bool) error { +// panic("mock out the AppendSnappyProtobuf method") +// }, +// AppendTimeSeriesFunc: func(ctx context.Context, data pp_pkg_model.TimeSeriesBatch, state *cppbridge.StateV2, commitToWal bool) (cppbridge.RelabelerStats, error) { +// panic("mock out the AppendTimeSeries method") +// }, +// MergeOutOfOrderChunksFunc: func() { +// panic("mock out the MergeOutOfOrderChunks method") +// }, +// } +// +// // use mockedAdapter in code that requires processor.Adapter +// // and then make assertions. +// +// } +type AdapterMock struct { + // AppendHashdexFunc mocks the AppendHashdex method. + AppendHashdexFunc func(ctx context.Context, hashdex cppbridge.ShardedData, state *cppbridge.StateV2, commitToWal bool) error + + // AppendSnappyProtobufFunc mocks the AppendSnappyProtobuf method. + AppendSnappyProtobufFunc func(ctx context.Context, compressedData pp_pkg_model.ProtobufData, state *cppbridge.StateV2, commitToWal bool) error + + // AppendTimeSeriesFunc mocks the AppendTimeSeries method. + AppendTimeSeriesFunc func(ctx context.Context, data pp_pkg_model.TimeSeriesBatch, state *cppbridge.StateV2, commitToWal bool) (cppbridge.RelabelerStats, error) + + // MergeOutOfOrderChunksFunc mocks the MergeOutOfOrderChunks method. + MergeOutOfOrderChunksFunc func() + + // calls tracks calls to the methods. + calls struct { + // AppendHashdex holds details about calls to the AppendHashdex method. + AppendHashdex []struct { + // Ctx is the ctx argument value. + Ctx context.Context + // Hashdex is the hashdex argument value. + Hashdex cppbridge.ShardedData + // State is the state argument value. + State *cppbridge.StateV2 + // CommitToWal is the commitToWal argument value. + CommitToWal bool + } + // AppendSnappyProtobuf holds details about calls to the AppendSnappyProtobuf method. + AppendSnappyProtobuf []struct { + // Ctx is the ctx argument value. + Ctx context.Context + // CompressedData is the compressedData argument value. + CompressedData pp_pkg_model.ProtobufData + // State is the state argument value. + State *cppbridge.StateV2 + // CommitToWal is the commitToWal argument value. + CommitToWal bool + } + // AppendTimeSeries holds details about calls to the AppendTimeSeries method. + AppendTimeSeries []struct { + // Ctx is the ctx argument value. + Ctx context.Context + // Data is the data argument value. + Data pp_pkg_model.TimeSeriesBatch + // State is the state argument value. + State *cppbridge.StateV2 + // CommitToWal is the commitToWal argument value. + CommitToWal bool + } + // MergeOutOfOrderChunks holds details about calls to the MergeOutOfOrderChunks method. + MergeOutOfOrderChunks []struct { + } + } + lockAppendHashdex sync.RWMutex + lockAppendSnappyProtobuf sync.RWMutex + lockAppendTimeSeries sync.RWMutex + lockMergeOutOfOrderChunks sync.RWMutex +} + +// AppendHashdex calls AppendHashdexFunc. +func (mock *AdapterMock) AppendHashdex(ctx context.Context, hashdex cppbridge.ShardedData, state *cppbridge.StateV2, commitToWal bool) error { + if mock.AppendHashdexFunc == nil { + panic("AdapterMock.AppendHashdexFunc: method is nil but Adapter.AppendHashdex was just called") + } + callInfo := struct { + Ctx context.Context + Hashdex cppbridge.ShardedData + State *cppbridge.StateV2 + CommitToWal bool + }{ + Ctx: ctx, + Hashdex: hashdex, + State: state, + CommitToWal: commitToWal, + } + mock.lockAppendHashdex.Lock() + mock.calls.AppendHashdex = append(mock.calls.AppendHashdex, callInfo) + mock.lockAppendHashdex.Unlock() + return mock.AppendHashdexFunc(ctx, hashdex, state, commitToWal) +} + +// AppendHashdexCalls gets all the calls that were made to AppendHashdex. +// Check the length with: +// +// len(mockedAdapter.AppendHashdexCalls()) +func (mock *AdapterMock) AppendHashdexCalls() []struct { + Ctx context.Context + Hashdex cppbridge.ShardedData + State *cppbridge.StateV2 + CommitToWal bool +} { + var calls []struct { + Ctx context.Context + Hashdex cppbridge.ShardedData + State *cppbridge.StateV2 + CommitToWal bool + } + mock.lockAppendHashdex.RLock() + calls = mock.calls.AppendHashdex + mock.lockAppendHashdex.RUnlock() + return calls +} + +// AppendSnappyProtobuf calls AppendSnappyProtobufFunc. +func (mock *AdapterMock) AppendSnappyProtobuf(ctx context.Context, compressedData pp_pkg_model.ProtobufData, state *cppbridge.StateV2, commitToWal bool) error { + if mock.AppendSnappyProtobufFunc == nil { + panic("AdapterMock.AppendSnappyProtobufFunc: method is nil but Adapter.AppendSnappyProtobuf was just called") + } + callInfo := struct { + Ctx context.Context + CompressedData pp_pkg_model.ProtobufData + State *cppbridge.StateV2 + CommitToWal bool + }{ + Ctx: ctx, + CompressedData: compressedData, + State: state, + CommitToWal: commitToWal, + } + mock.lockAppendSnappyProtobuf.Lock() + mock.calls.AppendSnappyProtobuf = append(mock.calls.AppendSnappyProtobuf, callInfo) + mock.lockAppendSnappyProtobuf.Unlock() + return mock.AppendSnappyProtobufFunc(ctx, compressedData, state, commitToWal) +} + +// AppendSnappyProtobufCalls gets all the calls that were made to AppendSnappyProtobuf. +// Check the length with: +// +// len(mockedAdapter.AppendSnappyProtobufCalls()) +func (mock *AdapterMock) AppendSnappyProtobufCalls() []struct { + Ctx context.Context + CompressedData pp_pkg_model.ProtobufData + State *cppbridge.StateV2 + CommitToWal bool +} { + var calls []struct { + Ctx context.Context + CompressedData pp_pkg_model.ProtobufData + State *cppbridge.StateV2 + CommitToWal bool + } + mock.lockAppendSnappyProtobuf.RLock() + calls = mock.calls.AppendSnappyProtobuf + mock.lockAppendSnappyProtobuf.RUnlock() + return calls +} + +// AppendTimeSeries calls AppendTimeSeriesFunc. +func (mock *AdapterMock) AppendTimeSeries(ctx context.Context, data pp_pkg_model.TimeSeriesBatch, state *cppbridge.StateV2, commitToWal bool) (cppbridge.RelabelerStats, error) { + if mock.AppendTimeSeriesFunc == nil { + panic("AdapterMock.AppendTimeSeriesFunc: method is nil but Adapter.AppendTimeSeries was just called") + } + callInfo := struct { + Ctx context.Context + Data pp_pkg_model.TimeSeriesBatch + State *cppbridge.StateV2 + CommitToWal bool + }{ + Ctx: ctx, + Data: data, + State: state, + CommitToWal: commitToWal, + } + mock.lockAppendTimeSeries.Lock() + mock.calls.AppendTimeSeries = append(mock.calls.AppendTimeSeries, callInfo) + mock.lockAppendTimeSeries.Unlock() + return mock.AppendTimeSeriesFunc(ctx, data, state, commitToWal) +} + +// AppendTimeSeriesCalls gets all the calls that were made to AppendTimeSeries. +// Check the length with: +// +// len(mockedAdapter.AppendTimeSeriesCalls()) +func (mock *AdapterMock) AppendTimeSeriesCalls() []struct { + Ctx context.Context + Data pp_pkg_model.TimeSeriesBatch + State *cppbridge.StateV2 + CommitToWal bool +} { + var calls []struct { + Ctx context.Context + Data pp_pkg_model.TimeSeriesBatch + State *cppbridge.StateV2 + CommitToWal bool + } + mock.lockAppendTimeSeries.RLock() + calls = mock.calls.AppendTimeSeries + mock.lockAppendTimeSeries.RUnlock() + return calls +} + +// MergeOutOfOrderChunks calls MergeOutOfOrderChunksFunc. +func (mock *AdapterMock) MergeOutOfOrderChunks() { + if mock.MergeOutOfOrderChunksFunc == nil { + panic("AdapterMock.MergeOutOfOrderChunksFunc: method is nil but Adapter.MergeOutOfOrderChunks was just called") + } + callInfo := struct { + }{} + mock.lockMergeOutOfOrderChunks.Lock() + mock.calls.MergeOutOfOrderChunks = append(mock.calls.MergeOutOfOrderChunks, callInfo) + mock.lockMergeOutOfOrderChunks.Unlock() + mock.MergeOutOfOrderChunksFunc() +} + +// MergeOutOfOrderChunksCalls gets all the calls that were made to MergeOutOfOrderChunks. +// Check the length with: +// +// len(mockedAdapter.MergeOutOfOrderChunksCalls()) +func (mock *AdapterMock) MergeOutOfOrderChunksCalls() []struct { +} { + var calls []struct { + } + mock.lockMergeOutOfOrderChunks.RLock() + calls = mock.calls.MergeOutOfOrderChunks + mock.lockMergeOutOfOrderChunks.RUnlock() + return calls +} diff --git a/pp-pkg/handler/processor/interface.go b/pp-pkg/handler/processor/interface.go index e15c944fe5..f7f8252ade 100644 --- a/pp-pkg/handler/processor/interface.go +++ b/pp-pkg/handler/processor/interface.go @@ -5,22 +5,27 @@ import ( "github.com/prometheus/prometheus/pp-pkg/handler/decoder" "github.com/prometheus/prometheus/pp-pkg/handler/model" + pp_pkg_model "github.com/prometheus/prometheus/pp-pkg/model" "github.com/prometheus/prometheus/pp/go/cppbridge" - "github.com/prometheus/prometheus/pp/go/relabeler" ) +//go:generate -command moq go run github.com/matryer/moq --rm --skip-ensure --pkg processor_test --out + +//go:generate moq metric_stream_moq_test.go . MetricStream type MetricStream interface { Metadata() model.Metadata Read(ctx context.Context) (*model.Segment, error) Write(ctx context.Context, status model.StreamSegmentProcessingStatus) error } +//go:generate moq refill_moq_test.go . Refill type Refill interface { Metadata() model.Metadata Read(ctx context.Context) (*model.Segment, error) Write(ctx context.Context, status model.RefillProcessingStatus) error } +//go:generate moq remote_write_moq_test.go . RemoteWrite type RemoteWrite interface { Metadata() model.Metadata Read(ctx context.Context) (*model.RemoteWriteBuffer, error) @@ -31,10 +36,42 @@ type DecoderBuilder interface { Build(metadata model.Metadata) decoder.Decoder } -// Receiver interface. -type Receiver interface { - AppendSnappyProtobuf(ctx context.Context, compressedData relabeler.ProtobufData, relabelerID string, commitToWal bool) error - AppendHashdex(ctx context.Context, hashdex cppbridge.ShardedData, relabelerID string, commitToWal bool) error - // MergeOutOfOrderChunks merge chunks with out of order data chunks. - MergeOutOfOrderChunks(ctx context.Context) +// Adapter for implementing the [Queryable] interface and append data. +// +//go:generate moq adapter_moq_test.go . Adapter +type Adapter interface { + // AppendHashdex append incoming [cppbridge.HashdexContent] to [Head]. + AppendHashdex( + ctx context.Context, + hashdex cppbridge.ShardedData, + state *cppbridge.StateV2, + commitToWal bool, + ) error + + // AppendTimeSeries append TimeSeries data to [Head]. + AppendTimeSeries( + ctx context.Context, + data pp_pkg_model.TimeSeriesBatch, + state *cppbridge.StateV2, + commitToWal bool, + ) (cppbridge.RelabelerStats, error) + + // AppendSnappyProtobuf append compressed via snappy Protobuf data to [Head]. + AppendSnappyProtobuf( + ctx context.Context, + compressedData pp_pkg_model.ProtobufData, + state *cppbridge.StateV2, + commitToWal bool, + ) error + + // MergeOutOfOrderChunks send signal to merge chunks with out of order data chunks. + MergeOutOfOrderChunks() +} + +// StatesStorage stores the [cppbridge.State]'s. +// +//go:generate moq states_storage_moq_test.go . StatesStorage +type StatesStorage interface { + // GetStateByID returns [cppbridge.State] by state ID if exist. + GetStateByID(stateID string) (*cppbridge.StateV2, bool) } diff --git a/pp-pkg/handler/processor/metric_stream_moq_test.go b/pp-pkg/handler/processor/metric_stream_moq_test.go new file mode 100644 index 0000000000..45746d3acc --- /dev/null +++ b/pp-pkg/handler/processor/metric_stream_moq_test.go @@ -0,0 +1,159 @@ +// Code generated by moq; DO NOT EDIT. +// github.com/matryer/moq + +package processor_test + +import ( + "context" + "github.com/prometheus/prometheus/pp-pkg/handler/model" + "sync" +) + +// MetricStreamMock is a mock implementation of processor.MetricStream. +// +// func TestSomethingThatUsesMetricStream(t *testing.T) { +// +// // make and configure a mocked processor.MetricStream +// mockedMetricStream := &MetricStreamMock{ +// MetadataFunc: func() model.Metadata { +// panic("mock out the Metadata method") +// }, +// ReadFunc: func(ctx context.Context) (*model.Segment, error) { +// panic("mock out the Read method") +// }, +// WriteFunc: func(ctx context.Context, status model.StreamSegmentProcessingStatus) error { +// panic("mock out the Write method") +// }, +// } +// +// // use mockedMetricStream in code that requires processor.MetricStream +// // and then make assertions. +// +// } +type MetricStreamMock struct { + // MetadataFunc mocks the Metadata method. + MetadataFunc func() model.Metadata + + // ReadFunc mocks the Read method. + ReadFunc func(ctx context.Context) (*model.Segment, error) + + // WriteFunc mocks the Write method. + WriteFunc func(ctx context.Context, status model.StreamSegmentProcessingStatus) error + + // calls tracks calls to the methods. + calls struct { + // Metadata holds details about calls to the Metadata method. + Metadata []struct { + } + // Read holds details about calls to the Read method. + Read []struct { + // Ctx is the ctx argument value. + Ctx context.Context + } + // Write holds details about calls to the Write method. + Write []struct { + // Ctx is the ctx argument value. + Ctx context.Context + // Status is the status argument value. + Status model.StreamSegmentProcessingStatus + } + } + lockMetadata sync.RWMutex + lockRead sync.RWMutex + lockWrite sync.RWMutex +} + +// Metadata calls MetadataFunc. +func (mock *MetricStreamMock) Metadata() model.Metadata { + if mock.MetadataFunc == nil { + panic("MetricStreamMock.MetadataFunc: method is nil but MetricStream.Metadata was just called") + } + callInfo := struct { + }{} + mock.lockMetadata.Lock() + mock.calls.Metadata = append(mock.calls.Metadata, callInfo) + mock.lockMetadata.Unlock() + return mock.MetadataFunc() +} + +// MetadataCalls gets all the calls that were made to Metadata. +// Check the length with: +// +// len(mockedMetricStream.MetadataCalls()) +func (mock *MetricStreamMock) MetadataCalls() []struct { +} { + var calls []struct { + } + mock.lockMetadata.RLock() + calls = mock.calls.Metadata + mock.lockMetadata.RUnlock() + return calls +} + +// Read calls ReadFunc. +func (mock *MetricStreamMock) Read(ctx context.Context) (*model.Segment, error) { + if mock.ReadFunc == nil { + panic("MetricStreamMock.ReadFunc: method is nil but MetricStream.Read was just called") + } + callInfo := struct { + Ctx context.Context + }{ + Ctx: ctx, + } + mock.lockRead.Lock() + mock.calls.Read = append(mock.calls.Read, callInfo) + mock.lockRead.Unlock() + return mock.ReadFunc(ctx) +} + +// ReadCalls gets all the calls that were made to Read. +// Check the length with: +// +// len(mockedMetricStream.ReadCalls()) +func (mock *MetricStreamMock) ReadCalls() []struct { + Ctx context.Context +} { + var calls []struct { + Ctx context.Context + } + mock.lockRead.RLock() + calls = mock.calls.Read + mock.lockRead.RUnlock() + return calls +} + +// Write calls WriteFunc. +func (mock *MetricStreamMock) Write(ctx context.Context, status model.StreamSegmentProcessingStatus) error { + if mock.WriteFunc == nil { + panic("MetricStreamMock.WriteFunc: method is nil but MetricStream.Write was just called") + } + callInfo := struct { + Ctx context.Context + Status model.StreamSegmentProcessingStatus + }{ + Ctx: ctx, + Status: status, + } + mock.lockWrite.Lock() + mock.calls.Write = append(mock.calls.Write, callInfo) + mock.lockWrite.Unlock() + return mock.WriteFunc(ctx, status) +} + +// WriteCalls gets all the calls that were made to Write. +// Check the length with: +// +// len(mockedMetricStream.WriteCalls()) +func (mock *MetricStreamMock) WriteCalls() []struct { + Ctx context.Context + Status model.StreamSegmentProcessingStatus +} { + var calls []struct { + Ctx context.Context + Status model.StreamSegmentProcessingStatus + } + mock.lockWrite.RLock() + calls = mock.calls.Write + mock.lockWrite.RUnlock() + return calls +} diff --git a/pp-pkg/handler/processor/refill_moq_test.go b/pp-pkg/handler/processor/refill_moq_test.go new file mode 100644 index 0000000000..1d21aadac7 --- /dev/null +++ b/pp-pkg/handler/processor/refill_moq_test.go @@ -0,0 +1,159 @@ +// Code generated by moq; DO NOT EDIT. +// github.com/matryer/moq + +package processor_test + +import ( + "context" + "github.com/prometheus/prometheus/pp-pkg/handler/model" + "sync" +) + +// RefillMock is a mock implementation of processor.Refill. +// +// func TestSomethingThatUsesRefill(t *testing.T) { +// +// // make and configure a mocked processor.Refill +// mockedRefill := &RefillMock{ +// MetadataFunc: func() model.Metadata { +// panic("mock out the Metadata method") +// }, +// ReadFunc: func(ctx context.Context) (*model.Segment, error) { +// panic("mock out the Read method") +// }, +// WriteFunc: func(ctx context.Context, status model.RefillProcessingStatus) error { +// panic("mock out the Write method") +// }, +// } +// +// // use mockedRefill in code that requires processor.Refill +// // and then make assertions. +// +// } +type RefillMock struct { + // MetadataFunc mocks the Metadata method. + MetadataFunc func() model.Metadata + + // ReadFunc mocks the Read method. + ReadFunc func(ctx context.Context) (*model.Segment, error) + + // WriteFunc mocks the Write method. + WriteFunc func(ctx context.Context, status model.RefillProcessingStatus) error + + // calls tracks calls to the methods. + calls struct { + // Metadata holds details about calls to the Metadata method. + Metadata []struct { + } + // Read holds details about calls to the Read method. + Read []struct { + // Ctx is the ctx argument value. + Ctx context.Context + } + // Write holds details about calls to the Write method. + Write []struct { + // Ctx is the ctx argument value. + Ctx context.Context + // Status is the status argument value. + Status model.RefillProcessingStatus + } + } + lockMetadata sync.RWMutex + lockRead sync.RWMutex + lockWrite sync.RWMutex +} + +// Metadata calls MetadataFunc. +func (mock *RefillMock) Metadata() model.Metadata { + if mock.MetadataFunc == nil { + panic("RefillMock.MetadataFunc: method is nil but Refill.Metadata was just called") + } + callInfo := struct { + }{} + mock.lockMetadata.Lock() + mock.calls.Metadata = append(mock.calls.Metadata, callInfo) + mock.lockMetadata.Unlock() + return mock.MetadataFunc() +} + +// MetadataCalls gets all the calls that were made to Metadata. +// Check the length with: +// +// len(mockedRefill.MetadataCalls()) +func (mock *RefillMock) MetadataCalls() []struct { +} { + var calls []struct { + } + mock.lockMetadata.RLock() + calls = mock.calls.Metadata + mock.lockMetadata.RUnlock() + return calls +} + +// Read calls ReadFunc. +func (mock *RefillMock) Read(ctx context.Context) (*model.Segment, error) { + if mock.ReadFunc == nil { + panic("RefillMock.ReadFunc: method is nil but Refill.Read was just called") + } + callInfo := struct { + Ctx context.Context + }{ + Ctx: ctx, + } + mock.lockRead.Lock() + mock.calls.Read = append(mock.calls.Read, callInfo) + mock.lockRead.Unlock() + return mock.ReadFunc(ctx) +} + +// ReadCalls gets all the calls that were made to Read. +// Check the length with: +// +// len(mockedRefill.ReadCalls()) +func (mock *RefillMock) ReadCalls() []struct { + Ctx context.Context +} { + var calls []struct { + Ctx context.Context + } + mock.lockRead.RLock() + calls = mock.calls.Read + mock.lockRead.RUnlock() + return calls +} + +// Write calls WriteFunc. +func (mock *RefillMock) Write(ctx context.Context, status model.RefillProcessingStatus) error { + if mock.WriteFunc == nil { + panic("RefillMock.WriteFunc: method is nil but Refill.Write was just called") + } + callInfo := struct { + Ctx context.Context + Status model.RefillProcessingStatus + }{ + Ctx: ctx, + Status: status, + } + mock.lockWrite.Lock() + mock.calls.Write = append(mock.calls.Write, callInfo) + mock.lockWrite.Unlock() + return mock.WriteFunc(ctx, status) +} + +// WriteCalls gets all the calls that were made to Write. +// Check the length with: +// +// len(mockedRefill.WriteCalls()) +func (mock *RefillMock) WriteCalls() []struct { + Ctx context.Context + Status model.RefillProcessingStatus +} { + var calls []struct { + Ctx context.Context + Status model.RefillProcessingStatus + } + mock.lockWrite.RLock() + calls = mock.calls.Write + mock.lockWrite.RUnlock() + return calls +} diff --git a/pp-pkg/handler/processor/refill_processor.go b/pp-pkg/handler/processor/refill_processor.go index c9991622e3..8232aac644 100644 --- a/pp-pkg/handler/processor/refill_processor.go +++ b/pp-pkg/handler/processor/refill_processor.go @@ -16,7 +16,8 @@ import ( type RefillProcessor struct { decoderBuilder DecoderBuilder - receiver Receiver + adapter Adapter + states StatesStorage logger log.Logger criticalErrorCount *prometheus.CounterVec @@ -29,14 +30,16 @@ type RefillProcessor struct { func NewRefillProcessor( decoderBuilder DecoderBuilder, - receiver Receiver, + adapter Adapter, + states StatesStorage, logger log.Logger, registerer prometheus.Registerer, ) *RefillProcessor { factory := util.NewUnconflictRegisterer(registerer) return &RefillProcessor{ decoderBuilder: decoderBuilder, - receiver: receiver, + adapter: adapter, + states: states, logger: log.With(logger, "component", "refill_processor"), criticalErrorCount: factory.NewCounterVec(prometheus.CounterOpts{ Name: "remote_write_opprotocol_processor_critical_error_count", @@ -67,6 +70,16 @@ func NewRefillProcessor( func (p *RefillProcessor) Process(ctx context.Context, refill Refill) error { meta := refill.Metadata() + + state, ok := p.states.GetStateByID(meta.RelabelerID) + if !ok { + p.criticalErrorCount.With(prometheus.Labels{ + "error": ErrUnknownRelablerID.Error(), + "processor_type": "stream", + }).Inc() + return ErrUnknownRelablerID + } + decoder := p.decoderBuilder.Build(meta) defer func() { _ = decoder.Close() }() @@ -80,7 +93,9 @@ func (p *RefillProcessor) Process(ctx context.Context, refill Refill) error { if err != nil { if errors.Is(err, io.EOF) { if disErr := decoder.Discard(); disErr != nil { - p.criticalErrorCount.With(prometheus.Labels{"error": disErr.Error(), "processor_type": "refill"}).Inc() + p.criticalErrorCount.With( + prometheus.Labels{"error": disErr.Error(), "processor_type": "refill"}, + ).Inc() } p.writtenSeriesCount.With(prometheus.Labels{"processor_type": "refill"}).Add(float64(decodedSeries)) @@ -90,7 +105,7 @@ func (p *RefillProcessor) Process(ctx context.Context, refill Refill) error { prometheus.Labels{"processor_type": "refill", "status_code": "200"}, ).Inc() - p.receiver.MergeOutOfOrderChunks(ctx) + p.adapter.MergeOutOfOrderChunks() return refill.Write(ctx, model.RefillProcessingStatus{Code: http.StatusOK}) } @@ -111,7 +126,7 @@ func (p *RefillProcessor) Process(ctx context.Context, refill Refill) error { decodedSamples += hashdexContent.Samples() p.decodedSampleCount.With(prometheus.Labels{"processor_type": "refill"}).Add(float64(hashdexContent.Samples())) - if err = p.receiver.AppendHashdex(ctx, hashdexContent.ShardedData(), meta.RelabelerID, true); err != nil { + if err = p.adapter.AppendHashdex(ctx, hashdexContent.ShardedData(), state, true); err != nil { p.criticalErrorCount.With(prometheus.Labels{"error": err.Error(), "processor_type": "refill"}).Inc() return fmt.Errorf("failed to append decoded segment: %w", err) } diff --git a/pp-pkg/handler/processor/refill_processor_test.go b/pp-pkg/handler/processor/refill_processor_test.go index 3f7d9f2360..922846c0e4 100644 --- a/pp-pkg/handler/processor/refill_processor_test.go +++ b/pp-pkg/handler/processor/refill_processor_test.go @@ -37,28 +37,41 @@ func (s *RefillProcessorSuite) TestProcess() { buffers := pool.New(8, 100e3, 2, func(sz int) any { return make([]byte, 0, sz) }) blockStorage := block.NewStorage(tmpDir, buffers) - mr := &metricReceiver{appendFn: func(ctx context.Context, hashdex cppbridge.ShardedData, relabelerID string) error { - return nil - }} + ar := &AdapterMock{ + AppendHashdexFunc: func(context.Context, cppbridge.ShardedData, *cppbridge.StateV2, bool) error { + return nil + }, + MergeOutOfOrderChunksFunc: func() {}, + } + + metadata := model.Metadata{ + TenantID: "", + BlockID: uuid.New(), + ShardID: 0, + ShardsLog: 0, + SegmentEncodingVersion: 3, + RelabelerID: uuid.New().String(), + } + + states := &StatesStorageMock{ + GetStateByIDFunc: func(stateID string) (*cppbridge.StateV2, bool) { + if metadata.RelabelerID != stateID { + return nil, false + } - blockID := uuid.New() - shardID := uint16(0) - shardLog := uint8(0) - segmentEncodingVersion := cppbridge.EncodersVersion() + return nil, true + }, + } var expectedStatus model.RefillProcessingStatus gen := &segmentGenerator{segmentSize: 10} - refill := &testRefill{ - metadata: model.Metadata{ - TenantID: "", - BlockID: blockID, - ShardID: shardID, - ShardsLog: shardLog, - SegmentEncodingVersion: segmentEncodingVersion, + refill := &RefillMock{ + MetadataFunc: func() model.Metadata { + return metadata }, - readFn: func(ctx context.Context) (*model.Segment, error) { + ReadFunc: func(context.Context) (*model.Segment, error) { if len(gen.segments) == 5 { return nil, io.EOF } @@ -75,14 +88,14 @@ func (s *RefillProcessorSuite) TestProcess() { segment, readErr := gen.generate() return &segment.encoded, readErr }, - writeFn: func(ctx context.Context, status model.RefillProcessingStatus) error { + WriteFunc: func(_ context.Context, status model.RefillProcessingStatus) error { expectedStatus = status return nil }, } decoderBuilder := ppcore.NewBuilder(blockStorage) - refillProcessor := processor.NewRefillProcessor(decoderBuilder, mr, log.NewNopLogger(), nil) + refillProcessor := processor.NewRefillProcessor(decoderBuilder, ar, states, log.NewNopLogger(), nil) ctx, cancel := context.WithTimeout(context.Background(), time.Minute*5) defer cancel() @@ -101,28 +114,41 @@ func (s *RefillProcessorSuite) TestProcessWithError() { buffers := pool.New(8, 100e3, 2, func(sz int) any { return make([]byte, 0, sz) }) blockStorage := block.NewStorage(tmpDir, buffers) - mr := &metricReceiver{appendFn: func(ctx context.Context, hashdex cppbridge.ShardedData, relabelerID string) error { - return nil - }} + ar := &AdapterMock{ + AppendHashdexFunc: func(context.Context, cppbridge.ShardedData, *cppbridge.StateV2, bool) error { + return nil + }, + MergeOutOfOrderChunksFunc: func() {}, + } - blockID := uuid.New() - shardID := uint16(0) - shardLog := uint8(0) - segmentEncodingVersion := cppbridge.EncodersVersion() + metadata := model.Metadata{ + TenantID: "", + BlockID: uuid.New(), + ShardID: 0, + ShardsLog: 0, + SegmentEncodingVersion: 3, + RelabelerID: uuid.New().String(), + } + + states := &StatesStorageMock{ + GetStateByIDFunc: func(stateID string) (*cppbridge.StateV2, bool) { + if metadata.RelabelerID != stateID { + return nil, false + } + + return nil, true + }, + } gen := &segmentGenerator{segmentSize: 10} fakeErr := errors.New("read error") - refill := &testRefill{ - metadata: model.Metadata{ - TenantID: "", - BlockID: blockID, - ShardID: shardID, - ShardsLog: shardLog, - SegmentEncodingVersion: segmentEncodingVersion, + refill := &RefillMock{ + MetadataFunc: func() model.Metadata { + return metadata }, - readFn: func(ctx context.Context) (*model.Segment, error) { + ReadFunc: func(context.Context) (*model.Segment, error) { if len(gen.segments) == 3 { return &model.Segment{}, fakeErr } @@ -130,13 +156,13 @@ func (s *RefillProcessorSuite) TestProcessWithError() { segment, readErr := gen.generate() return &segment.encoded, readErr }, - writeFn: func(ctx context.Context, status model.RefillProcessingStatus) error { + WriteFunc: func(context.Context, model.RefillProcessingStatus) error { return nil }, } decoderBuilder := ppcore.NewBuilder(blockStorage) - refillProcessor := processor.NewRefillProcessor(decoderBuilder, mr, log.NewNopLogger(), nil) + refillProcessor := processor.NewRefillProcessor(decoderBuilder, ar, states, log.NewNopLogger(), nil) ctx, cancel := context.WithTimeout(context.Background(), time.Minute*5) defer cancel() @@ -144,25 +170,3 @@ func (s *RefillProcessorSuite) TestProcessWithError() { err = refillProcessor.Process(ctx, refill) s.Require().ErrorIs(err, fakeErr) } - -// -// testRefill -// - -type testRefill struct { - metadata model.Metadata - readFn func(ctx context.Context) (*model.Segment, error) - writeFn func(ctx context.Context, status model.RefillProcessingStatus) error -} - -func (s *testRefill) Metadata() model.Metadata { - return s.metadata -} - -func (s *testRefill) Read(ctx context.Context) (*model.Segment, error) { - return s.readFn(ctx) -} - -func (s *testRefill) Write(ctx context.Context, status model.RefillProcessingStatus) error { - return s.writeFn(ctx, status) -} diff --git a/pp-pkg/handler/processor/remote_write_moq_test.go b/pp-pkg/handler/processor/remote_write_moq_test.go new file mode 100644 index 0000000000..3080685fe3 --- /dev/null +++ b/pp-pkg/handler/processor/remote_write_moq_test.go @@ -0,0 +1,159 @@ +// Code generated by moq; DO NOT EDIT. +// github.com/matryer/moq + +package processor_test + +import ( + "context" + "github.com/prometheus/prometheus/pp-pkg/handler/model" + "sync" +) + +// RemoteWriteMock is a mock implementation of processor.RemoteWrite. +// +// func TestSomethingThatUsesRemoteWrite(t *testing.T) { +// +// // make and configure a mocked processor.RemoteWrite +// mockedRemoteWrite := &RemoteWriteMock{ +// MetadataFunc: func() model.Metadata { +// panic("mock out the Metadata method") +// }, +// ReadFunc: func(ctx context.Context) (*model.RemoteWriteBuffer, error) { +// panic("mock out the Read method") +// }, +// WriteFunc: func(ctx context.Context, status model.RemoteWriteProcessingStatus) error { +// panic("mock out the Write method") +// }, +// } +// +// // use mockedRemoteWrite in code that requires processor.RemoteWrite +// // and then make assertions. +// +// } +type RemoteWriteMock struct { + // MetadataFunc mocks the Metadata method. + MetadataFunc func() model.Metadata + + // ReadFunc mocks the Read method. + ReadFunc func(ctx context.Context) (*model.RemoteWriteBuffer, error) + + // WriteFunc mocks the Write method. + WriteFunc func(ctx context.Context, status model.RemoteWriteProcessingStatus) error + + // calls tracks calls to the methods. + calls struct { + // Metadata holds details about calls to the Metadata method. + Metadata []struct { + } + // Read holds details about calls to the Read method. + Read []struct { + // Ctx is the ctx argument value. + Ctx context.Context + } + // Write holds details about calls to the Write method. + Write []struct { + // Ctx is the ctx argument value. + Ctx context.Context + // Status is the status argument value. + Status model.RemoteWriteProcessingStatus + } + } + lockMetadata sync.RWMutex + lockRead sync.RWMutex + lockWrite sync.RWMutex +} + +// Metadata calls MetadataFunc. +func (mock *RemoteWriteMock) Metadata() model.Metadata { + if mock.MetadataFunc == nil { + panic("RemoteWriteMock.MetadataFunc: method is nil but RemoteWrite.Metadata was just called") + } + callInfo := struct { + }{} + mock.lockMetadata.Lock() + mock.calls.Metadata = append(mock.calls.Metadata, callInfo) + mock.lockMetadata.Unlock() + return mock.MetadataFunc() +} + +// MetadataCalls gets all the calls that were made to Metadata. +// Check the length with: +// +// len(mockedRemoteWrite.MetadataCalls()) +func (mock *RemoteWriteMock) MetadataCalls() []struct { +} { + var calls []struct { + } + mock.lockMetadata.RLock() + calls = mock.calls.Metadata + mock.lockMetadata.RUnlock() + return calls +} + +// Read calls ReadFunc. +func (mock *RemoteWriteMock) Read(ctx context.Context) (*model.RemoteWriteBuffer, error) { + if mock.ReadFunc == nil { + panic("RemoteWriteMock.ReadFunc: method is nil but RemoteWrite.Read was just called") + } + callInfo := struct { + Ctx context.Context + }{ + Ctx: ctx, + } + mock.lockRead.Lock() + mock.calls.Read = append(mock.calls.Read, callInfo) + mock.lockRead.Unlock() + return mock.ReadFunc(ctx) +} + +// ReadCalls gets all the calls that were made to Read. +// Check the length with: +// +// len(mockedRemoteWrite.ReadCalls()) +func (mock *RemoteWriteMock) ReadCalls() []struct { + Ctx context.Context +} { + var calls []struct { + Ctx context.Context + } + mock.lockRead.RLock() + calls = mock.calls.Read + mock.lockRead.RUnlock() + return calls +} + +// Write calls WriteFunc. +func (mock *RemoteWriteMock) Write(ctx context.Context, status model.RemoteWriteProcessingStatus) error { + if mock.WriteFunc == nil { + panic("RemoteWriteMock.WriteFunc: method is nil but RemoteWrite.Write was just called") + } + callInfo := struct { + Ctx context.Context + Status model.RemoteWriteProcessingStatus + }{ + Ctx: ctx, + Status: status, + } + mock.lockWrite.Lock() + mock.calls.Write = append(mock.calls.Write, callInfo) + mock.lockWrite.Unlock() + return mock.WriteFunc(ctx, status) +} + +// WriteCalls gets all the calls that were made to Write. +// Check the length with: +// +// len(mockedRemoteWrite.WriteCalls()) +func (mock *RemoteWriteMock) WriteCalls() []struct { + Ctx context.Context + Status model.RemoteWriteProcessingStatus +} { + var calls []struct { + Ctx context.Context + Status model.RemoteWriteProcessingStatus + } + mock.lockWrite.RLock() + calls = mock.calls.Write + mock.lockWrite.RUnlock() + return calls +} diff --git a/pp-pkg/handler/processor/remote_write_processor.go b/pp-pkg/handler/processor/remote_write_processor.go index 94970a85dd..2c8abf8529 100644 --- a/pp-pkg/handler/processor/remote_write_processor.go +++ b/pp-pkg/handler/processor/remote_write_processor.go @@ -2,6 +2,7 @@ package processor import ( "context" + "errors" "net/http" "strconv" @@ -11,22 +12,33 @@ import ( "github.com/prometheus/prometheus/pp/go/util" ) -var AlwaysCommit = true +var ( + // AlwaysCommit commit flags. + AlwaysCommit = true + // ErrUnknownRelablerID error when relabler ID not found. + ErrUnknownRelablerID = errors.New("unknown relabler id") +) + +// RemoteWriteProcessor RemoteWrite processor. type RemoteWriteProcessor struct { - receiver Receiver + adapter Adapter + states StatesStorage responseStatusCodeCount *prometheus.CounterVec } +// NewRemoteWriteProcessor init new [RemoteWriteProcessor]. func NewRemoteWriteProcessor( - receiver Receiver, + adapter Adapter, + states StatesStorage, registerer prometheus.Registerer, ) *RemoteWriteProcessor { factory := util.NewUnconflictRegisterer(registerer) return &RemoteWriteProcessor{ - receiver: receiver, + adapter: adapter, + states: states, responseStatusCodeCount: factory.NewCounterVec(prometheus.CounterOpts{ Name: "remote_write_opprotocol_processor_response_status_code", Help: "Number of 200/400 status codes responded with.", @@ -34,6 +46,7 @@ func NewRemoteWriteProcessor( } } +// Process read remote write data and append to adapter. func (p *RemoteWriteProcessor) Process(ctx context.Context, remoteWrite RemoteWrite) error { status := model.RemoteWriteProcessingStatus{Code: http.StatusOK} defer func() { @@ -43,6 +56,13 @@ func (p *RemoteWriteProcessor) Process(ctx context.Context, remoteWrite RemoteWr _ = remoteWrite.Write(ctx, status) }() + state, ok := p.states.GetStateByID(remoteWrite.Metadata().RelabelerID) + if !ok { + status.Code = http.StatusPreconditionFailed + status.Message = ErrUnknownRelablerID.Error() + return ErrUnknownRelablerID + } + rwb, err := remoteWrite.Read(ctx) if err != nil { status.Code = http.StatusBadRequest @@ -50,7 +70,7 @@ func (p *RemoteWriteProcessor) Process(ctx context.Context, remoteWrite RemoteWr return err } - if err := p.receiver.AppendSnappyProtobuf(ctx, rwb, remoteWrite.Metadata().RelabelerID, AlwaysCommit); err != nil { + if err := p.adapter.AppendSnappyProtobuf(ctx, rwb, state, AlwaysCommit); err != nil { status.Code = http.StatusBadRequest status.Message = err.Error() return err diff --git a/pp-pkg/handler/processor/remote_write_processor_test.go b/pp-pkg/handler/processor/remote_write_processor_test.go index a8270454c1..e4b1e97a5d 100644 --- a/pp-pkg/handler/processor/remote_write_processor_test.go +++ b/pp-pkg/handler/processor/remote_write_processor_test.go @@ -12,6 +12,7 @@ import ( "github.com/prometheus/prometheus/pp-pkg/handler/model" "github.com/prometheus/prometheus/pp-pkg/handler/processor" + pp_pkg_model "github.com/prometheus/prometheus/pp-pkg/model" "github.com/prometheus/prometheus/pp/go/cppbridge" ) @@ -24,35 +25,49 @@ func TestRemoteWriteProcessorSuite(t *testing.T) { } func (s *RemoteWriteProcessorSuite) TestProcess() { - mr := &metricReceiver{appendFn: func(ctx context.Context, hashdex cppbridge.ShardedData, relabelerID string) error { - return nil - }} - - blockID := uuid.New() - shardID := uint16(0) - shardLog := uint8(0) - segmentEncodingVersion := cppbridge.EncodersVersion() - - var expectedStatus model.RemoteWriteProcessingStatus - - rw := &testRemoteWrite{ - metadata: model.Metadata{ - TenantID: "", - BlockID: blockID, - ShardID: shardID, - ShardsLog: shardLog, - SegmentEncodingVersion: segmentEncodingVersion, + ar := &AdapterMock{ + AppendSnappyProtobufFunc: func(context.Context, pp_pkg_model.ProtobufData, *cppbridge.StateV2, bool) error { + return nil + }, + } + + metadata := model.Metadata{ + TenantID: "", + BlockID: uuid.New(), + ShardID: 0, + ShardsLog: 0, + SegmentEncodingVersion: 3, + RelabelerID: uuid.New().String(), + } + + states := &StatesStorageMock{ + GetStateByIDFunc: func(stateID string) (*cppbridge.StateV2, bool) { + if metadata.RelabelerID != stateID { + return nil, false + } + + return nil, true + }, + } + + var actualStatus model.RemoteWriteProcessingStatus + + rw := &RemoteWriteMock{ + MetadataFunc: func() model.Metadata { + return metadata }, - readFn: func(ctx context.Context) (*model.RemoteWriteBuffer, error) { + + ReadFunc: func(context.Context) (*model.RemoteWriteBuffer, error) { return &model.RemoteWriteBuffer{}, nil }, - writeFn: func(ctx context.Context, status model.RemoteWriteProcessingStatus) error { - expectedStatus = status + + WriteFunc: func(_ context.Context, status model.RemoteWriteProcessingStatus) error { + actualStatus = status return nil }, } - rwProcessor := processor.NewRemoteWriteProcessor(mr, nil) + rwProcessor := processor.NewRemoteWriteProcessor(ar, states, nil) ctx, cancel := context.WithTimeout(context.Background(), time.Minute*5) defer cancel() @@ -60,41 +75,54 @@ func (s *RemoteWriteProcessorSuite) TestProcess() { err := rwProcessor.Process(ctx, rw) s.Require().NoError(err) - s.Equal(http.StatusOK, expectedStatus.Code) + s.Equal(http.StatusOK, actualStatus.Code) } -func (s *RemoteWriteProcessorSuite) TestProcessWithError() { - mr := &metricReceiver{appendFn: func(ctx context.Context, hashdex cppbridge.ShardedData, relabelerID string) error { - return nil - }} +func (s *RemoteWriteProcessorSuite) TestProcessWithErrorRead() { + ar := &AdapterMock{ + AppendSnappyProtobufFunc: func(context.Context, pp_pkg_model.ProtobufData, *cppbridge.StateV2, bool) error { + return nil + }, + } - blockID := uuid.New() - shardID := uint16(0) - shardLog := uint8(0) - segmentEncodingVersion := cppbridge.EncodersVersion() + metadata := model.Metadata{ + TenantID: "", + BlockID: uuid.New(), + ShardID: 0, + ShardsLog: 0, + SegmentEncodingVersion: 3, + RelabelerID: uuid.New().String(), + } - fakeErr := errors.New("read error") + states := &StatesStorageMock{ + GetStateByIDFunc: func(stateID string) (*cppbridge.StateV2, bool) { + if metadata.RelabelerID != stateID { + return nil, false + } - var expectedStatus model.RemoteWriteProcessingStatus + return nil, true + }, + } + + fakeErr := errors.New("read error") + var actualStatus model.RemoteWriteProcessingStatus - rw := &testRemoteWrite{ - metadata: model.Metadata{ - TenantID: "", - BlockID: blockID, - ShardID: shardID, - ShardsLog: shardLog, - SegmentEncodingVersion: segmentEncodingVersion, + rw := &RemoteWriteMock{ + MetadataFunc: func() model.Metadata { + return metadata }, - readFn: func(ctx context.Context) (*model.RemoteWriteBuffer, error) { + + ReadFunc: func(context.Context) (*model.RemoteWriteBuffer, error) { return nil, fakeErr }, - writeFn: func(ctx context.Context, status model.RemoteWriteProcessingStatus) error { - expectedStatus = status + + WriteFunc: func(_ context.Context, status model.RemoteWriteProcessingStatus) error { + actualStatus = status return nil }, } - rwProcessor := processor.NewRemoteWriteProcessor(mr, nil) + rwProcessor := processor.NewRemoteWriteProcessor(ar, states, nil) ctx, cancel := context.WithTimeout(context.Background(), time.Minute*5) defer cancel() @@ -102,28 +130,57 @@ func (s *RemoteWriteProcessorSuite) TestProcessWithError() { err := rwProcessor.Process(ctx, rw) s.Require().ErrorIs(err, fakeErr) - s.Equal(expectedStatus.Code, http.StatusBadRequest) - s.Equal(expectedStatus.Message, fakeErr.Error()) + s.Equal(http.StatusBadRequest, actualStatus.Code) + s.Equal(fakeErr.Error(), actualStatus.Message) } -// -// testRemoteWrite -// +func (s *RemoteWriteProcessorSuite) TestProcessWithErrorGetStateByID() { + ar := &AdapterMock{ + AppendSnappyProtobufFunc: func(context.Context, pp_pkg_model.ProtobufData, *cppbridge.StateV2, bool) error { + return nil + }, + } -type testRemoteWrite struct { - metadata model.Metadata - readFn func(ctx context.Context) (*model.RemoteWriteBuffer, error) - writeFn func(ctx context.Context, status model.RemoteWriteProcessingStatus) error -} + metadata := model.Metadata{ + TenantID: "", + BlockID: uuid.New(), + ShardID: 0, + ShardsLog: 0, + SegmentEncodingVersion: 3, + RelabelerID: uuid.New().String(), + } -func (s *testRemoteWrite) Metadata() model.Metadata { - return s.metadata -} + states := &StatesStorageMock{ + GetStateByIDFunc: func(string) (*cppbridge.StateV2, bool) { + return nil, false + }, + } -func (s *testRemoteWrite) Read(ctx context.Context) (*model.RemoteWriteBuffer, error) { - return s.readFn(ctx) -} + var actualStatus model.RemoteWriteProcessingStatus + + rw := &RemoteWriteMock{ + MetadataFunc: func() model.Metadata { + return metadata + }, + + ReadFunc: func(context.Context) (*model.RemoteWriteBuffer, error) { + return &model.RemoteWriteBuffer{}, nil + }, + + WriteFunc: func(_ context.Context, status model.RemoteWriteProcessingStatus) error { + actualStatus = status + return nil + }, + } + + rwProcessor := processor.NewRemoteWriteProcessor(ar, states, nil) + + ctx, cancel := context.WithTimeout(context.Background(), time.Minute*5) + defer cancel() + + err := rwProcessor.Process(ctx, rw) + s.Require().ErrorIs(err, processor.ErrUnknownRelablerID) -func (s *testRemoteWrite) Write(ctx context.Context, status model.RemoteWriteProcessingStatus) error { - return s.writeFn(ctx, status) + s.Equal(http.StatusPreconditionFailed, actualStatus.Code) + s.Equal(processor.ErrUnknownRelablerID.Error(), actualStatus.Message) } diff --git a/pp-pkg/handler/processor/states_storage_moq_test.go b/pp-pkg/handler/processor/states_storage_moq_test.go new file mode 100644 index 0000000000..a0cb7696e0 --- /dev/null +++ b/pp-pkg/handler/processor/states_storage_moq_test.go @@ -0,0 +1,71 @@ +// Code generated by moq; DO NOT EDIT. +// github.com/matryer/moq + +package processor_test + +import ( + "github.com/prometheus/prometheus/pp/go/cppbridge" + "sync" +) + +// StatesStorageMock is a mock implementation of processor.StatesStorage. +// +// func TestSomethingThatUsesStatesStorage(t *testing.T) { +// +// // make and configure a mocked processor.StatesStorage +// mockedStatesStorage := &StatesStorageMock{ +// GetStateByIDFunc: func(stateID string) (*cppbridge.StateV2, bool) { +// panic("mock out the GetStateByID method") +// }, +// } +// +// // use mockedStatesStorage in code that requires processor.StatesStorage +// // and then make assertions. +// +// } +type StatesStorageMock struct { + // GetStateByIDFunc mocks the GetStateByID method. + GetStateByIDFunc func(stateID string) (*cppbridge.StateV2, bool) + + // calls tracks calls to the methods. + calls struct { + // GetStateByID holds details about calls to the GetStateByID method. + GetStateByID []struct { + // StateID is the stateID argument value. + StateID string + } + } + lockGetStateByID sync.RWMutex +} + +// GetStateByID calls GetStateByIDFunc. +func (mock *StatesStorageMock) GetStateByID(stateID string) (*cppbridge.StateV2, bool) { + if mock.GetStateByIDFunc == nil { + panic("StatesStorageMock.GetStateByIDFunc: method is nil but StatesStorage.GetStateByID was just called") + } + callInfo := struct { + StateID string + }{ + StateID: stateID, + } + mock.lockGetStateByID.Lock() + mock.calls.GetStateByID = append(mock.calls.GetStateByID, callInfo) + mock.lockGetStateByID.Unlock() + return mock.GetStateByIDFunc(stateID) +} + +// GetStateByIDCalls gets all the calls that were made to GetStateByID. +// Check the length with: +// +// len(mockedStatesStorage.GetStateByIDCalls()) +func (mock *StatesStorageMock) GetStateByIDCalls() []struct { + StateID string +} { + var calls []struct { + StateID string + } + mock.lockGetStateByID.RLock() + calls = mock.calls.GetStateByID + mock.lockGetStateByID.RUnlock() + return calls +} diff --git a/pp-pkg/handler/processor/stream_processor.go b/pp-pkg/handler/processor/stream_processor.go index 1b80ac4677..2bfa79a929 100644 --- a/pp-pkg/handler/processor/stream_processor.go +++ b/pp-pkg/handler/processor/stream_processor.go @@ -13,7 +13,8 @@ import ( type StreamProcessor struct { decoderBuilder DecoderBuilder - receiver Receiver + adapter Adapter + states StatesStorage criticalErrorCount *prometheus.CounterVec rejectedSegmentCount *prometheus.CounterVec @@ -26,14 +27,16 @@ type StreamProcessor struct { func NewStreamProcessor( decoderBuilder DecoderBuilder, - receiver Receiver, + adapter Adapter, + states StatesStorage, registerer prometheus.Registerer, ) *StreamProcessor { factory := util.NewUnconflictRegisterer(registerer) return &StreamProcessor{ decoderBuilder: decoderBuilder, - receiver: receiver, + adapter: adapter, + states: states, criticalErrorCount: factory.NewCounterVec(prometheus.CounterOpts{ Name: "remote_write_opprotocol_processor_critical_error_count", Help: "Total number of critical errors occurred during serving metric stream.", @@ -67,6 +70,16 @@ func NewStreamProcessor( func (p *StreamProcessor) Process(ctx context.Context, stream MetricStream) error { meta := stream.Metadata() + + state, ok := p.states.GetStateByID(meta.RelabelerID) + if !ok { + p.criticalErrorCount.With(prometheus.Labels{ + "error": ErrUnknownRelablerID.Error(), + "processor_type": "stream", + }).Inc() + return ErrUnknownRelablerID + } + decoder := p.decoderBuilder.Build(meta) defer func() { _ = decoder.Close() }() @@ -98,10 +111,10 @@ func (p *StreamProcessor) Process(ctx context.Context, stream MetricStream) erro Timestamp: hashdexContent.CreatedAt(), } - if err = p.receiver.AppendHashdex( + if err = p.adapter.AppendHashdex( ctx, hashdexContent.ShardedData(), - meta.RelabelerID, + state, AlwaysCommit, ); err != nil { processingStatus.Code = model.ProcessingStatusRejected diff --git a/pp-pkg/handler/processor/stream_processor_test.go b/pp-pkg/handler/processor/stream_processor_test.go index 16129c6246..7e3a9bb8d0 100644 --- a/pp-pkg/handler/processor/stream_processor_test.go +++ b/pp-pkg/handler/processor/stream_processor_test.go @@ -19,52 +19,9 @@ import ( "github.com/prometheus/prometheus/pp-pkg/handler/storage/block" "github.com/prometheus/prometheus/pp/go/cppbridge" coremodel "github.com/prometheus/prometheus/pp/go/model" - "github.com/prometheus/prometheus/pp/go/relabeler" "github.com/prometheus/prometheus/util/pool" ) -type testStream struct { - metadata model.Metadata - readFn func(ctx context.Context) (*model.Segment, error) - writeFn func(ctx context.Context, status model.StreamSegmentProcessingStatus) error -} - -func (s *testStream) Metadata() model.Metadata { - return s.metadata -} - -func (s *testStream) Read(ctx context.Context) (*model.Segment, error) { - return s.readFn(ctx) -} - -func (s *testStream) Write(ctx context.Context, status model.StreamSegmentProcessingStatus) error { - return s.writeFn(ctx, status) -} - -type metricReceiver struct { - appendFn func(ctx context.Context, hashdex cppbridge.ShardedData, relabelerID string) error -} - -func (mr *metricReceiver) AppendHashdex( - ctx context.Context, - hashdex cppbridge.ShardedData, - relabelerID string, - _ bool, -) error { - return mr.appendFn(ctx, hashdex, relabelerID) -} - -func (mr *metricReceiver) AppendSnappyProtobuf( - ctx context.Context, - data relabeler.ProtobufData, - relabelerID string, - _ bool, -) error { - return nil -} - -func (*metricReceiver) MergeOutOfOrderChunks(_ context.Context) {} - type segmentContainer struct { timeSeries []coremodel.TimeSeries encoded model.Segment @@ -129,33 +86,47 @@ func TestStreamProcessor_Process(t *testing.T) { buffers := pool.New(8, 100e3, 2, func(sz int) interface{} { return make([]byte, 0, sz) }) blockStorage := block.NewStorage(tmpDir, buffers) - mr := &metricReceiver{appendFn: func(ctx context.Context, hashdex cppbridge.ShardedData, relabelerID string) error { - return nil - }} + + ar := &AdapterMock{ + AppendHashdexFunc: func(context.Context, cppbridge.ShardedData, *cppbridge.StateV2, bool) error { + return nil + }, + } + + metadata := model.Metadata{ + TenantID: "", + BlockID: uuid.New(), + ShardID: 0, + ShardsLog: 0, + SegmentEncodingVersion: 3, + RelabelerID: uuid.New().String(), + } + + states := &StatesStorageMock{ + GetStateByIDFunc: func(stateID string) (*cppbridge.StateV2, bool) { + if metadata.RelabelerID != stateID { + return nil, false + } + + return nil, true + }, + } decoderBuilder := ppcore.NewBuilder(blockStorage) - streamProcessor := processor.NewStreamProcessor(decoderBuilder, mr, nil) + streamProcessor := processor.NewStreamProcessor(decoderBuilder, ar, states, nil) resolvec := make(chan struct{}, 1) - blockID := uuid.New() - shardID := uint16(0) - shardLog := uint8(0) - segmentEncodingVersion := cppbridge.EncodersVersion() gen := &segmentGenerator{segmentSize: 10} iteration := 0 fakeErr := errors.New("read error") - stream := &testStream{ - metadata: model.Metadata{ - TenantID: "", - BlockID: blockID, - ShardID: shardID, - ShardsLog: shardLog, - SegmentEncodingVersion: segmentEncodingVersion, + stream := &MetricStreamMock{ + MetadataFunc: func() model.Metadata { + return metadata }, - readFn: func(ctx context.Context) (*model.Segment, error) { + ReadFunc: func(context.Context) (*model.Segment, error) { resolvec <- struct{}{} if len(gen.segments) == 3 && iteration == 0 { @@ -167,7 +138,7 @@ func TestStreamProcessor_Process(t *testing.T) { segment, readErr := gen.generate() return &segment.encoded, readErr }, - writeFn: func(ctx context.Context, status model.StreamSegmentProcessingStatus) error { + WriteFunc: func(context.Context, model.StreamSegmentProcessingStatus) error { <-resolvec return nil }, @@ -179,15 +150,11 @@ func TestStreamProcessor_Process(t *testing.T) { err = streamProcessor.Process(ctx, stream) require.ErrorIs(t, err, fakeErr) - stream = &testStream{ - metadata: model.Metadata{ - TenantID: "", - BlockID: blockID, - ShardID: shardID, - ShardsLog: shardLog, - SegmentEncodingVersion: segmentEncodingVersion, + stream = &MetricStreamMock{ + MetadataFunc: func() model.Metadata { + return metadata }, - readFn: func(ctx context.Context) (*model.Segment, error) { + ReadFunc: func(context.Context) (*model.Segment, error) { resolvec <- struct{}{} if len(gen.segments) == 5 && iteration == 1 { @@ -199,7 +166,7 @@ func TestStreamProcessor_Process(t *testing.T) { segment, readErr := gen.generate() return &segment.encoded, readErr }, - writeFn: func(ctx context.Context, status model.StreamSegmentProcessingStatus) error { + WriteFunc: func(context.Context, model.StreamSegmentProcessingStatus) error { <-resolvec return nil }, @@ -208,15 +175,11 @@ func TestStreamProcessor_Process(t *testing.T) { err = streamProcessor.Process(ctx, stream) require.ErrorIs(t, err, fakeErr) - stream = &testStream{ - metadata: model.Metadata{ - TenantID: "", - BlockID: blockID, - ShardID: shardID, - ShardsLog: shardLog, - SegmentEncodingVersion: segmentEncodingVersion, + stream = &MetricStreamMock{ + MetadataFunc: func() model.Metadata { + return metadata }, - readFn: func(ctx context.Context) (*model.Segment, error) { + ReadFunc: func(context.Context) (*model.Segment, error) { resolvec <- struct{}{} if len(gen.segments) == 5 && iteration == 2 { @@ -236,7 +199,7 @@ func TestStreamProcessor_Process(t *testing.T) { segment, readErr := gen.generate() return &segment.encoded, readErr }, - writeFn: func(ctx context.Context, status model.StreamSegmentProcessingStatus) error { + WriteFunc: func(context.Context, model.StreamSegmentProcessingStatus) error { <-resolvec return nil }, diff --git a/pp-pkg/handler/states.go b/pp-pkg/handler/states.go new file mode 100644 index 0000000000..d47bc99d46 --- /dev/null +++ b/pp-pkg/handler/states.go @@ -0,0 +1,84 @@ +package handler + +import ( + "fmt" + "sync" + + "github.com/prometheus/prometheus/config" + "github.com/prometheus/prometheus/pp/go/cppbridge" +) + +// +// States +// + +// void empty struct. +var void = struct{}{} + +// StatesStorage stores the [cppbridge.State]'s. +type StatesStorage struct { + m map[string]*cppbridge.StateV2 + mx sync.RWMutex +} + +// NewStatesStorage init new [StatesStorage]. +func NewStatesStorage() *StatesStorage { + return &StatesStorage{ + m: map[string]*cppbridge.StateV2{config.TransparentRelabeler: cppbridge.NewTransitionStateV2()}, + mx: sync.RWMutex{}, + } +} + +// ApplyConfig updates the [StatesStorage]'s configs. +func (s *StatesStorage) ApplyConfig(conf *config.Config) error { + rwcfgs := conf.RemoteWriteReceiverConfig() + if len(rwcfgs.Configs) == 0 { + return nil + } + + updated := make(map[string]struct{}, len(rwcfgs.Configs)+1) + updated[config.TransparentRelabeler] = void + + s.mx.Lock() + defer s.mx.Unlock() + for _, cfg := range rwcfgs.Configs { + stateID := cfg.GetName() + rcfg := cfg.GetConfigs() + + if st, ok := s.m[stateID]; ok { + if st.StatelessRelabeler().EqualConfigs(rcfg) { + updated[stateID] = void + continue + } + } + + statelessRelabeler, err := cppbridge.NewStatelessRelabeler(rcfg) + if err != nil { + return fmt.Errorf("failed creating stateless relabeler for %s: %w", stateID, err) + } + + state := cppbridge.NewStateV2() + state.SetStatelessRelabeler(statelessRelabeler) + + s.m[stateID] = state + updated[stateID] = void + } + + for stateID := range s.m { + if _, ok := updated[stateID]; !ok { + // clear unnecessary + delete(s.m, stateID) + } + } + + return nil +} + +// GetStateByID returns [cppbridge.State] by state ID if exist. +func (s *StatesStorage) GetStateByID(stateID string) (*cppbridge.StateV2, bool) { + s.mx.RLock() + state, ok := s.m[stateID] + s.mx.RUnlock() + + return state, ok +} diff --git a/pp-pkg/handler/states_test.go b/pp-pkg/handler/states_test.go new file mode 100644 index 0000000000..977355d202 --- /dev/null +++ b/pp-pkg/handler/states_test.go @@ -0,0 +1,159 @@ +package handler_test + +import ( + "testing" + + "github.com/prometheus/prometheus/config" + pp_pkg_config "github.com/prometheus/prometheus/pp-pkg/config" + "github.com/prometheus/prometheus/pp-pkg/handler" + "github.com/prometheus/prometheus/pp/go/cppbridge" + rconfig "github.com/prometheus/prometheus/pp/go/relabeler/config" + "github.com/stretchr/testify/suite" +) + +type StatesStorageSuite struct { + suite.Suite +} + +func TestStatesStorageSuite(t *testing.T) { + suite.Run(t, new(StatesStorageSuite)) +} + +func (s *StatesStorageSuite) TestHappyPath() { + states := handler.NewStatesStorage() + + state, ok := states.GetStateByID(config.TransparentRelabeler) + + s.Require().True(ok) + s.Require().NotNil(state) + s.Require().True(state.IsTransition()) +} + +func (s *StatesStorageSuite) TestNotExist() { + states := handler.NewStatesStorage() + + state, ok := states.GetStateByID("test") + s.Require().False(ok) + s.Require().Nil(state) +} + +func (s *StatesStorageSuite) TestApplyConfigEmpty() { + states := handler.NewStatesStorage() + cfg := &config.Config{ + RemoteWriteConfigs: []*config.PPRemoteWriteConfig{}, + } + + states.ApplyConfig(cfg) + + state, ok := states.GetStateByID(config.TransparentRelabeler) + s.Require().True(ok) + s.Require().NotNil(state) + s.Require().True(state.IsTransition()) +} + +func (s *StatesStorageSuite) TestApplyConfig() { + states := handler.NewStatesStorage() + cfg := &config.Config{ + ReceiverConfig: pp_pkg_config.RemoteWriteReceiverConfig{ + Configs: []*rconfig.InputRelabelerConfig{ + { + Name: "test", + RelabelConfigs: []*cppbridge.RelabelConfig{}, + }, + }, + }, + } + + states.ApplyConfig(cfg) + + state, ok := states.GetStateByID(config.TransparentRelabeler) + s.Require().True(ok) + s.Require().NotNil(state) + s.Require().True(state.IsTransition()) + + state, ok = states.GetStateByID("test") + s.Require().True(ok) + s.Require().NotNil(state) + s.Require().False(state.IsTransition()) +} + +func (s *StatesStorageSuite) TestApplyConfigDouble() { + states := handler.NewStatesStorage() + cfg := &config.Config{ + ReceiverConfig: pp_pkg_config.RemoteWriteReceiverConfig{ + Configs: []*rconfig.InputRelabelerConfig{ + { + Name: "test", + RelabelConfigs: []*cppbridge.RelabelConfig{}, + }, + }, + }, + } + + states.ApplyConfig(cfg) + + state, ok := states.GetStateByID(config.TransparentRelabeler) + s.Require().True(ok) + s.Require().NotNil(state) + s.Require().True(state.IsTransition()) + + state, ok = states.GetStateByID("test") + s.Require().True(ok) + s.Require().NotNil(state) + s.Require().False(state.IsTransition()) + + states.ApplyConfig(cfg) + + state, ok = states.GetStateByID("test") + s.Require().True(ok) + s.Require().NotNil(state) + s.Require().False(state.IsTransition()) +} + +func (s *StatesStorageSuite) TestApplyConfigDoubleChange() { + states := handler.NewStatesStorage() + cfg := &config.Config{ + ReceiverConfig: pp_pkg_config.RemoteWriteReceiverConfig{ + Configs: []*rconfig.InputRelabelerConfig{ + { + Name: "test", + RelabelConfigs: []*cppbridge.RelabelConfig{}, + }, + }, + }, + } + + states.ApplyConfig(cfg) + + state, ok := states.GetStateByID(config.TransparentRelabeler) + s.Require().True(ok) + s.Require().NotNil(state) + s.Require().True(state.IsTransition()) + + state, ok = states.GetStateByID("test") + s.Require().True(ok) + s.Require().NotNil(state) + s.Require().False(state.IsTransition()) + + cfg = &config.Config{ + ReceiverConfig: pp_pkg_config.RemoteWriteReceiverConfig{ + Configs: []*rconfig.InputRelabelerConfig{ + { + Name: "test2", + RelabelConfigs: []*cppbridge.RelabelConfig{}, + }, + }, + }, + } + + states.ApplyConfig(cfg) + + state, ok = states.GetStateByID("test") + s.Require().False(ok) + s.Require().Nil(state) + + state, ok = states.GetStateByID("test2") + s.Require().True(ok) + s.Require().NotNil(state) + s.Require().False(state.IsTransition()) +} diff --git a/pp-pkg/logger/logger.go b/pp-pkg/logger/logger.go new file mode 100644 index 0000000000..6f7e020067 --- /dev/null +++ b/pp-pkg/logger/logger.go @@ -0,0 +1,33 @@ +package logger + +import ( + "fmt" + + "github.com/go-kit/log" + "github.com/go-kit/log/level" + + "github.com/prometheus/prometheus/pp/go/logger" +) + +const msg = "msg" + +// InitLogHandler init log handler for pp. +func InitLogHandler(l log.Logger) { + l = log.With(l, "pp_caller", log.Caller(4)) //revive:disable-line:add-constant // caller id + + logger.Debugf = func(template string, args ...any) { + _ = level.Debug(l).Log(msg, fmt.Sprintf(template, args...)) + } + + logger.Infof = func(template string, args ...any) { + _ = level.Info(l).Log(msg, fmt.Sprintf(template, args...)) + } + + logger.Warnf = func(template string, args ...any) { + _ = level.Warn(l).Log(msg, fmt.Sprintf(template, args...)) + } + + logger.Errorf = func(template string, args ...any) { + _ = level.Error(l).Log(msg, fmt.Sprintf(template, args...)) + } +} diff --git a/pp-pkg/model/interface.go b/pp-pkg/model/interface.go new file mode 100644 index 0000000000..6f8f1c845c --- /dev/null +++ b/pp-pkg/model/interface.go @@ -0,0 +1,23 @@ +package model + +import "github.com/prometheus/prometheus/pp/go/model" + +// +// ProtobufData +// + +// ProtobufData is an universal interface for blob protobuf data. +type ProtobufData interface { + Bytes() []byte + Destroy() +} + +// +// TimeSeriesBatch +// + +// TimeSeriesBatch is an universal interface for batch [model.TimeSeries]. +type TimeSeriesBatch interface { + TimeSeries() []model.TimeSeries + Destroy() +} diff --git a/pp-pkg/receiver/appender.go b/pp-pkg/receiver/appender.go deleted file mode 100644 index 9df2268890..0000000000 --- a/pp-pkg/receiver/appender.go +++ /dev/null @@ -1,83 +0,0 @@ -package receiver - -import ( - "context" - - "github.com/prometheus/prometheus/model/exemplar" - "github.com/prometheus/prometheus/model/histogram" - "github.com/prometheus/prometheus/model/labels" - "github.com/prometheus/prometheus/model/metadata" - "github.com/prometheus/prometheus/pp/go/model" - "github.com/prometheus/prometheus/storage" -) - -type timeSeriesData struct { - timeSeries []model.TimeSeries -} - -func (d *timeSeriesData) TimeSeries() []model.TimeSeries { - return d.timeSeries -} - -func (d *timeSeriesData) Destroy() { - d.timeSeries = nil -} - -type promAppender struct { - ctx context.Context - receiver *Receiver - relabelerID string - data *timeSeriesData -} - -func newPromAppender(ctx context.Context, receiver *Receiver, relabelerID string) *promAppender { - return &promAppender{ - ctx: ctx, - receiver: receiver, - relabelerID: relabelerID, - data: &timeSeriesData{}, - } -} - -func (a *promAppender) Append(ref storage.SeriesRef, l labels.Labels, t int64, v float64) (storage.SeriesRef, error) { - lsb := model.NewLabelSetBuilder() - l.Range(func(label labels.Label) { - lsb.Add(label.Name, label.Value) - }) - - a.data.timeSeries = append(a.data.timeSeries, model.TimeSeries{ - LabelSet: lsb.Build(), - Timestamp: uint64(t), - Value: v, - }) - return 0, nil -} - -func (a *promAppender) Commit() error { - if len(a.data.timeSeries) == 0 { - return nil - } - - _, err := a.receiver.AppendTimeSeries(a.ctx, a.data, nil, a.relabelerID, false) - return err -} - -func (a *promAppender) Rollback() error { - return nil -} - -func (a *promAppender) AppendExemplar(ref storage.SeriesRef, l labels.Labels, e exemplar.Exemplar) (storage.SeriesRef, error) { - return 0, nil -} - -func (a *promAppender) AppendHistogram(ref storage.SeriesRef, l labels.Labels, t int64, h *histogram.Histogram, fh *histogram.FloatHistogram) (storage.SeriesRef, error) { - return 0, nil -} - -func (a *promAppender) UpdateMetadata(ref storage.SeriesRef, l labels.Labels, m metadata.Metadata) (storage.SeriesRef, error) { - return 0, nil -} - -func (a *promAppender) AppendCTZeroSample(ref storage.SeriesRef, l labels.Labels, t, ct int64) (storage.SeriesRef, error) { - return 0, nil -} diff --git a/pp-pkg/receiver/receiver.go b/pp-pkg/receiver/receiver.go deleted file mode 100644 index a78630f239..0000000000 --- a/pp-pkg/receiver/receiver.go +++ /dev/null @@ -1,927 +0,0 @@ -// Copyright OpCore - -package receiver - -import ( - "context" - "errors" - "fmt" - "os" - "path" - "path/filepath" - "sync" - "time" - - "github.com/go-kit/log" - "github.com/go-kit/log/level" - "github.com/google/uuid" - "github.com/jonboulle/clockwork" - "github.com/prometheus/client_golang/prometheus" - common_config "github.com/prometheus/common/config" - "go.uber.org/atomic" - "gopkg.in/yaml.v2" - - prom_config "github.com/prometheus/prometheus/config" - "github.com/prometheus/prometheus/model/labels" - "github.com/prometheus/prometheus/model/relabel" - pp_pkg_config "github.com/prometheus/prometheus/pp-pkg/config" - "github.com/prometheus/prometheus/pp-pkg/dialer" - "github.com/prometheus/prometheus/pp/go/cppbridge" - "github.com/prometheus/prometheus/pp/go/relabeler" - "github.com/prometheus/prometheus/pp/go/relabeler/appender" - "github.com/prometheus/prometheus/pp/go/relabeler/block" - "github.com/prometheus/prometheus/pp/go/relabeler/config" - "github.com/prometheus/prometheus/pp/go/relabeler/distributor" - "github.com/prometheus/prometheus/pp/go/relabeler/head/catalog" - headmanager "github.com/prometheus/prometheus/pp/go/relabeler/head/manager" - "github.com/prometheus/prometheus/pp/go/relabeler/head/ready" - rlogger "github.com/prometheus/prometheus/pp/go/relabeler/logger" - "github.com/prometheus/prometheus/pp/go/relabeler/querier" - "github.com/prometheus/prometheus/pp/go/util" - "github.com/prometheus/prometheus/storage" - "github.com/prometheus/prometheus/util/annotations" -) - -const defaultShutdownTimeout = 40 * time.Second - -const ( - // status receiver is initial. - statusInitial uint32 = iota - - // status receiver is running. - statusRunning - - // status receiver is shutdowned. - statusShutdowned -) - -var DefaultNumberOfShards uint16 = 2 - -type HeadConfig struct { - inputRelabelerConfigs []*config.InputRelabelerConfig - numberOfShards uint16 -} - -type HeadConfigStorage struct { - ptr atomic.Pointer[HeadConfig] -} - -func (s *HeadConfigStorage) Load() *HeadConfig { - return s.ptr.Load() -} - -func (s *HeadConfigStorage) Get() ([]*config.InputRelabelerConfig, uint16) { - cfg := s.ptr.Load() - return cfg.inputRelabelerConfigs, cfg.numberOfShards -} - -func (s *HeadConfigStorage) Store(headConfig *HeadConfig) { - s.ptr.Store(headConfig) -} - -type Receiver struct { - ctx context.Context - - distributor *distributor.Distributor - appender *appender.QueryableAppender - storage *appender.QueryableStorage - rotator *appender.RotateCommiter - metricsWriteTrigger *appender.MetricsWriteTrigger - - headConfigStorage *HeadConfigStorage - hashdexFactory relabeler.HashdexFactory - hashdexLimits cppbridge.WALHashdexLimits - haTracker relabeler.HATracker - clock clockwork.Clock - registerer prometheus.Registerer - logger log.Logger - workingDir string - clientID string - cgogc *cppbridge.CGOGC - shutdowner *util.GracefulShutdowner - status atomic.Uint32 -} - -type RotationInfo struct { - BlockDuration time.Duration - Seed uint64 -} - -type HeadActivator struct { - catalog *catalog.Catalog -} - -func newHeadActivator(catalog *catalog.Catalog) *HeadActivator { - return &HeadActivator{catalog: catalog} -} - -func (ha *HeadActivator) Activate(headID string) error { - _, err := ha.catalog.SetStatus(headID, catalog.StatusActive) - return err -} - -func NewReceiver( - ctx context.Context, - logger log.Logger, - registerer prometheus.Registerer, - receiverCfg *pp_pkg_config.RemoteWriteReceiverConfig, - workingDir string, - remoteWriteCfgs []*prom_config.PPRemoteWriteConfig, - dataDir string, - rotationInfo RotationInfo, - headCatalog *catalog.Catalog, - triggerNotifier *ReloadBlocksTriggerNotifier, - readyNotifier ready.Notifier, - commitInterval time.Duration, - maxRetentionDuration time.Duration, - headRetentionTimeout time.Duration, - writeTimeout time.Duration, - maxSegmentSize uint32, - unloadDataStorage bool, -) (*Receiver, error) { - if logger == nil { - logger = log.NewNopLogger() - } - - clientID, err := readClientID(logger, workingDir) - if err != nil { - level.Error(logger).Log("msg", "failed read client id", "err", err) - return nil, err - } - - initLogHandler(logger) - clock := clockwork.NewRealClock() - - numberOfShards := receiverCfg.NumberOfShards - if numberOfShards == 0 { - numberOfShards = DefaultNumberOfShards - } - - destinationGroups, err := makeDestinationGroups( - ctx, - clock, - registerer, - workingDir, - clientID, - remoteWriteCfgs, - numberOfShards, - ) - if err != nil { - level.Error(logger).Log("msg", "failed to init DestinationGroups", "err", err) - return nil, err - } - - headConfigStorage := &HeadConfigStorage{} - - headConfigStorage.Store(&HeadConfig{ - inputRelabelerConfigs: receiverCfg.Configs, - numberOfShards: numberOfShards, - }) - - dataDir, err = filepath.Abs(dataDir) - if err != nil { - return nil, err - } - - var unloadDataStorageInterval time.Duration - if unloadDataStorage { - unloadDataStorageInterval = appender.DefaultMergeDuration - } - - headManager, err := headmanager.New( - dataDir, - clock, - headConfigStorage, - headCatalog, - maxSegmentSize, - registerer, - unloadDataStorageInterval, - ) - if err != nil { - return nil, fmt.Errorf("failed to create head manager: %w", err) - } - - activeHead, rotatedHeads, err := headManager.Restore(rotationInfo.BlockDuration, unloadDataStorageInterval) - if err != nil { - return nil, fmt.Errorf("failed to restore heads: %w", err) - } - readyNotifier.NotifyReady() - queryableStorage := appender.NewQueryableStorageWithWriteNotifier( - block.NewWriter(dataDir, block.DefaultChunkSegmentSize, rotationInfo.BlockDuration, registerer), - registerer, - querier.NewMetrics(registerer, querier.QueryableStorageSource), - triggerNotifier, - clock, - maxRetentionDuration, - headRetentionTimeout, - writeTimeout, - rotatedHeads..., - ) - - hd := appender.NewRotatableHead(activeHead, queryableStorage, headManager, newHeadActivator(headCatalog)) - - var appenderHead relabeler.Head = hd - if len(os.Getenv("OPCORE_ROTATION_HEAP_DEBUG")) > 0 { - heapProfileWriter := util.NewHeapProfileWriter(filepath.Join(dataDir, "heap_profiles")) - appenderHead = appender.NewHeapProfileWritableHead(appenderHead, heapProfileWriter) - } - - dstrb := distributor.NewDistributor(*destinationGroups) - app := appender.NewQueryableAppender( - ctx, - appenderHead, - dstrb, - querier.NewMetrics(registerer, querier.QueryableAppenderSource), - ) - mwt := appender.NewMetricsWriteTrigger(ctx, appender.DefaultMetricWriteInterval, app, queryableStorage) - - r := &Receiver{ - ctx: ctx, - distributor: dstrb, - appender: app, - storage: queryableStorage, - headConfigStorage: headConfigStorage, - rotator: appender.NewRotateCommiter( - ctx, - app, - relabeler.NewRotateTimerWithSeed(clock, rotationInfo.BlockDuration, rotationInfo.Seed), - appender.NewConstantIntervalTimer(clock, commitInterval), - appender.NewConstantIntervalTimer(clock, appender.DefaultMergeDuration), - unloadDataStorage, - registerer, - ), - - metricsWriteTrigger: mwt, - hashdexFactory: cppbridge.HashdexFactory{}, - hashdexLimits: cppbridge.DefaultWALHashdexLimits(), - haTracker: relabeler.NewHighAvailabilityTracker(ctx, registerer, clock), - clock: clock, - registerer: registerer, - logger: logger, - workingDir: workingDir, - clientID: clientID, - cgogc: cppbridge.NewCGOGC(registerer), - shutdowner: util.NewGracefulShutdowner(), - status: atomic.Uint32{}, - } - - level.Info(logger).Log("msg", "created") - - return r, nil -} - -// AppendHashdex append incoming Hashdex data to relabeling. -func (rr *Receiver) AppendHashdex( - ctx context.Context, - hashdex cppbridge.ShardedData, - relabelerID string, - commitToWal bool, -) error { - if rr.haTracker.IsDrop(hashdex.Cluster(), hashdex.Replica()) { - return nil - } - incomingData := &relabeler.IncomingData{Hashdex: hashdex} - _, err := rr.appender.Append(ctx, incomingData, nil, relabelerID, commitToWal) - return err -} - -// AppendSnappyProtobuf append compressed via snappy Protobuf data to relabeling hashdex data. -func (rr *Receiver) AppendSnappyProtobuf( - ctx context.Context, - compressedData relabeler.ProtobufData, - relabelerID string, - commitToWal bool, -) error { - hx, err := cppbridge.NewWALSnappyProtobufHashdex(compressedData.Bytes(), rr.hashdexLimits) - compressedData.Destroy() - if err != nil { - return err - } - - if rr.haTracker.IsDrop(hx.Cluster(), hx.Replica()) { - return nil - } - - incomingData := &relabeler.IncomingData{Hashdex: hx} - _, err = rr.appender.Append(ctx, incomingData, nil, relabelerID, commitToWal) - return err -} - -// AppendTimeSeries append TimeSeries data to relabeling hashdex data. -func (rr *Receiver) AppendTimeSeries( - ctx context.Context, - data relabeler.TimeSeriesData, - state *cppbridge.State, - relabelerID string, - commitToWal bool, -) (cppbridge.RelabelerStats, error) { - hx, err := rr.hashdexFactory.GoModel(data.TimeSeries(), rr.hashdexLimits) - if err != nil { - data.Destroy() - return cppbridge.RelabelerStats{}, err - } - - if rr.haTracker.IsDrop(hx.Cluster(), hx.Replica()) { - data.Destroy() - return cppbridge.RelabelerStats{}, nil - } - incomingData := &relabeler.IncomingData{Hashdex: hx, Data: data} - return rr.appender.AppendWithStaleNans( - ctx, - incomingData, - state, - relabelerID, - commitToWal, - ) -} - -func (rr *Receiver) AppendTimeSeriesHashdex( - ctx context.Context, - hashdex cppbridge.ShardedData, - state *cppbridge.State, - relabelerID string, - commitToWal bool, -) (cppbridge.RelabelerStats, error) { - return rr.appender.AppendWithStaleNans( - ctx, - &relabeler.IncomingData{Hashdex: hashdex}, - state, - relabelerID, - commitToWal, - ) -} - -// Appender create a new appender for head. -func (rr *Receiver) Appender(ctx context.Context) storage.Appender { - return newPromAppender(ctx, rr, prom_config.TransparentRelabeler) -} - -// ApplyConfig update config. -func (rr *Receiver) ApplyConfig(cfg *prom_config.Config) error { - level.Info(rr.logger).Log("msg", "reconfiguration start") - defer level.Info(rr.logger).Log("msg", "reconfiguration completed") - - rCfg, err := cfg.GetReceiverConfig() - if err != nil { - return err - } - - numberOfShards := rCfg.NumberOfShards - if numberOfShards == 0 { - numberOfShards = DefaultNumberOfShards - } - - rr.headConfigStorage.Store(&HeadConfig{ - inputRelabelerConfigs: rCfg.Configs, - numberOfShards: numberOfShards, - }) - - err = rr.appender.Reconfigure( - rr.ctx, - HeadConfigureFunc(func(head relabeler.Head) error { - return head.Reconfigure(rr.ctx, rCfg.Configs, numberOfShards) - }), - DistributorConfigureFunc(func(dstrb relabeler.Distributor) error { - mxdgupds := new(sync.Mutex) - dgupds, err := makeDestinationGroupUpdates( - cfg.RemoteWriteConfigs, - rr.workingDir, - rr.clientID, - numberOfShards, - ) - if err != nil { - level.Error(rr.logger).Log("msg", "failed to init destination group update", "err", err) - return err - } - mxDelete := new(sync.Mutex) - toDelete := []int{} - - dgs := dstrb.DestinationGroups() - if err = dgs.RangeGo(func(destinationGroupID int, dg *relabeler.DestinationGroup) error { - var rangeErr error - dgu, ok := dgupds[dg.Name()] - if !ok { - mxDelete.Lock() - toDelete = append(toDelete, destinationGroupID) - mxDelete.Unlock() - ctxShutdown, cancel := context.WithTimeout(rr.ctx, defaultShutdownTimeout) - if rangeErr = dg.Shutdown(ctxShutdown); err != nil { - level.Error(rr.logger).Log("msg", "failed shutdown DestinationGroup", "err", rangeErr) - } - cancel() - return nil - } - - if !dg.Equal(dgu.DestinationGroupConfig) || - !dg.EqualDialers(dgu.DialersConfigs) { - var dialers []relabeler.Dialer - if !dg.EqualDialers(dgu.DialersConfigs) { - dialers, rangeErr = makeDialers(rr.clock, rr.registerer, dgu.DialersConfigs) - if rangeErr != nil { - return rangeErr - } - } - - if rangeErr = dg.ResetTo(dgu.DestinationGroupConfig, dialers); err != nil { - return rangeErr - } - } - mxdgupds.Lock() - delete(dgupds, dg.Name()) - mxdgupds.Unlock() - return nil - }); err != nil { - level.Error(rr.logger).Log("msg", "failed to apply config DestinationGroups", "err", err) - return err - } - // delete unused DestinationGroup - dgs.RemoveByID(toDelete) - - // create new DestinationGroup - for _, dgupd := range dgupds { - dialers, err := makeDialers(rr.clock, rr.registerer, dgupd.DialersConfigs) - if err != nil { - level.Error(rr.logger).Log("msg", "failed to make new dialers", "err", err) - return err - } - - dg, err := relabeler.NewDestinationGroup( - rr.ctx, - dgupd.DestinationGroupConfig, - encoderSelector, - refillCtor, - refillSenderCtor, - rr.clock, - dialers, - rr.registerer, - ) - if err != nil { - level.Error(rr.logger).Log("msg", "failed to init DestinationGroup", "err", err) - return err - } - - dgs.Add(dg) - } - dstrb.SetDestinationGroups(dgs) - - return nil - }), - ) - if err != nil { - return err - } - - return nil -} - -// GetState create new state. -func (rr *Receiver) GetState() *cppbridge.State { - return cppbridge.NewState(rr.headConfigStorage.Load().numberOfShards) -} - -func (rr *Receiver) HeadQueryable() storage.Queryable { - return rr.appender -} - -func (rr *Receiver) HeadStatus(ctx context.Context, limit int) relabeler.HeadStatus { - return rr.appender.HeadStatus(ctx, limit) -} - -// LowestSentTimestamp returns the lowest sent timestamp across all queues. -func (*Receiver) LowestSentTimestamp() int64 { - return 0 -} - -// MergeOutOfOrderChunks merge chunks with out of order data chunks. -func (rr *Receiver) MergeOutOfOrderChunks(ctx context.Context) { - rr.appender.MergeOutOfOrderChunks(ctx) -} - -// Querier calls f() with the given parameters. -// Returns a querier.MultiQuerier combining of appenderQuerier and storageQuerier. -func (rr *Receiver) Querier(mint, maxt int64) (storage.Querier, error) { - appenderQuerier, err := rr.appender.Querier(mint, maxt) - if err != nil { - return nil, err - } - - storageQuerier, err := rr.storage.Querier(mint, maxt) - if err != nil { - return nil, errors.Join(err, appenderQuerier.Close()) - } - - return querier.NewMultiQuerier([]storage.Querier{appenderQuerier, storageQuerier}, nil), nil -} - -func (rr *Receiver) ChunkQuerier(mint, maxt int64) (storage.ChunkQuerier, error) { - appenderQuerier, err := rr.appender.ChunkQuerier(mint, maxt) - if err != nil { - return nil, err - } - - storageQuerier, err := rr.storage.ChunkQuerier(mint, maxt) - if err != nil { - return nil, errors.Join(err, appenderQuerier.Close()) - } - - return storage.NewMergeChunkQuerier( - nil, - []storage.ChunkQuerier{appenderQuerier, storageQuerier}, - storage.NewConcatenatingChunkSeriesMerger(), - ), nil -} - -// RelabelerIDIsExist check on exist relabelerID. -func (rr *Receiver) RelabelerIDIsExist(relabelerID string) bool { - cs := rr.headConfigStorage.Load() - for _, cfg := range cs.inputRelabelerConfigs { - if cfg.Name == relabelerID { - return true - } - } - - return false -} - -// Run main loop. -func (rr *Receiver) Run(context.Context) (err error) { - // fast exit if receiver is already running or shutdowned. - if !rr.status.CompareAndSwap(statusInitial, statusRunning) { - return nil - } - - defer rr.shutdowner.Done(err) - rr.storage.Run() - rr.rotator.Run() - <-rr.shutdowner.Signal() - return nil -} - -// Shutdown safe shutdown Receiver. -func (rr *Receiver) Shutdown(ctx context.Context) error { - // exit if receiver is not running. - if !rr.status.CompareAndSwap(statusRunning, statusShutdowned) { - return errors.Join( - rr.cgogc.Shutdown(ctx), - rr.metricsWriteTrigger.Close(), - rr.distributor.Shutdown(ctx), - rr.appender.Close(ctx), - ) - } - - return errors.Join( - rr.cgogc.Shutdown(ctx), - rr.metricsWriteTrigger.Close(), - rr.rotator.Close(), - rr.storage.Close(), - rr.distributor.Shutdown(ctx), - rr.appender.Close(ctx), - rr.shutdowner.Shutdown(ctx), - ) -} - -// makeDestinationGroups create DestinationGroups from configs. -func makeDestinationGroups( - ctx context.Context, - clock clockwork.Clock, - registerer prometheus.Registerer, - workingDir, clientID string, - rwCfgs []*prom_config.PPRemoteWriteConfig, - numberOfShards uint16, -) (*relabeler.DestinationGroups, error) { - dgs := make(relabeler.DestinationGroups, 0, len(rwCfgs)) - - for _, rwCfg := range rwCfgs { - if rwCfg.IsPrometheusProtocol() { - continue - } - - dgCfg, err := convertingDestinationGroupConfig(rwCfg, workingDir, numberOfShards) - if err != nil { - return nil, err - } - - dialersConfigs, err := convertingConfigDialers(clientID, rwCfg.Destinations) - if err != nil { - return nil, err - } - dialers, err := makeDialers(clock, registerer, dialersConfigs) - if err != nil { - return nil, err - } - - dg, err := relabeler.NewDestinationGroup( - ctx, - dgCfg, - encoderSelector, - refillCtor, - refillSenderCtor, - clock, - dialers, - registerer, - ) - if err != nil { - return nil, err - } - - dgs = append(dgs, dg) - } - - return &dgs, nil -} - -// makeDestinationGroupUpdates create update for DestinationGroups. -func makeDestinationGroupUpdates( - rwCfgs []*prom_config.PPRemoteWriteConfig, - workingDir, clientID string, - numberOfShards uint16, -) (map[string]*relabeler.DestinationGroupUpdate, error) { - dgus := make(map[string]*relabeler.DestinationGroupUpdate, len(rwCfgs)) - - for _, rwCfg := range rwCfgs { - if rwCfg.IsPrometheusProtocol() { - continue - } - - dgCfg, err := convertingDestinationGroupConfig(rwCfg, workingDir, numberOfShards) - if err != nil { - return nil, err - } - - dialersConfigs, err := convertingConfigDialers(clientID, rwCfg.Destinations) - if err != nil { - return nil, err - } - - dgus[rwCfg.Name] = &relabeler.DestinationGroupUpdate{ - DestinationGroupConfig: dgCfg, - DialersConfigs: dialersConfigs, - } - } - - return dgus, nil -} - -// convertingDestinationGroupConfig converting incoming config to internal DestinationGroupConfig. -func convertingDestinationGroupConfig( - rwCfg *prom_config.PPRemoteWriteConfig, - workingDir string, - numberOfShards uint16, -) (*relabeler.DestinationGroupConfig, error) { - rCfgs, err := convertingRelabelersConfig(rwCfg.WriteRelabelConfigs) - if err != nil { - return nil, err - } - - dgcfg := relabeler.NewDestinationGroupConfig( - rwCfg.Name, - workingDir, - rCfgs, - numberOfShards, - ) - - return dgcfg, nil -} - -// convertingRelabelersConfig converting incoming relabel config to internal relabel config. -func convertingRelabelersConfig(rCfgs []*relabel.Config) ([]*cppbridge.RelabelConfig, error) { - var crCfgs []*cppbridge.RelabelConfig - raw, err := yaml.Marshal(rCfgs) - if err != nil { - return nil, err - } - - if err = yaml.Unmarshal(raw, &crCfgs); err != nil { - return nil, err - } - - return crCfgs, nil -} - -// convertingConfigDialers converting and make internal dialer configs. -func convertingConfigDialers( - clientID string, - sCfgs []*prom_config.PPDestinationConfig, -) ([]*relabeler.DialersConfig, error) { - dialersConfigs := make([]*relabeler.DialersConfig, 0, len(sCfgs)) - for _, sCfg := range sCfgs { - tlsCfg, err := common_config.NewTLSConfig(&sCfg.HTTPClientConfig.TLSConfig) - if err != nil { - return nil, err - } - - ccfg, err := dialer.NewCommonConfig( - sCfg.URL.URL, - tlsCfg, - sCfg.Name, - ) - if err != nil { - return nil, err - } - - dialersConfigs = append( - dialersConfigs, - &relabeler.DialersConfig{ - DialerConfig: relabeler.NewDialerConfig( - sCfg.URL.URL, - clientID, - extractAccessToken(sCfg.HTTPClientConfig.Authorization), - ), - ConnDialerConfig: ccfg, - }, - ) - } - - return dialersConfigs, nil -} - -// extractAccessToken extract access token from Authorization config. -func extractAccessToken(authorization *common_config.Authorization) string { - if authorization == nil { - return "" - } - - return string(authorization.Credentials) -} - -// makeDialers create dialers from main config according to the specified parameters. -func makeDialers( - clock clockwork.Clock, - registerer prometheus.Registerer, - dialersConfig []*relabeler.DialersConfig, -) ([]relabeler.Dialer, error) { - dialers := make([]relabeler.Dialer, 0, len(dialersConfig)) - for i := range dialersConfig { - ccfg, ok := dialersConfig[i].ConnDialerConfig.(*dialer.CommonConfig) - if !ok { - return nil, fmt.Errorf("invalid dialer CommonConfig: %v", dialersConfig[i].ConnDialerConfig) - } - - d, err := dialer.DefaultDialer(ccfg, registerer) - if err != nil { - return nil, err - } - - tcpDialer := relabeler.NewWebSocketDialer( - d, - dialersConfig[i].DialerConfig, - clock, - registerer, - ) - dialers = append(dialers, tcpDialer) - } - - return dialers, nil -} - -// encoderSelector selector for constructors for encoders. -func encoderSelector(isShrinkable bool) relabeler.ManagerEncoderCtor { - if isShrinkable { - return func(shardID uint16, shardsNumberPower uint8) relabeler.ManagerEncoder { - return cppbridge.NewWALEncoderLightweight(shardID, shardsNumberPower) - } - } - - return func(shardID uint16, shardsNumberPower uint8) relabeler.ManagerEncoder { - return cppbridge.NewWALEncoder(shardID, shardsNumberPower) - } -} - -// refillCtor default contructor for refill. -func refillCtor( - workinDir string, - blockID uuid.UUID, - destinations []string, - shardsNumberPower uint8, - segmentEncodingVersion uint8, - alwaysToRefill bool, - name string, - registerer prometheus.Registerer, -) (relabeler.ManagerRefill, error) { - return relabeler.NewRefill( - workinDir, - shardsNumberPower, - segmentEncodingVersion, - blockID, - alwaysToRefill, - name, - registerer, - destinations..., - ) -} - -// refillSenderCtor default contructor for manager sender. -func refillSenderCtor( - rsmCfg relabeler.RefillSendManagerConfig, - workingDir string, - dialers []relabeler.Dialer, - clock clockwork.Clock, - name string, - registerer prometheus.Registerer, -) (relabeler.ManagerRefillSender, error) { - return relabeler.NewRefillSendManager(rsmCfg, workingDir, dialers, clock, name, registerer) -} - -// initLogHandler init log handler for ManagerKeeper. -func initLogHandler(logger log.Logger) { - logger = log.With(logger, "pp_caller", log.Caller(4)) - rlogger.Debugf = func(template string, args ...any) { - level.Debug(logger).Log("msg", fmt.Sprintf(template, args...)) - } - rlogger.Infof = func(template string, args ...any) { - level.Info(logger).Log("msg", fmt.Sprintf(template, args...)) - } - rlogger.Warnf = func(template string, args ...any) { - level.Warn(logger).Log("msg", fmt.Sprintf(template, args...)) - } - rlogger.Errorf = func(template string, args ...any) { - level.Error(logger).Log("msg", fmt.Sprintf(template, args...)) - } -} - -// readClientID read ClientID. -func readClientID(logger log.Logger, dir string) (string, error) { - if err := os.MkdirAll(dir, 0o700); err != nil { - return "", fmt.Errorf("mkdir %s: %w", filepath.Dir(dir), err) - } - clientIDPath := path.Join(dir, "client_id.uuid") - // Try reading UUID from the file. If not present, generate new one and write to file - data, err := os.ReadFile(clientIDPath) - switch { - case os.IsNotExist(err): - proxyUUID := uuid.NewString() - //revive:disable-next-line:add-constant file permissions simple readable as octa-number - if err = os.WriteFile(clientIDPath, []byte(proxyUUID), 0o644); err != nil { // #nosec G306 - return "", fmt.Errorf("failed to write proxy id: %w", err) - } - - level.Info(logger).Log("msg", "create new client id") - return proxyUUID, nil - - case err == nil: - //revive:disable-next-line:add-constant uuid len - if len(data) < 36 { - return "", fmt.Errorf("short client id: %d", len(data)) - } - - return string(data[:36]), nil - - default: - return "", fmt.Errorf("failed to read client id: %w", err) - } -} - -// -// NoopQuerier -// - -type NoopQuerier struct{} - -var _ storage.Querier = (*NoopQuerier)(nil) - -func (*NoopQuerier) Select(_ context.Context, _ bool, _ *storage.SelectHints, _ ...*labels.Matcher) storage.SeriesSet { - return &NoopSeriesSet{} -} - -func (q *NoopQuerier) LabelValues( - ctx context.Context, - name string, - hints *storage.LabelHints, - matchers ...*labels.Matcher, -) ([]string, annotations.Annotations, error) { - return []string{}, *annotations.New(), nil -} - -func (q *NoopQuerier) LabelNames( - ctx context.Context, - hints *storage.LabelHints, - matchers ...*labels.Matcher, -) ([]string, annotations.Annotations, error) { - return []string{}, *annotations.New(), nil -} - -func (*NoopQuerier) Close() error { - return nil -} - -// -// NoopSeriesSet -// - -type NoopSeriesSet struct{} - -func (*NoopSeriesSet) Next() bool { - return false -} - -func (*NoopSeriesSet) At() storage.Series { - return nil -} - -func (*NoopSeriesSet) Err() error { - return nil -} - -func (*NoopSeriesSet) Warnings() annotations.Annotations { - return nil -} diff --git a/pp-pkg/remote/config.go b/pp-pkg/remote/config.go index 764d74c2b8..2ee2223362 100644 --- a/pp-pkg/remote/config.go +++ b/pp-pkg/remote/config.go @@ -8,7 +8,7 @@ import ( "gopkg.in/yaml.v2" "github.com/prometheus/prometheus/config" - "github.com/prometheus/prometheus/pp/go/relabeler/remotewriter" + "github.com/prometheus/prometheus/pp/go/storage/remotewriter" ) func ApplyConfig(remoteWriter *remotewriter.RemoteWriter) func(promConfig *config.Config) error { diff --git a/pp-pkg/scrape/manager.go b/pp-pkg/scrape/manager.go index 64e994376c..376fdea297 100644 --- a/pp-pkg/scrape/manager.go +++ b/pp-pkg/scrape/manager.go @@ -18,31 +18,29 @@ import ( "github.com/prometheus/prometheus/config" "github.com/prometheus/prometheus/discovery/targetgroup" "github.com/prometheus/prometheus/model/labels" + pp_pkg_model "github.com/prometheus/prometheus/pp-pkg/model" "github.com/prometheus/prometheus/pp/go/cppbridge" - "github.com/prometheus/prometheus/pp/go/relabeler" "github.com/prometheus/prometheus/util/osutil" "github.com/prometheus/prometheus/util/pool" ) -type Receiver interface { - // AppendTimeSeries append TimeSeries data to relabeling hashdex data. +// Adapter for implementing the [Queryable] interface and append data. +type Adapter interface { + // AppendTimeSeries append TimeSeries data to [Head]. AppendTimeSeries( ctx context.Context, - data relabeler.TimeSeriesData, - state *cppbridge.State, - relabelerID string, + data pp_pkg_model.TimeSeriesBatch, + state *cppbridge.StateV2, commitToWal bool, ) (cppbridge.RelabelerStats, error) - // AppendTimeSeries append TimeSeries data to relabeling hashdex data. - AppendTimeSeriesHashdex( + + // AppendScraperHashdex append ScraperHashdex data to [Head]. + AppendScraperHashdex( ctx context.Context, hashdex cppbridge.ShardedData, - state *cppbridge.State, - relabelerID string, + state *cppbridge.StateV2, commitToWal bool, ) (cppbridge.RelabelerStats, error) - RelabelerIDIsExist(relabelerID string) bool - GetState() *cppbridge.State } // Options are the configuration parameters to the scrape manager. @@ -79,7 +77,7 @@ const DefaultNameEscapingScheme = model.ValueEncodingEscaping type Manager struct { opts *Options logger log.Logger - receiver Receiver + adapter Adapter graceShut chan struct{} offsetSeed uint64 // Global offsetSeed seed is used to spread scrape workload across HA setup. @@ -93,6 +91,8 @@ type Manager struct { triggerReload chan struct{} + reportStatelessRelabeler *cppbridge.StatelessRelabeler + metrics *scrapeMetrics } @@ -100,7 +100,7 @@ type Manager struct { func NewManager( o *Options, logger log.Logger, - receiver Receiver, + adapter Adapter, registerer prometheus.Registerer, ) (*Manager, error) { if o == nil { @@ -115,18 +115,24 @@ func NewManager( return nil, fmt.Errorf("failed to create scrape manager due to error: %w", err) } + reportStatelessRelabeler, err := cppbridge.NewStatelessRelabeler([]*cppbridge.RelabelConfig{}) + if err != nil { + return nil, fmt.Errorf("failed creating report stateless relabeler: %w", err) + } + m := &Manager{ - receiver: receiver, - opts: o, - logger: logger, - scrapeConfigs: make(map[string]*config.ScrapeConfig), - scrapePools: make(map[string]*scrapePool), - graceShut: make(chan struct{}), - triggerReload: make(chan struct{}, 1), - metrics: sm, - buffers: pool.New(1e3, 100e6, 2, func(sz int) interface{} { return make([]byte, 0, sz) }), - bufferBuilders: newBuildersPool(), - bufferBatches: newbatchesPool(), + adapter: adapter, + opts: o, + logger: logger, + scrapeConfigs: make(map[string]*config.ScrapeConfig), + scrapePools: make(map[string]*scrapePool), + graceShut: make(chan struct{}), + triggerReload: make(chan struct{}, 1), + metrics: sm, + buffers: pool.New(1e3, 100e6, 2, func(sz int) interface{} { return make([]byte, 0, sz) }), + bufferBuilders: newBuildersPool(), + bufferBatches: newbatchesPool(), + reportStatelessRelabeler: reportStatelessRelabeler, } m.metrics.setTargetMetadataCacheGatherer(m) @@ -197,7 +203,8 @@ func (m *Manager) reload() { m.metrics.targetScrapePools.Inc() sp, err := newScrapePool( scrapeConfig, - m.receiver, + m.adapter, + m.reportStatelessRelabeler, m.offsetSeed, log.With(m.logger, "scrape_pool", setName), m.buffers, diff --git a/pp-pkg/scrape/scrape.go b/pp-pkg/scrape/scrape.go index da23a2a5d8..ada6f9cc21 100644 --- a/pp-pkg/scrape/scrape.go +++ b/pp-pkg/scrape/scrape.go @@ -100,7 +100,7 @@ type ScraperHashdex interface { Parse(buffer []byte, defaultTimestamp int64) (uint32, error) // RangeMetadata calls f sequentially for each metadata present in the hashdex. // If f returns false, range stops the iteration. - RangeMetadata(f func(metadata cppbridge.WALScraperHashdexMetadata) bool) + RangeMetadata(func(cppbridge.WALScraperHashdexMetadata) bool) // Cluster get Cluster name. Cluster() string // Replica get Replica name. @@ -111,24 +111,26 @@ type scrapeLoopOptions struct { target *Target scraper scraper metricLimits *cppbridge.MetricLimits + statelessRelabeler *cppbridge.StatelessRelabeler + cache *scrapeCache + mrc []*relabel.Config + interval time.Duration + timeout time.Duration + validationScheme model.ValidationScheme honorLabels bool honorTimestamps bool trackTimestampsStaleness bool - interval time.Duration - timeout time.Duration scrapeClassicHistograms bool - validationScheme model.ValidationScheme - mrc []*relabel.Config - cache *scrapeCache enableCompression bool } // scrapePool manages scrapes for sets of targets. type scrapePool struct { - receiver Receiver - logger log.Logger - cancel context.CancelFunc - httpOpts []config_util.HTTPClientOption + statelessRelabeler *cppbridge.StatelessRelabeler + reportStatelessRelabeler *cppbridge.StatelessRelabeler + logger log.Logger + cancel context.CancelFunc + httpOpts []config_util.HTTPClientOption // mtx must not be taken after targetMtx. mtx sync.Mutex @@ -154,7 +156,8 @@ type scrapePool struct { func newScrapePool( cfg *config.ScrapeConfig, - receiver Receiver, + adapter Adapter, + reportStatelessRelabeler *cppbridge.StatelessRelabeler, offsetSeed uint64, logger log.Logger, buffers *pool.Pool, @@ -163,11 +166,6 @@ func newScrapePool( options *Options, metrics *scrapeMetrics, ) (*scrapePool, error) { - scrapeName := config.ScrapePrefix + cfg.JobName - if !receiver.RelabelerIDIsExist(scrapeName) { - return nil, fmt.Errorf("relabeler id not found for scrape name: %s", scrapeName) - } - if logger == nil { logger = log.NewNopLogger() } @@ -177,21 +175,40 @@ func newScrapePool( return nil, fmt.Errorf("error creating HTTP client: %w", err) } + if reportStatelessRelabeler == nil { + reportStatelessRelabeler, err = cppbridge.NewStatelessRelabeler([]*cppbridge.RelabelConfig{}) + if err != nil { + return nil, fmt.Errorf("failed creating report stateless relabeler: %w", err) + } + } + + rcfgs, err := cfg.PPMetricRelabelConfigs() + if err != nil { + return nil, fmt.Errorf("failed get pp relabel configs: %w", err) + } + + statelessRelabeler, err := cppbridge.NewStatelessRelabeler(rcfgs) + if err != nil { + return nil, fmt.Errorf("failed creating stateless relabeler: %w", err) + } + ctx, cancel := context.WithCancel(context.Background()) sp := &scrapePool{ - cancel: cancel, - receiver: receiver, - config: cfg, - client: client, - activeTargets: map[uint64]*Target{}, - targetsCache: map[uint64][]*Target{}, - loops: map[uint64]loop{}, - symbolTable: labels.NewSymbolTable(), - logger: logger, - metrics: metrics, - httpOpts: options.HTTPClientOptions, - noDefaultPort: options.NoDefaultPort, + statelessRelabeler: statelessRelabeler, + reportStatelessRelabeler: reportStatelessRelabeler, + logger: logger, + cancel: cancel, + config: cfg, + client: client, + activeTargets: map[uint64]*Target{}, + targetsCache: map[uint64][]*Target{}, + loops: map[uint64]loop{}, + symbolTable: labels.NewSymbolTable(), + metrics: metrics, + httpOpts: options.HTTPClientOptions, + noDefaultPort: options.NoDefaultPort, } + sp.newLoop = func(opts scrapeLoopOptions) loop { // Update the targets retrieval function for metadata to a new scrape cache. cache := opts.cache @@ -227,8 +244,10 @@ func newScrapePool( return newScrapeLoop( ctx, opts.scraper, - receiver, + adapter, log.With(logger, "target", opts.target), + opts.statelessRelabeler, + sp.reportStatelessRelabeler, targetOptions, buffers, bufferBuilders, @@ -334,6 +353,17 @@ func (sp *scrapePool) reload(cfg *config.ScrapeConfig) error { oldClient := sp.client sp.client = client + rcfgs, err := cfg.PPMetricRelabelConfigs() + if err != nil { + return fmt.Errorf("failed get pp relabel configs: %w", err) + } + + statelessRelabeler, err := cppbridge.NewStatelessRelabeler(rcfgs) + if err != nil { + return fmt.Errorf("failed creating stateless relabeler: %w", err) + } + sp.statelessRelabeler = statelessRelabeler + sp.metrics.targetScrapePoolTargetLimit.WithLabelValues(sp.config.JobName).Set(float64(sp.config.TargetLimit)) sp.restartLoops(reuseCache) @@ -363,6 +393,7 @@ func (sp *scrapePool) restartLoops(reuseCache bool) { enableCompression = sp.config.EnableCompression trackTimestampsStaleness = sp.config.TrackTimestampsStaleness mrc = sp.config.MetricRelabelConfigs + statelessRelabeler = sp.statelessRelabeler ) validationScheme := model.LegacyValidation @@ -397,6 +428,7 @@ func (sp *scrapePool) restartLoops(reuseCache bool) { target: t, scraper: s, metricLimits: metricLimits, + statelessRelabeler: statelessRelabeler, honorLabels: honorLabels, honorTimestamps: honorTimestamps, enableCompression: enableCompression, @@ -507,6 +539,7 @@ func (sp *scrapePool) sync(targets []*Target) { trackTimestampsStaleness = sp.config.TrackTimestampsStaleness mrc = sp.config.MetricRelabelConfigs scrapeClassicHistograms = sp.config.ScrapeClassicHistograms + statelessRelabeler = sp.statelessRelabeler ) validationScheme := model.LegacyValidation @@ -537,6 +570,7 @@ func (sp *scrapePool) sync(targets []*Target) { target: t, scraper: s, metricLimits: metricLimits, + statelessRelabeler: statelessRelabeler, honorLabels: honorLabels, honorTimestamps: honorTimestamps, enableCompression: enableCompression, @@ -755,10 +789,10 @@ type loop interface { type scrapeLoop struct { scraper scraper - receiver Receiver + adapter Adapter logger log.Logger - state *cppbridge.State - reportState *cppbridge.State + state *cppbridge.StateV2 + reportState *cppbridge.StateV2 cache *scrapeCache buffers *pool.Pool bufferBuilders *buildersPool @@ -796,8 +830,10 @@ type scrapeLoop struct { func newScrapeLoop( ctx context.Context, sc scraper, - receiver Receiver, + adapter Adapter, logger log.Logger, + statelessRelabeler *cppbridge.StatelessRelabeler, + reportStatelessRelabeler *cppbridge.StatelessRelabeler, options *cppbridge.RelabelerOptions, buffers *pool.Pool, bufferBuilders *buildersPool, @@ -840,11 +876,13 @@ func newScrapeLoop( appenderCtx = ContextWithTarget(appenderCtx, target) } - state := receiver.GetState() + state := cppbridge.NewStateV2WithoutLock() + state.SetStatelessRelabeler(statelessRelabeler) state.EnableTrackStaleness() state.SetRelabelerOptions(options) - reportState := receiver.GetState() + reportState := cppbridge.NewStateV2WithoutLock() + reportState.SetStatelessRelabeler(reportStatelessRelabeler) reportState.SetRelabelerOptions(&cppbridge.RelabelerOptions{ TargetLabels: options.TargetLabels, HonorTimestamps: true, @@ -852,7 +890,7 @@ func newScrapeLoop( sl := &scrapeLoop{ scraper: sc, - receiver: receiver, + adapter: adapter, logger: logger, state: state, reportState: reportState, @@ -1097,11 +1135,10 @@ func (sl *scrapeLoop) endOfRunStaleness(last time.Time, ticker *time.Ticker, int // sl.context would have been cancelled, hence using sl.appenderCtx. emptyBatch := sl.bufferBatches.get() sl.state.SetDefTimestamp(timestamp.FromTime(staleTime)) - if _, err := sl.receiver.AppendTimeSeries( + if _, err := sl.adapter.AppendTimeSeries( sl.appenderCtx, emptyBatch, sl.state, - sl.scrapeName, CommitToWalOnAppend, ); err != nil { level.Warn(sl.logger).Log("msg", "Stale append failed", "err", err) @@ -1127,7 +1164,11 @@ func (sl *scrapeLoop) getCache() *scrapeCache { return sl.cache } -// TODO Delete. +// append via prometheus parser. +// +//revive:disable-next-line:cognitive-complexity // legacy code +//revive:disable-next-line:function-length // legacy code +//revive:disable-next-line:cyclomatic // legacy code func (sl *scrapeLoop) append( b []byte, contentType string, @@ -1136,11 +1177,10 @@ func (sl *scrapeLoop) append( // if input is empty for stalenan if len(b) == 0 { sl.state.SetDefTimestamp(timestamp.FromTime(ts)) - _, err = sl.receiver.AppendTimeSeries( + _, err = sl.adapter.AppendTimeSeries( sl.appenderCtx, sl.bufferBatches.get(), sl.state, - sl.scrapeName, CommitToWalOnAppend, ) return 0, stats, err @@ -1235,11 +1275,10 @@ loop: } sl.state.SetDefTimestamp(defTime) - stats, err = sl.receiver.AppendTimeSeries( + stats, err = sl.adapter.AppendTimeSeries( sl.appenderCtx, batch, sl.state, - sl.scrapeName, CommitToWalOnAppend, ) if err != nil { @@ -1289,11 +1328,10 @@ func (sl *scrapeLoop) appendCpp( } sl.state.SetDefTimestamp(timestamp.FromTime(ts)) - stats, err = sl.receiver.AppendTimeSeriesHashdex( + stats, err = sl.adapter.AppendScraperHashdex( sl.appenderCtx, hashdex, sl.state, - sl.scrapeName, CommitToWalOnAppend, ) if err != nil { @@ -1442,11 +1480,10 @@ func (sl *scrapeLoop) report( } } - if _, err = sl.receiver.AppendTimeSeries( + if _, err = sl.adapter.AppendTimeSeries( sl.appenderCtx, batch, sl.reportState, - config.TransparentRelabeler, CommitToWalOnAppend, ); err != nil { return @@ -1491,11 +1528,10 @@ func (sl *scrapeLoop) reportStale(start time.Time) (err error) { } } - if _, err = sl.receiver.AppendTimeSeries( + if _, err = sl.adapter.AppendTimeSeries( sl.appenderCtx, batch, sl.reportState, - config.TransparentRelabeler, CommitToWalOnAppend, ); err != nil { return diff --git a/pp-pkg/storage/adapter.go b/pp-pkg/storage/adapter.go new file mode 100644 index 0000000000..261a0430a9 --- /dev/null +++ b/pp-pkg/storage/adapter.go @@ -0,0 +1,295 @@ +package storage + +import ( + "context" + "math" + "time" + + "github.com/jonboulle/clockwork" + "github.com/prometheus/client_golang/prometheus" + + "github.com/prometheus/prometheus/pp-pkg/model" + "github.com/prometheus/prometheus/pp/go/cppbridge" + "github.com/prometheus/prometheus/pp/go/hatracker" + pp_storage "github.com/prometheus/prometheus/pp/go/storage" + "github.com/prometheus/prometheus/pp/go/storage/appender" + "github.com/prometheus/prometheus/pp/go/storage/head/services" + "github.com/prometheus/prometheus/pp/go/storage/querier" + "github.com/prometheus/prometheus/pp/go/util" + "github.com/prometheus/prometheus/storage" +) + +// +// Adapter +// + +var _ storage.Storage = (*Adapter)(nil) + +// Adapter for implementing the [Queryable] interface and append data. +type Adapter struct { + proxy *pp_storage.Proxy + haTracker *hatracker.HighAvailabilityTracker + hashdexFactory cppbridge.HashdexFactory + hashdexLimits cppbridge.WALHashdexLimits + transparentState *cppbridge.StateV2 + mergeOutOfOrderChunks func() + + // stat + activeQuerierMetrics *querier.Metrics + storageQuerierMetrics *querier.Metrics + appendDuration prometheus.Histogram +} + +// NewAdapter init new [Adapter]. +func NewAdapter( + clock clockwork.Clock, + proxy *pp_storage.Proxy, + mergeOutOfOrderChunks func(), + registerer prometheus.Registerer, +) *Adapter { + factory := util.NewUnconflictRegisterer(registerer) + return &Adapter{ + proxy: proxy, + haTracker: hatracker.NewHighAvailabilityTracker(clock, registerer), + hashdexFactory: cppbridge.HashdexFactory{}, + hashdexLimits: cppbridge.DefaultWALHashdexLimits(), + transparentState: cppbridge.NewTransitionStateV2(), + mergeOutOfOrderChunks: mergeOutOfOrderChunks, + activeQuerierMetrics: querier.NewMetrics(registerer, querier.QueryableAppenderSource), + storageQuerierMetrics: querier.NewMetrics(registerer, querier.QueryableStorageSource), + appendDuration: factory.NewHistogram( + prometheus.HistogramOpts{ + Name: "prompp_adapter_append_duration", + Help: "Append to head duration in microseconds", + Buckets: []float64{ + 50, 100, 250, 500, 750, + 1000, 2500, 5000, 7500, + 10000, 25000, 50000, 75000, + 100000, 500000, + }, + }, + ), + } +} + +// AppendHashdex append incoming [cppbridge.HashdexContent] to [Head]. +func (ar *Adapter) AppendHashdex( + ctx context.Context, + hashdex cppbridge.ShardedData, + state *cppbridge.StateV2, + commitToWal bool, +) error { + if ar.haTracker.IsDrop(hashdex.Cluster(), hashdex.Replica()) { + return nil + } + + start := time.Now() + defer func() { + ar.appendDuration.Observe(float64(time.Since(start).Microseconds())) + }() + + return ar.proxy.With(ctx, func(h *pp_storage.Head) error { + _, _, err := appender.New(h, services.CFViaRange).Append( + ctx, + &appender.IncomingData{Hashdex: hashdex}, + state, + commitToWal, + ) + + return err + }) +} + +// AppendScraperHashdex append ScraperHashdex data to [Head]. +func (ar *Adapter) AppendScraperHashdex( + ctx context.Context, + hashdex cppbridge.ShardedData, + state *cppbridge.StateV2, + commitToWal bool, +) (stats cppbridge.RelabelerStats, err error) { + start := time.Now() + defer func() { + ar.appendDuration.Observe(float64(time.Since(start).Microseconds())) + }() + + _ = ar.proxy.With(ctx, func(h *pp_storage.Head) error { + _, stats, err = appender.New(h, services.CFViaRange).Append( + ctx, + &appender.IncomingData{Hashdex: hashdex}, + state, + commitToWal, + ) + + return nil + }) + + return stats, err +} + +// AppendSnappyProtobuf append compressed via snappy Protobuf data to [Head]. +func (ar *Adapter) AppendSnappyProtobuf( + ctx context.Context, + compressedData model.ProtobufData, + state *cppbridge.StateV2, + commitToWal bool, +) error { + hx, err := cppbridge.NewWALSnappyProtobufHashdex(compressedData.Bytes(), ar.hashdexLimits) + compressedData.Destroy() + if err != nil { + return err + } + + if ar.haTracker.IsDrop(hx.Cluster(), hx.Replica()) { + return nil + } + + start := time.Now() + defer func() { + ar.appendDuration.Observe(float64(time.Since(start).Microseconds())) + }() + + return ar.proxy.With(ctx, func(h *pp_storage.Head) error { + _, _, err := appender.New(h, services.CFViaRange).Append( + ctx, + &appender.IncomingData{Hashdex: hx}, + state, + commitToWal, + ) + + return err + }) +} + +// AppendTimeSeries append TimeSeries data to [Head]. +func (ar *Adapter) AppendTimeSeries( + ctx context.Context, + data model.TimeSeriesBatch, + state *cppbridge.StateV2, + commitToWal bool, +) (stats cppbridge.RelabelerStats, err error) { + hx, err := ar.hashdexFactory.GoModel(data.TimeSeries(), ar.hashdexLimits) + if err != nil { + data.Destroy() + return stats, err + } + + if ar.haTracker.IsDrop(hx.Cluster(), hx.Replica()) { + data.Destroy() + return stats, nil + } + + start := time.Now() + defer func() { + ar.appendDuration.Observe(float64(time.Since(start).Microseconds())) + }() + + _ = ar.proxy.With(ctx, func(h *pp_storage.Head) error { + _, stats, err = appender.New(h, services.CFViaRange).Append( + ctx, + &appender.IncomingData{Hashdex: hx, Data: data}, + state, + commitToWal, + ) + + return nil + }) + + return stats, err +} + +// Appender create a new [storage.Appender] for [Head]. +func (ar *Adapter) Appender(ctx context.Context) storage.Appender { + return newTimeSeriesAppender(ctx, ar, ar.transparentState) +} + +// ChunkQuerier provides querying access over time series data of a fixed time range. +// Returns new Chunk Querier that merges results of given primary and secondary chunk queriers. +func (ar *Adapter) ChunkQuerier(mint, maxt int64) (storage.ChunkQuerier, error) { + queriers := make([]storage.ChunkQuerier, 0, 1) //revive:disable-line:add-constant // the best way + ahead := ar.proxy.Get() + queriers = append( + queriers, + querier.NewChunkQuerier(ahead, querier.NewNoOpShardedDeduplicator, mint, maxt, nil), + ) + + for _, head := range ar.proxy.Heads() { + if ahead.ID() == head.ID() { + continue + } + + queriers = append( + queriers, + querier.NewChunkQuerier(head, querier.NewNoOpShardedDeduplicator, mint, maxt, nil), + ) + } + + return storage.NewMergeChunkQuerier( + nil, + queriers, + storage.NewConcatenatingChunkSeriesMerger(), + ), nil +} + +// Close closes the storage and all its underlying resources. +// Implements the [storage.Storage] interface. +func (ar *Adapter) Close() error { + ar.haTracker.Destroy() + return nil +} + +// HeadQuerier returns [storage.Querier] from active head. +func (ar *Adapter) HeadQuerier(mint, maxt int64) (storage.Querier, error) { + return querier.NewQuerier( + ar.proxy.Get(), + querier.NewNoOpShardedDeduplicator, + mint, + maxt, + nil, + ar.activeQuerierMetrics, + ), nil +} + +// HeadStatus returns stats of Head. +func (ar *Adapter) HeadStatus(ctx context.Context, limit int) (*querier.HeadStatus, error) { + return querier.QueryHeadStatus(ctx, ar.proxy.Get(), limit) +} + +// LowestSentTimestamp returns the lowest sent timestamp across all queues. +func (*Adapter) LowestSentTimestamp() int64 { + return 0 +} + +// MergeOutOfOrderChunks send signal to merge chunks with out of order data chunks. +func (ar *Adapter) MergeOutOfOrderChunks() { + ar.mergeOutOfOrderChunks() +} + +// Querier calls f() with the given parameters. +// Returns a [querier.MultiQuerier] combining of primary and secondary queriers. +func (ar *Adapter) Querier(mint, maxt int64) (storage.Querier, error) { + queriers := make([]storage.Querier, 0, 1) //revive:disable-line:add-constant // the best way + ahead := ar.proxy.Get() + queriers = append( + queriers, + querier.NewQuerier(ahead, querier.NewNoOpShardedDeduplicator, mint, maxt, nil, ar.activeQuerierMetrics), + ) + + for _, head := range ar.proxy.Heads() { + if ahead.ID() == head.ID() { + continue + } + + queriers = append( + queriers, + querier.NewQuerier(head, querier.NewNoOpShardedDeduplicator, mint, maxt, nil, ar.storageQuerierMetrics), + ) + } + + return querier.NewMultiQuerier(queriers, nil), nil +} + +// StartTime returns the oldest timestamp stored in the storage. +// Implements the [storage.Storage] interface. +func (*Adapter) StartTime() (int64, error) { + return math.MaxInt64, nil +} diff --git a/pp-pkg/storage/appender.go b/pp-pkg/storage/appender.go new file mode 100644 index 0000000000..2db4f3b731 --- /dev/null +++ b/pp-pkg/storage/appender.go @@ -0,0 +1,123 @@ +package storage + +import ( + "context" + + "github.com/prometheus/prometheus/model/exemplar" + "github.com/prometheus/prometheus/model/histogram" + "github.com/prometheus/prometheus/model/labels" + "github.com/prometheus/prometheus/model/metadata" + "github.com/prometheus/prometheus/pp/go/cppbridge" + "github.com/prometheus/prometheus/pp/go/model" + "github.com/prometheus/prometheus/storage" +) + +// timeSeriesBatch implementation buffer of [TimeSeriesBatch]. +type timeSeriesBatch struct { + timeSeries []model.TimeSeries +} + +// TimeSeries returns slice [model.TimeSeries]. +func (d *timeSeriesBatch) TimeSeries() []model.TimeSeries { + return d.timeSeries +} + +// Destroy buffered data. +func (d *timeSeriesBatch) Destroy() { + d.timeSeries = nil +} + +// TimeSeriesAppender appender for rules, aggregates the [model.TimeSeries] batch and append to head, +// implementation [storage.Appender]. +type TimeSeriesAppender struct { + ctx context.Context + adapter *Adapter + state *cppbridge.StateV2 + batch *timeSeriesBatch +} + +func newTimeSeriesAppender( + ctx context.Context, + adapter *Adapter, + state *cppbridge.StateV2, +) *TimeSeriesAppender { + return &TimeSeriesAppender{ + ctx: ctx, + adapter: adapter, + state: state, + batch: &timeSeriesBatch{}, + } +} + +// Append adds a sample pair for the given series, implementation [storage.Appender]. +func (a *TimeSeriesAppender) Append( + _ storage.SeriesRef, + l labels.Labels, + t int64, + v float64, +) (storage.SeriesRef, error) { + lsb := model.NewLabelSetBuilder() + l.Range(func(label labels.Label) { + lsb.Add(label.Name, label.Value) + }) + + a.batch.timeSeries = append(a.batch.timeSeries, model.TimeSeries{ + LabelSet: lsb.Build(), + Timestamp: uint64(t), // #nosec G115 // no overflow + Value: v, + }) + return 0, nil +} + +// AppendCTZeroSample do nothing, implementation [storage.Appender]. +func (*TimeSeriesAppender) AppendCTZeroSample( + _ storage.SeriesRef, + _ labels.Labels, + _, _ int64, +) (storage.SeriesRef, error) { + return 0, nil +} + +// AppendExemplar do nothing, implementation [storage.Appender]. +func (*TimeSeriesAppender) AppendExemplar( + _ storage.SeriesRef, + _ labels.Labels, + _ exemplar.Exemplar, +) (storage.SeriesRef, error) { + return 0, nil +} + +// AppendHistogram do nothing, implementation [storage.Appender]. +func (*TimeSeriesAppender) AppendHistogram( + _ storage.SeriesRef, + _ labels.Labels, + _ int64, + _ *histogram.Histogram, + _ *histogram.FloatHistogram, +) (storage.SeriesRef, error) { + return 0, nil +} + +// Commit adds aggregated series to the head, implementation [storage.Appender]. +func (a *TimeSeriesAppender) Commit() error { + if len(a.batch.timeSeries) == 0 { + return nil + } + + _, err := a.adapter.AppendTimeSeries(a.ctx, a.batch, a.state, false) + return err +} + +// Rollback do nothing, implementation [storage.Appender]. +func (*TimeSeriesAppender) Rollback() error { + return nil +} + +// UpdateMetadata do nothing, implementation [storage.Appender]. +func (*TimeSeriesAppender) UpdateMetadata( + _ storage.SeriesRef, + _ labels.Labels, + _ metadata.Metadata, +) (storage.SeriesRef, error) { + return 0, nil +} diff --git a/pp-pkg/storage/remote_read.go b/pp-pkg/storage/remote_read.go index ab65dee9d6..194b3ddb1f 100644 --- a/pp-pkg/storage/remote_read.go +++ b/pp-pkg/storage/remote_read.go @@ -1,14 +1,13 @@ package storage import ( - "crypto/md5" + "crypto/md5" // #nosec G501 // cryptographic strength is not required "encoding/hex" "fmt" "sync" "time" "github.com/go-kit/log" - "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/common/model" "gopkg.in/yaml.v2" @@ -33,10 +32,7 @@ type RemoteRead struct { // NewRemoteRead returns a RemoteRead. func NewRemoteRead( l log.Logger, - reg prometheus.Registerer, stCallback func() (int64, error), - walDir string, - flushDeadline time.Duration, ) *RemoteRead { if l == nil { l = log.NewNopLogger() @@ -55,7 +51,6 @@ func NewRemoteRead( func (s *RemoteRead) ApplyConfig(conf *config.Config) error { s.mtx.Lock() defer s.mtx.Unlock() - // Update read clients readHashes := make(map[string]struct{}) queryables := make([]storage.SampleAndChunkQueryable, 0, len(conf.RemoteReadConfigs)) @@ -74,7 +69,7 @@ func (s *RemoteRead) ApplyConfig(conf *config.Config) error { // Set the queue name to the config hash if the user has not set // a name in their remote write config so we can still differentiate // between queues that have the same remote write endpoint. - name := hash[:6] + name := hash[:6] //revive:disable-line:add-constant // 6 characters for the name if rrConf.Name != "" { name = rrConf.Name } @@ -106,16 +101,6 @@ func (s *RemoteRead) ApplyConfig(conf *config.Config) error { return nil } -// // StartTime implements the Storage interface. -// func (s *RemoteRead) StartTime() (int64, error) { -// return int64(model.Latest), nil -// } - -// // Appender implements the Storage interface. -// func (s *RemoteRead) Appender(_ context.Context) storage.Appender { -// return noOpAppender{} -// } - // Querier returns a storage.MergeQuerier combining the remote client queriers // of each configured remote read endpoint. // Returned querier will never return error as all queryables are assumed best effort. @@ -183,6 +168,6 @@ func toHash(data interface{}) (string, error) { if err != nil { return "", err } - hash := md5.Sum(bytes) + hash := md5.Sum(bytes) // #nosec G401 // cryptographic strength is not required return hex.EncodeToString(hash[:]), nil } diff --git a/pp-pkg/storage/storage.go b/pp-pkg/storage/storage.go index fd091684c0..455c6d6ae7 100644 --- a/pp-pkg/storage/storage.go +++ b/pp-pkg/storage/storage.go @@ -13,61 +13,94 @@ import ( "github.com/prometheus/prometheus/util/annotations" ) +// QueryableStorage implements [storage.Storage], wrap around [storage.SampleAndChunkQueryable]. type QueryableStorage struct { queryable storage.SampleAndChunkQueryable } -func (s *QueryableStorage) Querier(mint, maxt int64) (storage.Querier, error) { - return s.queryable.Querier(mint, maxt) +// NewQueryableStorage init a new [QueryableStorage]. +func NewQueryableStorage(queryable storage.SampleAndChunkQueryable) *QueryableStorage { + return &QueryableStorage{queryable: queryable} } +// Appender implements [storage.Storage], wrap around [storage.SampleAndChunkQueryable]. +func (*QueryableStorage) Appender(_ context.Context) storage.Appender { + return noOpAppender{} +} + +// ChunkQuerier implements [storage.Storage], wrap around [storage.SampleAndChunkQueryable]. func (s *QueryableStorage) ChunkQuerier(mint, maxt int64) (storage.ChunkQuerier, error) { return s.queryable.ChunkQuerier(mint, maxt) } -func (s *QueryableStorage) Appender(_ context.Context) storage.Appender { - return noOpAppender{} +// Close implements [storage.Storage], wrap around [storage.SampleAndChunkQueryable]. +func (*QueryableStorage) Close() error { + return nil } -func (s *QueryableStorage) StartTime() (int64, error) { - return int64(model.Latest), nil +// Querier implements [storage.Storage], wrap around [storage.SampleAndChunkQueryable]. +func (s *QueryableStorage) Querier(mint, maxt int64) (storage.Querier, error) { + return s.queryable.Querier(mint, maxt) } -func (s *QueryableStorage) Close() error { - return nil +// StartTime implements [storage.Storage], wrap around [storage.SampleAndChunkQueryable]. +func (*QueryableStorage) StartTime() (int64, error) { + return int64(model.Latest), nil } -func NewQueryableStorage(queryable storage.SampleAndChunkQueryable) *QueryableStorage { - return &QueryableStorage{queryable: queryable} -} +// +// noOpAppender +// +// noOpAppender implements [storage.Appender], do nothing. type noOpAppender struct{} +// Append implements [storage.Appender], do nothing. func (noOpAppender) Append(_ storage.SeriesRef, _ labels.Labels, _ int64, _ float64) (storage.SeriesRef, error) { return 0, nil } -func (noOpAppender) Commit() error { - return nil +// AppendCTZeroSample implements [storage.Appender], do nothing. +func (noOpAppender) AppendCTZeroSample(_ storage.SeriesRef, _ labels.Labels, _, _ int64) (storage.SeriesRef, error) { + return 0, nil } -func (noOpAppender) Rollback() error { - return nil +// AppendExemplar implements [storage.Appender], do nothing. +func (noOpAppender) AppendExemplar( + _ storage.SeriesRef, + _ labels.Labels, + _ exemplar.Exemplar, +) (storage.SeriesRef, error) { + return 0, nil } -func (noOpAppender) AppendExemplar(ref storage.SeriesRef, l labels.Labels, e exemplar.Exemplar) (storage.SeriesRef, error) { +// AppendHistogram implements [storage.Appender], do nothing. +func (noOpAppender) AppendHistogram( + _ storage.SeriesRef, + _ labels.Labels, + _ int64, + _ *histogram.Histogram, + _ *histogram.FloatHistogram, +) (storage.SeriesRef, error) { return 0, nil } -func (noOpAppender) AppendHistogram(ref storage.SeriesRef, l labels.Labels, t int64, h *histogram.Histogram, fh *histogram.FloatHistogram) (storage.SeriesRef, error) { - return 0, nil +// Commit implements [storage.Appender], do nothing. +func (noOpAppender) Commit() error { + return nil } -func (noOpAppender) UpdateMetadata(ref storage.SeriesRef, l labels.Labels, m metadata.Metadata) (storage.SeriesRef, error) { - return 0, nil +// Rollback implements [storage.Appender], do nothing. +func (noOpAppender) Rollback() error { + return nil } -func (noOpAppender) AppendCTZeroSample(ref storage.SeriesRef, l labels.Labels, t, ct int64) (storage.SeriesRef, error) { +// UpdateMetadata implements [storage.Appender], do nothing. +func (noOpAppender) UpdateMetadata( + _ storage.SeriesRef, + _ labels.Labels, + _ metadata.Metadata, +) (storage.SeriesRef, error) { return 0, nil } @@ -86,7 +119,7 @@ func (*noOpStorage) Appender(_ context.Context) storage.Appender { } // Querier implements storage.Storage. -func (*noOpStorage) Querier(mint, maxt int64) (storage.Querier, error) { +func (*noOpStorage) Querier(_, _ int64) (storage.Querier, error) { return noOpQuerier{}, nil } @@ -220,16 +253,16 @@ func (noOpChunkSeriesSet) Next() bool { } // At implements storage.ChunkSeriesSet. -func (n noOpChunkSeriesSet) At() storage.ChunkSeries { +func (noOpChunkSeriesSet) At() storage.ChunkSeries { return nil } // Err implements storage.ChunkSeriesSet. -func (n noOpChunkSeriesSet) Err() error { +func (noOpChunkSeriesSet) Err() error { return nil } // Warnings implements storage.ChunkSeriesSet. -func (n noOpChunkSeriesSet) Warnings() annotations.Annotations { +func (noOpChunkSeriesSet) Warnings() annotations.Annotations { return nil } diff --git a/pp-pkg/tsdb/db.go b/pp-pkg/tsdb/db.go index f5ab64d89c..8d156c12ed 100644 --- a/pp-pkg/tsdb/db.go +++ b/pp-pkg/tsdb/db.go @@ -1,12 +1,13 @@ package tsdb import ( + "path/filepath" + "slices" + "github.com/oklog/ulid" - "github.com/prometheus/prometheus/pp/go/relabeler/head/catalog" + "github.com/prometheus/prometheus/pp/go/storage/catalog" "github.com/prometheus/prometheus/tsdb" "github.com/prometheus/prometheus/tsdb/fileutil" - "path/filepath" - "slices" ) // PPBlocksToDelete returns a filter which decides time based and size based @@ -104,7 +105,7 @@ func BeyondSizeRetention(db *tsdb.DB, dir string, catalog *catalog.Catalog, bloc func catalogHeadsSize(dir string, catalog *catalog.Catalog) (catalogSize int64) { catalogSize += catalog.OnDiskSize() - heads, _ := catalog.List(nil, nil) + heads := catalog.List(nil, nil) for _, h := range heads { catalogSize += headSize(filepath.Join(dir, h.Dir())) } diff --git a/pp/.golangci.yml b/pp/.golangci.yml index 9572d7ffe2..83c92b26b4 100644 --- a/pp/.golangci.yml +++ b/pp/.golangci.yml @@ -7,9 +7,6 @@ run: # default concurrency is a available CPU number concurrency: 4 - # timeout for analysis, e.g. 30s, 5m, default is 1m - deadline: 10m - # exit code when at least one issue was found, default is 1 issues-exit-code: 1 @@ -17,9 +14,6 @@ run: tests: true # output configuration options output: - # colored-line-number|line-number|json|tab|checkstyle|code-climate, default is "colored-line-number" - format: colored-line-number - # print lines of code with issue, default is true print-issued-lines: true @@ -29,8 +23,6 @@ output: # all available settings of specific linters linters-settings: govet: - # report about shadowed variables - check-shadowing: true # settings per analyzer settings: printf: # analyzer name, run `go tool vet help` to see all analyzers @@ -39,6 +31,10 @@ linters-settings: - (github.com/golangci/golangci-lint/pkg/logutils.Log).Warnf - (github.com/golangci/golangci-lint/pkg/logutils.Log).Errorf - (github.com/golangci/golangci-lint/pkg/logutils.Log).Fatalf + shadow: + # Whether to be strict about shadowing; can be noisy. + # Default: false + strict: true gofmt: # simplify code: gofmt with `-s` option, true by default simplify: true @@ -61,12 +57,15 @@ linters-settings: locale: US ignore-words: - someword - unused: - # treat code as a program (not a library) and report unused exported identifiers; default is false. - # XXX: if you enable this setting, unused will report a lot of false-positives in text editors: - # if it's called for subdir of a project it can't find funcs usages. All text editor integrations - # with golangci-lint call it on a directory with the changed file. - check-exported: false + # unused: + # # treat code as a program (not a library) and report unused exported identifiers; default is false. + # # XXX: if you enable this setting, unused will report a lot of false-positives in text editors: + # # if it's called for subdir of a project it can't find funcs usages. All text editor integrations + # # with golangci-lint call it on a directory with the changed file. + # check-exported: false + # # Mark all exported fields as used. + # # default: true + # exported-fields-are-used: false nakedret: # make an issue if func has more lines of code than this setting and it has naked returns; default is 30 max-func-lines: 30 @@ -128,7 +127,7 @@ linters-settings: - name: function-result-limit arguments: - 3 - - name: imports-blacklist + - name: imports-blocklist # imports-blacklist arguments: - github.com/davecgh/go-spew/spew - name: range-val-in-closure @@ -155,6 +154,21 @@ linters-settings: arguments: - 30 - 50 + testifylint: + disable: + - float-compare + - go-require + enable: + - bool-compare + - compares + - empty + - error-is-as + - error-nil + - expected-actual + - len + - require-error + - suite-dont-use-pkg + - suite-extra-assert-call linters: disable-all: true @@ -177,6 +191,7 @@ linters: - unconvert # Remove unnecessary type conversions [fast: true, auto-fix: false] - unused # Checks Go code for unused constants, variables, functions and types [fast: false, auto-fix: false] - revive + - testifylint # Checks usage of github.com/stretchr/testify. fast: false issues: diff --git a/pp/WORKSPACE b/pp/WORKSPACE index b76ef933ff..717cf53bd4 100644 --- a/pp/WORKSPACE +++ b/pp/WORKSPACE @@ -1,8 +1,7 @@ workspace(name = "prompp") load("@bazel_tools//tools/build_defs/repo:git.bzl", "git_repository") -load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") -load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_file") +load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive", "http_file") http_archive( name = "bazel_skylib", @@ -41,7 +40,7 @@ git_repository( http_archive( name = "google_benchmark", patches = [ - "//third_party/patches/google_benchmark:0001-BUILD.bazel.patch" + "//third_party/patches/google_benchmark:0001-BUILD.bazel.patch", ], sha256 = "35a77f46cc782b16fac8d3b107fbfbb37dcd645f7c28eee19f3b8e0758b48994", strip_prefix = "benchmark-1.9.0/", @@ -82,6 +81,7 @@ http_archive( build_file = "//third_party:parallel_hashmap.BUILD", patches = [ "//third_party/patches/parallel_hashmap:0001-svacer_fixes.patch", + "//third_party/patches/parallel_hashmap:btree.h.patch", ], sha256 = "b61435437713e2d98ce2a5539a0bff7e6e9e6a6b9fe507dbf490a852b8c2904f", strip_prefix = "parallel-hashmap-1.35", @@ -118,6 +118,9 @@ http_archive( "//third_party/patches/roaring:0001-disable-test-dependencies.patch", "//third_party/patches/roaring:0002-svacer_fixes.patch", ], + patch_cmds = [ + "cp cpp/* include/roaring" + ], sha256 = "a037e12a3f7c8c2abb3e81fc9669c23e274ffa2d8670d2034a2e05969e53689b", strip_prefix = "CRoaring-1.3.0/", url = "https://github.com/RoaringBitmap/CRoaring/archive/refs/tags/v1.3.0.zip", @@ -195,8 +198,8 @@ http_archive( http_file( name = "fastfloat_header", downloaded_file_path = "fastfloat/fast_float.h", - url = "https://github.com/fastfloat/fast_float/releases/download/v8.0.0/fast_float.h", sha256 = "1335e82c61fda54476ecbd94b92356deebeb3f0122802c3f103ee528ac08624e", + url = "https://github.com/fastfloat/fast_float/releases/download/v8.0.0/fast_float.h", ) local_repository( diff --git a/pp/bare_bones/concepts.h b/pp/bare_bones/concepts.h index 62e0053bfc..605e0aac5d 100644 --- a/pp/bare_bones/concepts.h +++ b/pp/bare_bones/concepts.h @@ -14,6 +14,31 @@ concept has_allocated_memory = requires(const T& t) { { t.allocated_memory() } -> std::convertible_to; }; +template +concept has_allocated_memory_field = requires(const T& t) { + { t.allocated_memory } -> std::convertible_to; +}; + +template +concept has_earliest_timestamp_field = requires(const T& t) { + { t.earliest_timestamp } -> std::convertible_to; +}; + +template +concept has_latest_timestamp_field = requires(const T& t) { + { t.latest_timestamp } -> std::convertible_to; +}; + +template +concept has_series_field = requires(const T& t) { + { t.series } -> std::convertible_to; +}; + +template +concept has_remainder_size_field = requires(const T& t) { + { t.remainder_size } -> std::convertible_to; +}; + template concept dereferenceable_has_allocated_memory = requires(const T& t) { { t->allocated_memory() } -> std::convertible_to; @@ -30,8 +55,8 @@ concept has_size = requires(const T& t) { }; template -concept has_reserve = requires(T& r) { - { r.reserve(size_t{}) }; +concept has_reserve = requires(T& r, uint32_t n) { + { r.reserve(n) } -> std::same_as; }; template diff --git a/pp/bare_bones/memory.h b/pp/bare_bones/memory.h index 4c55789f89..ca8642acc8 100644 --- a/pp/bare_bones/memory.h +++ b/pp/bare_bones/memory.h @@ -101,20 +101,6 @@ template struct MemoryControlBlock { using SizeType = uint32_t; - MemoryControlBlock() = default; - MemoryControlBlock(const MemoryControlBlock&) = delete; - MemoryControlBlock(MemoryControlBlock&& other) noexcept : data(std::exchange(other.data, nullptr)), data_size(std::exchange(other.data_size, 0)) {} - - MemoryControlBlock& operator=(const MemoryControlBlock&) = delete; - PROMPP_ALWAYS_INLINE MemoryControlBlock& operator=(MemoryControlBlock&& other) noexcept { - if (this != &other) [[likely]] { - data = std::exchange(other.data, nullptr); - data_size = std::exchange(other.data_size, 0); - } - - return *this; - } - T* data{}; SizeType data_size{}; }; @@ -123,22 +109,6 @@ template struct MemoryControlBlockWithItemCount { using SizeType = uint32_t; - MemoryControlBlockWithItemCount() = default; - MemoryControlBlockWithItemCount(const MemoryControlBlockWithItemCount&) = delete; - MemoryControlBlockWithItemCount(MemoryControlBlockWithItemCount&& other) noexcept - : data(std::exchange(other.data, nullptr)), data_size(std::exchange(other.data_size, 0)), items_count(std::exchange(other.items_count, 0)) {} - - MemoryControlBlockWithItemCount& operator=(const MemoryControlBlockWithItemCount&) = delete; - PROMPP_ALWAYS_INLINE MemoryControlBlockWithItemCount& operator=(MemoryControlBlockWithItemCount&& other) noexcept { - if (this != &other) [[likely]] { - data = std::exchange(other.data, nullptr); - data_size = std::exchange(other.data_size, 0); - items_count = std::exchange(other.items_count, 0); - } - - return *this; - } - T* data{}; SizeType data_size{}; SizeType items_count{}; @@ -152,7 +122,7 @@ class Memory : public GenericMemory, typename ControlBlo PROMPP_ALWAYS_INLINE Memory() noexcept = default; PROMPP_ALWAYS_INLINE Memory(const Memory& o) noexcept { copy(o); } - PROMPP_ALWAYS_INLINE Memory(Memory&& o) noexcept = default; + PROMPP_ALWAYS_INLINE Memory(Memory&& o) noexcept : control_block_(std::exchange(o.control_block_, {})) {}; PROMPP_ALWAYS_INLINE ~Memory() noexcept { std::free(control_block_.data); } PROMPP_ALWAYS_INLINE Memory& operator=(const Memory& o) noexcept { @@ -166,7 +136,7 @@ class Memory : public GenericMemory, typename ControlBlo PROMPP_ALWAYS_INLINE Memory& operator=(Memory&& o) noexcept { if (this != &o) [[likely]] { std::free(control_block_.data); - control_block_ = std::move(o.control_block_); + control_block_ = std::exchange(o.control_block_, {}); } return *this; @@ -202,7 +172,11 @@ class Memory : public GenericMemory, typename ControlBlo PROMPP_ALWAYS_INLINE void copy(const Memory& o) noexcept { static_assert(IsTriviallyCopyable::value, "it's not allowed to copy memory for non trivially copyable types"); - resize(o.control_block_.data_size); + T* data = control_block_.data; + control_block_ = o.control_block_; + control_block_.data = data; + + resize(control_block_.data_size); PRAGMA_DIAGNOSTIC(push) PRAGMA_DIAGNOSTIC(ignored DIAGNOSTIC_CLASS_MEMACCESS) diff --git a/pp/bare_bones/snug_composite.h b/pp/bare_bones/snug_composite.h index b2d2603ce7..1056b65155 100644 --- a/pp/bare_bones/snug_composite.h +++ b/pp/bare_bones/snug_composite.h @@ -110,14 +110,9 @@ class GenericDecodingTable { struct LessComparator { using is_transparent = void; - PROMPP_ALWAYS_INLINE explicit LessComparator(const GenericDecodingTable* decoding_table, bool* enabled) noexcept - : decoding_table_(decoding_table), enabled_(enabled) {} + PROMPP_ALWAYS_INLINE explicit LessComparator(const GenericDecodingTable* decoding_table) noexcept : decoding_table_(decoding_table) {} PROMPP_ALWAYS_INLINE bool operator()(const Proxy& a, const Proxy& b) const noexcept { - if (!*enabled_) { - return true; - } - return decoding_table_->items_[a].composite(decoding_table_->data_) < decoding_table_->items_[b].composite(decoding_table_->data_); } @@ -134,7 +129,6 @@ class GenericDecodingTable { private: const GenericDecodingTable* decoding_table_; - bool* enabled_; }; template @@ -263,7 +257,7 @@ class GenericDecodingTable { [[nodiscard]] PROMPP_ALWAYS_INLINE Hasher hasher() const noexcept { return Hasher(this); } [[nodiscard]] PROMPP_ALWAYS_INLINE EqualityComparator equality_comparator() const noexcept { return EqualityComparator(this); } - [[nodiscard]] PROMPP_ALWAYS_INLINE LessComparator less_comparator(bool* enabled) const noexcept { return LessComparator(this, enabled); } + [[nodiscard]] PROMPP_ALWAYS_INLINE LessComparator less_comparator() const noexcept { return LessComparator(this); } data_type data_; Vector> items_; @@ -293,7 +287,8 @@ class GenericDecodingTable { inline __attribute__((always_inline)) const auto& items() const noexcept { return items_; } - PROMPP_ALWAYS_INLINE void reserve(const GenericDecodingTable& other) { + template class> class FilamentOther, template class VectorOther> + PROMPP_ALWAYS_INLINE void reserve(const GenericDecodingTable& other) { items_.reserve(other.items_.size()); data_.reserve(other.data_); } @@ -791,7 +786,6 @@ class OrderedEncodingBimap : public GenericDecodingTable; Set set_; - bool set_soring_enabled_{true}; protected: PROMPP_ALWAYS_INLINE void after_items_load_impl(uint32_t first_loaded_id) noexcept { @@ -801,7 +795,7 @@ class OrderedEncodingBimap : public GenericDecodingTable; OrderedSet ordered_set_; - bool ordered_set_soring_enabled_{true}; using Set = phmap::flat_hash_set; Set set_; @@ -912,7 +905,7 @@ class EncodingBimapWithOrderedAccess : public GenericDecodingTable{1, 100, 1000})); +} + +TEST_F(BitsetConstructorsFixture, CopyAssignment) { + // Arrange + bs_.resize(1001); + + bs_.set(1); + bs_.set(100); + bs_.set(1000); + + // Act + BareBones::Bitset bs_copy = bs_; + + // Assert + EXPECT_TRUE(std::ranges::equal(bs_, bs_copy)); +} + +TEST_F(BitsetConstructorsFixture, CopyAssignmentNonEmpty) { + // Arrange + bs_.resize(1001); + + bs_.set(1); + bs_.set(100); + bs_.set(1000); + + BareBones::Bitset bs_copy; + bs_copy.resize(3); + bs_copy.set(0); + bs_copy.set(1); + bs_copy.set(2); + + // Act + bs_copy = bs_; + + // Assert + EXPECT_TRUE(std::ranges::equal(bs_, bs_copy)); +} + +TEST_F(BitsetConstructorsFixture, MoveAssignment) { + // Arrange + bs_.resize(1001); + + bs_.set(1); + bs_.set(100); + bs_.set(1000); + + // Act + BareBones::Bitset bs_move = std::move(bs_); + + // Assert + EXPECT_TRUE(std::ranges::equal(bs_move, std::initializer_list{1, 100, 1000})); +} + +TEST_F(BitsetConstructorsFixture, MoveAssignmentNonEmpty) { + // Arrange + bs_.resize(1001); + + bs_.set(1); + bs_.set(100); + bs_.set(1000); + + BareBones::Bitset bs_move; + bs_move.resize(3); + bs_move.set(0); + bs_move.set(1); + bs_move.set(2); + + // Act + bs_move = std::move(bs_); + + // Assert + EXPECT_TRUE(std::ranges::equal(bs_move, std::initializer_list{1, 100, 1000})); +} + } // namespace diff --git a/pp/bare_bones/tests/memory_tests.cpp b/pp/bare_bones/tests/memory_tests.cpp index 0c07a1b5e3..7a08654ebb 100644 --- a/pp/bare_bones/tests/memory_tests.cpp +++ b/pp/bare_bones/tests/memory_tests.cpp @@ -10,6 +10,7 @@ using BareBones::AllocationSizeCalculator; using BareBones::DefaultReallocator; using BareBones::Memory; using BareBones::MemoryControlBlock; +using BareBones::MemoryControlBlockWithItemCount; using BareBones::SharedMemory; using BareBones::SharedPtr; @@ -136,6 +137,47 @@ TEST_F(MemoryFixture, MoveOperator) { EXPECT_EQ(memory_size, memory2.size()); } +class MemoryWithItemCountFixture : public ::testing::Test { + protected: + Memory memory_; +}; + +TEST_F(MemoryWithItemCountFixture, CopyConstructor) { + // Arrange + memory_.resize_to_fit_at_least(1); + std::iota(memory_.begin(), memory_.end(), uint8_t{}); + memory_.control_block().items_count = 100; + + // Act + const auto memory2 = memory_; + + // Assert + EXPECT_NE(memory2, memory_); + ASSERT_EQ(memory_.size(), memory2.size()); + ASSERT_EQ(memory_.control_block().data_size, memory2.control_block().data_size); + ASSERT_EQ(memory_.control_block().items_count, memory2.control_block().items_count); + EXPECT_TRUE(std::ranges::equal(memory2, memory_)); +} + +TEST_F(MemoryWithItemCountFixture, CopyOperator) { + // Arrange + memory_.resize_to_fit_at_least(1); + std::iota(memory_.begin(), memory_.end(), uint8_t{}); + memory_.control_block().items_count = 100; + + // Act + decltype(memory_) memory2; + memory2.resize_to_fit_at_least(1); + memory2 = memory_; + + // Assert + EXPECT_NE(memory2, memory_); + ASSERT_EQ(memory_.size(), memory2.size()); + ASSERT_EQ(memory_.control_block().data_size, memory2.control_block().data_size); + ASSERT_EQ(memory_.control_block().items_count, memory2.control_block().items_count); + EXPECT_TRUE(std::ranges::equal(memory2, memory_)); +} + class SharedPtrFixture : public ::testing::Test { protected: template diff --git a/pp/entrypoint/head_wal.cpp b/pp/entrypoint/head_wal.cpp index 74808059f4..36b499e6a6 100644 --- a/pp/entrypoint/head_wal.cpp +++ b/pp/entrypoint/head_wal.cpp @@ -67,13 +67,8 @@ extern "C" void prompp_head_wal_encoder_add_inner_series(void* args, void* res) }; struct Result { - int64_t earliest_timestamp; - int64_t latest_timestamp; - size_t allocated_memory; - uint32_t samples; - uint32_t series; - uint32_t remainder_size; PromPP::Primitives::Go::Slice error; + uint32_t samples; }; const auto in = static_cast(args); @@ -93,14 +88,9 @@ extern "C" void prompp_head_wal_encoder_finalize(void* args, void* res) { }; struct Result { - int64_t earliest_timestamp; - int64_t latest_timestamp; - size_t allocated_memory; - uint32_t samples; - uint32_t series; - uint32_t remainder_size; PromPP::Primitives::Go::Slice segment; PromPP::Primitives::Go::Slice error; + uint32_t samples; }; const auto in = static_cast(args); diff --git a/pp/entrypoint/head_wal.h b/pp/entrypoint/head_wal.h index d15999a7cc..d88ccde4bc 100644 --- a/pp/entrypoint/head_wal.h +++ b/pp/entrypoint/head_wal.h @@ -45,13 +45,8 @@ void prompp_head_wal_encoder_dtor(void* args); * encoder uintptr // pointer to constructed encoder; * } * @param res { - * earliestTimestamp int64 // minimal sample timestamp in segment - * latestTimestamp int64 // maximal sample timestamp in segment - * allocatedMemory uint64 // size of allocated memory for label sets; - * samples uint32 // number of samples in segment - * series uint32 // number of series in segment - * remainderSize uint32 // rest of internal buffers capacity * error []byte // error string if thrown + * samples uint32 // number of samples in segment * } */ void prompp_head_wal_encoder_add_inner_series(void* args, void* res); @@ -63,14 +58,9 @@ void prompp_head_wal_encoder_add_inner_series(void* args, void* res); * encoder uintptr // pointer to constructed encoder * } * @param res { - * earliestTimestamp int64 // minimal sample timestamp in segment - * latestTimestamp int64 // maximal sample timestamp in segment - * allocatedMemory uint64 // size of allocated memory for label sets; - * samples uint32 // number of samples in segment - * series uint32 // number of series in segment - * remainderSize uint32 // rest of internal buffers capacity * segment []byte // segment content * error []byte // error string if thrown + * samples uint32 // number of samples in segment * } */ void prompp_head_wal_encoder_finalize(void* args, void* res); diff --git a/pp/entrypoint/primitives_lss.cpp b/pp/entrypoint/primitives_lss.cpp index b09f23d11e..f6e62d22b8 100644 --- a/pp/entrypoint/primitives_lss.cpp +++ b/pp/entrypoint/primitives_lss.cpp @@ -251,7 +251,41 @@ extern "C" void prompp_create_readonly_lss(void* args, void* res) { } extern "C" void prompp_primitives_lss_copy_added_series(uint64_t source_lss, uint64_t destination_lss) { - series_index::QueryableEncodingBimapCopier copier(std::get(*std::bit_cast(source_lss)), - std::get(*std::bit_cast(destination_lss))); + auto& src = std::get(*std::bit_cast(source_lss)); + auto& dst = std::get(*std::bit_cast(destination_lss)); + src.build_deferred_indexes(); + + series_index::QueryableEncodingBimapCopier copier(src, src.sorting_index(), src.added_series(), dst); + copier.copy_added_series_and_build_indexes(); +} + +using BitsetPtr = std::unique_ptr; + +extern "C" void prompp_primitives_lss_bitset_series(void* args, void* res) { + struct Arguments { + LssVariantPtr lss; + }; + struct Result { + BitsetPtr bitset; + }; + + const auto& src = std::get(*static_cast(args)->lss); + new (res) Result{.bitset = std::make_unique(src.added_series())}; +} + +extern "C" void prompp_primitives_lss_bitset_dtor(void* args) { + struct Arguments { + BitsetPtr bitset; + }; + + static_cast(args)->~Arguments(); +} + +extern "C" void prompp_primitives_readonly_lss_copy_added_series(uint64_t source_lss, uint64_t source_bitset, uint64_t destination_lss) { + const auto& src = std::get(*std::bit_cast(source_lss)); + const auto& src_bitset = *std::bit_cast(source_bitset); + auto& dst = std::get(*std::bit_cast(destination_lss)); + + series_index::QueryableEncodingBimapCopier copier(src, src.sorting_index(), src_bitset, dst); copier.copy_added_series_and_build_indexes(); } diff --git a/pp/entrypoint/primitives_lss.h b/pp/entrypoint/primitives_lss.h index 00852f975f..17d51954f5 100644 --- a/pp/entrypoint/primitives_lss.h +++ b/pp/entrypoint/primitives_lss.h @@ -194,6 +194,41 @@ void prompp_create_readonly_lss(void* args, void* res); */ void prompp_primitives_lss_copy_added_series(uint64_t source_lss, uint64_t destination_lss); +/** + * @brief returns a copy of the bitset of added series from the lss. + * + * @param args { + * lss uintptr // pointer to constructed queryable lss; + * } + * + * @param res { + * bitset uintptr // bitset of added series; + * } + */ +void prompp_primitives_lss_bitset_series(void* args, void* res); + +/** + * @brief destroy bitset of added series. + * + * @param args { + * bitset uintptr // bitset of added series; + * } + * + */ +void prompp_primitives_lss_bitset_dtor(void* args); + +/** + * @brief Copy the label sets from the source lss to the destination lss that were added source lss. + * + * @param source_lss pointer to source label sets; + * @param source_bitset pointer to source bitset; + * @param destination_lss pointer to destination label sets; + * + * @attention This binding used as a CGO call!!! + * + */ +void prompp_primitives_readonly_lss_copy_added_series(uint64_t source_lss, uint64_t source_bitset, uint64_t destination_lss); + #ifdef __cplusplus } // extern "C" #endif diff --git a/pp/entrypoint/prometheus_relabeler.cpp b/pp/entrypoint/prometheus_relabeler.cpp index d1a546e78d..0184fa35e5 100644 --- a/pp/entrypoint/prometheus_relabeler.cpp +++ b/pp/entrypoint/prometheus_relabeler.cpp @@ -231,14 +231,6 @@ extern "C" void prompp_prometheus_relabel_stalenans_state_dtor(void* args) { static_cast(args)->~Arguments(); } -extern "C" void prompp_prometheus_relabel_stalenans_state_reset(void* args) { - struct Arguments { - StaleNaNsStatePtr state; - }; - - static_cast(args)->state->reset(); -} - extern "C" void prompp_prometheus_per_shard_relabeler_input_relabeling_with_stalenans(void* args, void* res) { struct Arguments { PromPP::Primitives::Go::SliceView shards_inner_series; @@ -537,10 +529,325 @@ extern "C" void prompp_prometheus_cache_allocated_memory(void* args, void* res) new (res) Result{.allocated_memory = in->cache->allocated_memory()}; } -extern "C" void prompp_prometheus_cache_reset_to(void* args) { +extern "C" void prompp_prometheus_cache_update(void* args, void* res) { + struct Arguments { + PromPP::Primitives::Go::SliceView shards_relabeler_state_update; + CachePtr cache; + uint16_t relabeled_shard_id; + }; + struct Result { + PromPP::Primitives::Go::Slice error; + }; + + const auto* in = static_cast(args); + + try { + for (size_t id = 0; id != in->shards_relabeler_state_update.size(); ++id) { + if (in->shards_relabeler_state_update[id] == nullptr || in->shards_relabeler_state_update[id]->size() == 0) { + continue; + } + + in->cache->update(in->shards_relabeler_state_update[id], id); + } + } catch (...) { + auto* out = new (res) Result(); + auto err_stream = PromPP::Primitives::Go::BytesStream(&out->error); + entrypoint::handle_current_exception(err_stream); + } +} + +// +// PerGoroutineRelabeler +// + +using PerGoroutineRelabeler = PromPP::Prometheus::Relabel::PerGoroutineRelabeler; +using PerGoroutineRelabelerPtr = std::unique_ptr; + +extern "C" void prompp_prometheus_per_goroutine_relabeler_ctor(void* args, void* res) { + struct Arguments { + uint16_t number_of_shards; + uint16_t shard_id; + }; + struct Result { + PerGoroutineRelabelerPtr per_goroutine_relabeler; + }; + + auto* in = static_cast(args); + auto* out = new (res) Result(); + + out->per_goroutine_relabeler = std::make_unique(in->number_of_shards, in->shard_id); +} + +extern "C" void prompp_prometheus_per_goroutine_relabeler_dtor(void* args) { + struct Arguments { + PerGoroutineRelabelerPtr per_goroutine_relabeler; + }; + + static_cast(args)->~Arguments(); +} + +extern "C" void prompp_prometheus_per_goroutine_relabeler_input_relabeling(void* args, void* res) { + struct Arguments { + PromPP::Primitives::Go::SliceView shards_inner_series; + PromPP::Primitives::Go::SliceView shards_relabeled_series; + PromPP::Prometheus::Relabel::RelabelerOptions options; + PerGoroutineRelabelerPtr per_goroutine_relabeler; + PromPP::Prometheus::Relabel::StatelessRelabeler* stateless_relabeler; + HashdexVariant* hashdex; + CachePtr cache; + LssVariantPtr input_lss; + LssVariantPtr target_lss; + }; + struct Result { + uint32_t samples_added{0}; + uint32_t series_added{0}; + uint32_t series_drop{0}; + PromPP::Primitives::Go::Slice error; + bool target_lss_has_reallocations{}; + }; + + auto in = static_cast(args); + auto out = new (res) Result(); + + try { + std::visit( + [in, out](auto& hashdex) { + auto& input_lss = std::get(*in->input_lss); + auto& target_lss = std::get(*in->target_lss); + + const entrypoint::head::ReallocationsDetector reallocation_detector(target_lss); + in->per_goroutine_relabeler->input_relabeling(input_lss, target_lss, *in->cache, hashdex, in->options, *in->stateless_relabeler, *out, + in->shards_inner_series, in->shards_relabeled_series); + target_lss.build_deferred_indexes(); + out->target_lss_has_reallocations = reallocation_detector.has_reallocations(); + }, + *in->hashdex); + } catch (...) { + auto err_stream = PromPP::Primitives::Go::BytesStream(&out->error); + entrypoint::handle_current_exception(err_stream); + } +} + +extern "C" void prompp_prometheus_per_goroutine_relabeler_input_relabeling_from_cache(void* args, void* res) { + struct Arguments { + PromPP::Primitives::Go::SliceView shards_inner_series; + PromPP::Prometheus::Relabel::RelabelerOptions options; + PerGoroutineRelabelerPtr per_goroutine_relabeler; + HashdexVariant* hashdex; + CachePtr cache; + LssVariantPtr input_lss; + LssVariantPtr target_lss; + }; + struct Result { + uint32_t samples_added{0}; + uint32_t series_added{0}; + uint32_t series_drop{0}; + bool ok{}; + PromPP::Primitives::Go::Slice error; + }; + + auto in = static_cast(args); + auto out = new (res) Result(); + + try { + std::visit( + [in, out](auto& hashdex) { + auto& input_lss = std::get(*in->input_lss); + auto& target_lss = std::get(*in->target_lss); + + out->ok = + in->per_goroutine_relabeler->input_relabeling_from_cache(input_lss, target_lss, *in->cache, hashdex, in->options, *out, in->shards_inner_series); + }, + *in->hashdex); + } catch (...) { + auto err_stream = PromPP::Primitives::Go::BytesStream(&out->error); + entrypoint::handle_current_exception(err_stream); + } +} + +extern "C" void prompp_prometheus_per_goroutine_relabeler_input_relabeling_with_stalenans(void* args, void* res) { struct Arguments { + PromPP::Primitives::Go::SliceView shards_inner_series; + PromPP::Primitives::Go::SliceView shards_relabeled_series; + PromPP::Prometheus::Relabel::RelabelerOptions options; + PerGoroutineRelabelerPtr per_goroutine_relabeler; + PromPP::Prometheus::Relabel::StatelessRelabeler* stateless_relabeler; + HashdexVariant* hashdex; CachePtr cache; + LssVariantPtr input_lss; + LssVariantPtr target_lss; + StaleNaNsStatePtr state; + PromPP::Primitives::Timestamp def_timestamp; + }; + struct Result { + uint32_t samples_added{0}; + uint32_t series_added{0}; + uint32_t series_drop{0}; + PromPP::Primitives::Go::Slice error; + bool target_lss_has_reallocations{}; }; - static_cast(args)->cache->reset(); + auto in = static_cast(args); + auto out = new (res) Result(); + + try { + std::visit( + [in, out](auto& hashdex) { + auto& input_lss = std::get(*in->input_lss); + auto& target_lss = std::get(*in->target_lss); + + const entrypoint::head::ReallocationsDetector reallocation_detector(target_lss); + in->per_goroutine_relabeler->input_relabeling_with_stalenans(input_lss, target_lss, *in->cache, hashdex, in->options, *in->stateless_relabeler, *out, + in->shards_inner_series, in->shards_relabeled_series, *in->state, in->def_timestamp); + target_lss.build_deferred_indexes(); + out->target_lss_has_reallocations = reallocation_detector.has_reallocations(); + }, + *in->hashdex); + } catch (...) { + auto err_stream = PromPP::Primitives::Go::BytesStream(&out->error); + entrypoint::handle_current_exception(err_stream); + } +} + +extern "C" void prompp_prometheus_per_goroutine_relabeler_input_relabeling_with_stalenans_from_cache(void* args, void* res) { + struct Arguments { + PromPP::Primitives::Go::SliceView shards_inner_series; + PromPP::Prometheus::Relabel::RelabelerOptions options; + PerGoroutineRelabelerPtr per_goroutine_relabeler; + HashdexVariant* hashdex; + CachePtr cache; + LssVariantPtr input_lss; + LssVariantPtr target_lss; + StaleNaNsStatePtr state; + PromPP::Primitives::Timestamp def_timestamp; + }; + struct Result { + uint32_t samples_added{0}; + uint32_t series_added{0}; + uint32_t series_drop{0}; + bool ok{}; + PromPP::Primitives::Go::Slice error; + }; + + auto in = static_cast(args); + auto out = new (res) Result(); + + try { + std::visit( + [in, out](auto& hashdex) { + auto& input_lss = std::get(*in->input_lss); + auto& target_lss = std::get(*in->target_lss); + + out->ok = in->per_goroutine_relabeler->input_relabeling_with_stalenans_from_cache(input_lss, target_lss, *in->cache, hashdex, in->options, *out, + in->shards_inner_series, *in->state, in->def_timestamp); + }, + *in->hashdex); + } catch (...) { + auto err_stream = PromPP::Primitives::Go::BytesStream(&out->error); + entrypoint::handle_current_exception(err_stream); + } +} + +extern "C" void prompp_prometheus_per_goroutine_relabeler_input_transition_relabeling(void* args, void* res) { + struct Arguments { + PromPP::Primitives::Go::SliceView shards_inner_series; + PerGoroutineRelabelerPtr per_goroutine_relabeler; + HashdexVariant* hashdex; + LssVariantPtr target_lss; + }; + struct Result { + uint32_t samples_added{0}; + uint32_t series_added{0}; + uint32_t series_drop{0}; + PromPP::Primitives::Go::Slice error; + bool target_lss_has_reallocations{}; + }; + + auto in = static_cast(args); + auto out = new (res) Result(); + + try { + std::visit( + [in, out](auto& hashdex) { + auto& target_lss = std::get(*in->target_lss); + + const entrypoint::head::ReallocationsDetector reallocation_detector(target_lss); + in->per_goroutine_relabeler->input_transition_relabeling(target_lss, hashdex, *out, in->shards_inner_series); + target_lss.build_deferred_indexes(); + out->target_lss_has_reallocations = reallocation_detector.has_reallocations(); + }, + *in->hashdex); + } catch (...) { + auto err_stream = PromPP::Primitives::Go::BytesStream(&out->error); + entrypoint::handle_current_exception(err_stream); + } +} + +extern "C" void prompp_prometheus_per_goroutine_relabeler_input_transition_relabeling_only_read(void* args, void* res) { + struct Arguments { + PromPP::Primitives::Go::SliceView shards_inner_series; + PerGoroutineRelabelerPtr per_goroutine_relabeler; + HashdexVariant* hashdex; + LssVariantPtr target_lss; + }; + struct Result { + uint32_t samples_added{0}; + uint32_t series_added{0}; + uint32_t series_drop{0}; + bool ok{}; + PromPP::Primitives::Go::Slice error; + }; + + auto in = static_cast(args); + auto out = new (res) Result(); + + try { + std::visit( + [in, out](auto& hashdex) { + auto& target_lss = std::get(*in->target_lss); + + out->ok = in->per_goroutine_relabeler->input_transition_relabeling_only_read(target_lss, hashdex, *out, in->shards_inner_series); + }, + *in->hashdex); + } catch (...) { + auto err_stream = PromPP::Primitives::Go::BytesStream(&out->error); + entrypoint::handle_current_exception(err_stream); + } +} + +extern "C" void prompp_prometheus_per_goroutine_relabeler_append_relabeler_series(void* args, void* res) { + struct Arguments { + PromPP::Primitives::Go::SliceView shards_inner_series; + PromPP::Primitives::Go::SliceView shards_relabeled_series; + PromPP::Primitives::Go::SliceView shards_relabeler_state_update; + PerGoroutineRelabelerPtr per_goroutine_relabeler; + LssVariantPtr target_lss; + }; + struct Result { + PromPP::Primitives::Go::Slice error; + bool target_lss_has_reallocations{}; + }; + + const auto in = static_cast(args); + const auto out = new (res) Result(); + + try { + auto& lss = std::get(*in->target_lss); + const entrypoint::head::ReallocationsDetector reallocation_detector(lss); + + for (size_t id = 0; id != in->shards_relabeled_series.size(); ++id) { + if (in->shards_relabeled_series[id] == nullptr || in->shards_relabeled_series[id]->size() == 0) { + continue; + } + + in->per_goroutine_relabeler->append_relabeler_series(lss, in->shards_inner_series[id], in->shards_relabeled_series[id], + in->shards_relabeler_state_update[id]); + } + + lss.build_deferred_indexes(); + out->target_lss_has_reallocations = reallocation_detector.has_reallocations(); + } catch (...) { + auto err_stream = PromPP::Primitives::Go::BytesStream(&out->error); + entrypoint::handle_current_exception(err_stream); + } } diff --git a/pp/entrypoint/prometheus_relabeler.h b/pp/entrypoint/prometheus_relabeler.h index e0f3cc84bf..9c0a26b7aa 100644 --- a/pp/entrypoint/prometheus_relabeler.h +++ b/pp/entrypoint/prometheus_relabeler.h @@ -181,15 +181,6 @@ void prompp_prometheus_relabel_stalenans_state_ctor(void* res); */ void prompp_prometheus_relabel_stalenans_state_dtor(void* args); -/** - * @brief Reset StaleNaNsState. - * - * @param args { - * state uintptr // pointer to StaleNaNsState; - * } - */ -void prompp_prometheus_relabel_stalenans_state_reset(void* args); - /** * @brief relabeling incomig hashdex(first stage) with state stalenans. * @@ -399,13 +390,204 @@ void prompp_prometheus_cache_dtor(void* args); void prompp_prometheus_cache_allocated_memory(void* args, void* res); /** - * @brief reset cache and store lss generation. + * @brief add to cache relabled data(third stage). * * @param args { - * cache uintptr // pointer to constructed Cache; + * shards_relabeler_state_update []*RelabelerStateUpdate // pointer to RelabelerStateUpdate per source shard; + * cache uintptr // pointer to constructed Cache; + * relabeled_shard_id uint16 // relabeled shard id; + * } + * + * @param res { + * error []byte // error string if thrown; + * } + */ +void prompp_prometheus_cache_update(void* args, void* res); + +// +// PerGoroutineRelabeler +// + +/** + * @brief Construct a new PerGoroutineRelabeler. + * + * @param args { + * number_of_shards uint16 // total shards count; + * shard_id uint16 // current shard id; + * } + * + * @param res { + * per_goroutine_relabeler uintptr // pointer to constructed PerGoroutineRelabeler; + * } + */ +void prompp_prometheus_per_goroutine_relabeler_ctor(void* args, void* res); + +/** + * @brief Destroy PerGoroutineRelabeler. + * + * @param args { + * per_goroutine_relabeler uintptr // pointer of PerGoroutineRelabeler; + * } + */ +void prompp_prometheus_per_goroutine_relabeler_dtor(void* args); + +/** + * @brief relabeling incomig hashdex(first stage). + * + * @param args { + * shards_inner_series []*InnerSeries // go slice with InnerSeries; + * shards_relabeled_series []*RelabeledSeries // go slice with RelabeledSeries; + * options RelabelerOptions // object RelabelerOptions; + * per_goroutine_relabeler uintptr // pointer to constructed per goroutine relabeler; + * stateless_relabeler uintptr // pointer to constructed stateless relabeler; + * hashdex uintptr // pointer to filled hashdex; + * cache uintptr // pointer to constructed Cache; + * input_lss uintptr // pointer to constructed input label sets; + * target_lss uintptr // pointer to constructed target label sets; + * } + * + * @param res { + * samples_added uint32 // number of added samples; + * series_added uint32 // number of added series; + * series_drop uint32 // number of dropped series; + * error []byte // error string if thrown; + * target_lss_has_reallocations bool // true if target lss has reallocations + * } + */ +void prompp_prometheus_per_goroutine_relabeler_input_relabeling(void* args, void* res); + +/** + * @brief relabeling incomig hashdex(first stage) from cache. + * + * @param args { + * shards_inner_series []*InnerSeries // go slice with InnerSeries; + * options RelabelerOptions // object RelabelerOptions; + * per_goroutine_relabeler uintptr // pointer to constructed per goroutine relabeler; + * hashdex uintptr // pointer to filled hashdex; + * cache uintptr // pointer to constructed Cache; + * input_lss uintptr // pointer to constructed input label sets; + * target_lss uintptr // pointer to constructed target label sets; + * } + * + * @param res { + * samples_added uint32 // number of added samples; + * series_added uint32 // number of added series; + * series_drop uint32 // number of dropped series; + * ok bool // true if all label set find in cache; + * error []byte // error string if thrown; + * } + */ +void prompp_prometheus_per_goroutine_relabeler_input_relabeling_from_cache(void* args, void* res); + +/** + * @brief relabeling incomig hashdex(first stage) with state stalenans. + * + * @param args { + * shards_inner_series []*InnerSeries // go slice with InnerSeries; + * shards_relabeled_series []*RelabeledSeries // go slice with RelabeledSeries; + * options RelabelerOptions // object RelabelerOptions; + * per_goroutine_relabeler uintptr // pointer to constructed per goroutine relabeler; + * stateless_relabeler uintptr // pointer to constructed stateless relabeler; + * hashdex uintptr // pointer to filled hashdex; + * cache uintptr // pointer to constructed Cache; + * input_lss uintptr // pointer to constructed input label sets; + * target_lss uintptr // pointer to constructed target label sets; + * state uintptr // pointer to source state + * def_timestamp int64 // timestamp for metrics and StaleNaNs + * } + * + * @param res { + * samples_added uint32 // number of added samples; + * series_added uint32 // number of added series; + * series_drop uint32 // number of dropped series; + * error []byte // error string if thrown; + * target_lss_has_reallocations bool // true if target lss has reallocations + * } + */ +void prompp_prometheus_per_goroutine_relabeler_input_relabeling_with_stalenans(void* args, void* res); + +/** + * @brief relabeling incomig hashdex(first stage) from cache with state stalenans. + * + * @param args { + * shards_inner_series []*InnerSeries // go slice with InnerSeries; + * options RelabelerOptions // object RelabelerOptions; + * per_goroutine_relabeler uintptr // pointer to constructed per goroutine relabeler; + * hashdex uintptr // pointer to filled hashdex; + * cache uintptr // pointer to constructed Cache; + * input_lss uintptr // pointer to constructed input label sets; + * target_lss uintptr // pointer to constructed target label sets; + * state uintptr // pointer to source state + * def_timestamp int64 // timestamp for metrics and StaleNaNs + * } + * + * @param res { + * samples_added uint32 // number of added samples; + * series_added uint32 // number of added series; + * series_drop uint32 // number of dropped series; + * ok bool // true if all label set find in cache; + * error []byte // error string if thrown; + * } + */ +void prompp_prometheus_per_goroutine_relabeler_input_relabeling_with_stalenans_from_cache(void* args, void* res); + +/** + * @brief transparent relabeling incomig hashdex(first stage). + * + * @param args { + * shards_inner_series []*InnerSeries // go slice with InnerSeries; + * per_goroutine_relabeler uintptr // pointer to constructed per goroutine relabeler; + * hashdex uintptr // pointer to filled hashdex; + * target_lss uintptr // pointer to constructed target label sets; + * } + * + * @param res { + * samples_added uint32 // number of added samples; + * series_added uint32 // number of added series; + * series_drop uint32 // number of dropped series; + * error []byte // error string if thrown; + * target_lss_has_reallocations bool // true if target lss has reallocations + * } + */ +void prompp_prometheus_per_goroutine_relabeler_input_transition_relabeling(void* args, void* res); + +/** + * @brief transparent relabeling incomig hashdex(first stage) from cache. + * + * @param args { + * shards_inner_series []*InnerSeries // go slice with InnerSeries; + * per_goroutine_relabeler uintptr // pointer to constructed per goroutine relabeler; + * hashdex uintptr // pointer to filled hashdex; + * target_lss uintptr // pointer to constructed target label sets; + * } + * + * @param res { + * samples_added uint32 // number of added samples; + * series_added uint32 // number of added series; + * series_drop uint32 // number of dropped series; + * ok bool // true if all label set find in cache; + * error []byte // error string if thrown; + * } + */ +void prompp_prometheus_per_goroutine_relabeler_input_transition_relabeling_only_read(void* args, void* res); + +/** + * @brief add relabeled ls to lss, add to result and add to cache update(second stage). + * + * @param args { + * shards_inner_series []*InnerSeries // go InnerSeries per source shard; + * shards_relabeled_series []*RelabeledSeries // go RelabeledSeries per source shard; + * shards_relabeler_state_update []*RelabelerStateUpdate // pointer to RelabelerStateUpdate per source shard; + * per_goroutine_relabeler uintptr // pointer to constructed per goroutine relabeler; + * target_lss uintptr // pointer to constructed label sets; + * } + * + * @param res { + * error []byte // error string if thrown + * target_lss_has_reallocations bool // true if target lss has reallocations * } */ -void prompp_prometheus_cache_reset_to(void* args); +void prompp_prometheus_per_goroutine_relabeler_append_relabeler_series(void* args, void* res); #ifdef __cplusplus } // extern "C" diff --git a/pp/go/cppbridge/c_garbage_collector.go b/pp/go/cppbridge/c_garbage_collector.go index bd92f09afe..323a2e78e1 100644 --- a/pp/go/cppbridge/c_garbage_collector.go +++ b/pp/go/cppbridge/c_garbage_collector.go @@ -5,8 +5,9 @@ import ( "runtime" "time" - "github.com/prometheus/prometheus/pp/go/util" "github.com/prometheus/client_golang/prometheus" + + "github.com/prometheus/prometheus/pp/go/util" ) // garbage collector for objects initiated in GO but filled in C/C++, diff --git a/pp/go/cppbridge/entrypoint.go b/pp/go/cppbridge/entrypoint.go index 187fea0369..216c26ff99 100644 --- a/pp/go/cppbridge/entrypoint.go +++ b/pp/go/cppbridge/entrypoint.go @@ -98,6 +98,172 @@ var ( }, ) + // per_goroutine_relabeler input_relabeling + perGoroutineRelabelerInputRelabelingSum = util.NewUnconflictRegisterer(prometheus.DefaultRegisterer).NewCounter( + prometheus.CounterOpts{ + Name: "prompp_cppbridge_unsafecall_nanoseconds_sum", + Help: "The time duration cpp call.", + ConstLabels: prometheus.Labels{"object": "per_goroutine_relabeler", "method": "input_relabeling"}, + }, + ) + perGoroutineRelabelerInputRelabelingCount = util.NewUnconflictRegisterer(prometheus.DefaultRegisterer).NewCounter( + prometheus.CounterOpts{ + Name: "prompp_cppbridge_unsafecall_nanoseconds_count", + Help: "The time duration cpp call.", + ConstLabels: prometheus.Labels{"object": "per_goroutine_relabeler", "method": "input_relabeling"}, + }, + ) + + // per_goroutine_relabeler input_relabeling_from_cache + perGoroutineRelabelerInputRelabelingFromCacheSum = util.NewUnconflictRegisterer( + prometheus.DefaultRegisterer, + ).NewCounter( + prometheus.CounterOpts{ + Name: "prompp_cppbridge_unsafecall_nanoseconds_sum", + Help: "The time duration cpp call.", + ConstLabels: prometheus.Labels{ + "object": "per_goroutine_relabeler", + "method": "input_relabeling_from_cache", + }, + }, + ) + perGoroutineRelabelerInputRelabelingFromCacheCount = util.NewUnconflictRegisterer( + prometheus.DefaultRegisterer, + ).NewCounter( + prometheus.CounterOpts{ + Name: "prompp_cppbridge_unsafecall_nanoseconds_count", + Help: "The time duration cpp call.", + ConstLabels: prometheus.Labels{ + "object": "per_goroutine_relabeler", + "method": "input_relabeling_from_cache", + }, + }, + ) + + // per_goroutine_relabeler relabeling_with_stalenans + perGoroutineRelabelerInputRelabelingWithStalenansSum = util.NewUnconflictRegisterer( + prometheus.DefaultRegisterer, + ).NewCounter( + prometheus.CounterOpts{ + Name: "prompp_cppbridge_unsafecall_nanoseconds_sum", + Help: "The time duration cpp call.", + ConstLabels: prometheus.Labels{ + "object": "per_goroutine_relabeler", + "method": "relabeling_with_stalenans", + }, + }, + ) + perGoroutineRelabelerInputRelabelingWithStalenansCount = util.NewUnconflictRegisterer( + prometheus.DefaultRegisterer, + ).NewCounter( + prometheus.CounterOpts{ + Name: "prompp_cppbridge_unsafecall_nanoseconds_count", + Help: "The time duration cpp call.", + ConstLabels: prometheus.Labels{ + "object": "per_goroutine_relabeler", + "method": "relabeling_with_stalenans", + }, + }, + ) + + // per_goroutine_relabeler relabeling_with_stalenans_from_cache + perGoroutineRelabelerInputRelabelingWithStalenansFromCacheSum = util.NewUnconflictRegisterer( + prometheus.DefaultRegisterer, + ).NewCounter( + prometheus.CounterOpts{ + Name: "prompp_cppbridge_unsafecall_nanoseconds_sum", + Help: "The time duration cpp call.", + ConstLabels: prometheus.Labels{ + "object": "per_goroutine_relabeler", + "method": "relabeling_with_stalenans_from_cache", + }, + }, + ) + perGoroutineRelabelerInputRelabelingWithStalenansFromCacheCount = util.NewUnconflictRegisterer( + prometheus.DefaultRegisterer, + ).NewCounter( + prometheus.CounterOpts{ + Name: "prompp_cppbridge_unsafecall_nanoseconds_count", + Help: "The time duration cpp call.", + ConstLabels: prometheus.Labels{ + "object": "per_goroutine_relabeler", + "method": "relabeling_with_stalenans_from_cache", + }, + }, + ) + + // per_goroutine_relabeler input_transition_relabeling + perGoroutineRelabelerInputTransitionRelabelingSum = util.NewUnconflictRegisterer( + prometheus.DefaultRegisterer, + ).NewCounter( + prometheus.CounterOpts{ + Name: "prompp_cppbridge_unsafecall_nanoseconds_sum", + Help: "The time duration cpp call.", + ConstLabels: prometheus.Labels{ + "object": "per_goroutine_relabeler", + "method": "input_transition_relabeling", + }, + }, + ) + perGoroutineRelabelerInputTransitionRelabelingCount = util.NewUnconflictRegisterer( + prometheus.DefaultRegisterer, + ).NewCounter( + prometheus.CounterOpts{ + Name: "prompp_cppbridge_unsafecall_nanoseconds_count", + Help: "The time duration cpp call.", + ConstLabels: prometheus.Labels{ + "object": "per_goroutine_relabeler", + "method": "input_transition_relabeling", + }, + }, + ) + + // per_goroutine_relabeler input_transition_relabeling_only_read + perGoroutineRelabelerInputTransitionRelabelingOnlyReadSum = util.NewUnconflictRegisterer( + prometheus.DefaultRegisterer, + ).NewCounter( + prometheus.CounterOpts{ + Name: "prompp_cppbridge_unsafecall_nanoseconds_sum", + Help: "The time duration cpp call.", + ConstLabels: prometheus.Labels{ + "object": "per_goroutine_relabeler", + "method": "input_transition_relabeling_only_read", + }, + }, + ) + perGoroutineRelabelerInputTransitionRelabelingOnlyReadCount = util.NewUnconflictRegisterer( + prometheus.DefaultRegisterer, + ).NewCounter( + prometheus.CounterOpts{ + Name: "prompp_cppbridge_unsafecall_nanoseconds_count", + Help: "The time duration cpp call.", + ConstLabels: prometheus.Labels{ + "object": "per_goroutine_relabeler", + "method": "input_transition_relabeling_only_read", + }, + }, + ) + + // per_goroutine_relabeler append_relabeler_series + perGoroutineRelabelerAppendRelabelerSeriesSum = util.NewUnconflictRegisterer( + prometheus.DefaultRegisterer, + ).NewCounter( + prometheus.CounterOpts{ + Name: "prompp_cppbridge_unsafecall_nanoseconds_sum", + Help: "The time duration cpp call.", + ConstLabels: prometheus.Labels{"object": "per_goroutine_relabeler", "method": "append_relabeler_series"}, + }, + ) + perGoroutineRelabelerAppendRelabelerSeriesCount = util.NewUnconflictRegisterer( + prometheus.DefaultRegisterer, + ).NewCounter( + prometheus.CounterOpts{ + Name: "prompp_cppbridge_unsafecall_nanoseconds_count", + Help: "The time duration cpp call.", + ConstLabels: prometheus.Labels{"object": "per_goroutine_relabeler", "method": "append_relabeler_series"}, + }, + ) + // input_relabeler append_relabeler_series inputRelabelerAppendRelabelerSeriesSum = util.NewUnconflictRegisterer(prometheus.DefaultRegisterer).NewCounter( prometheus.CounterOpts{ @@ -1361,6 +1527,48 @@ func primitivesLSSCreateReadonlyLss(lss uintptr) uintptr { return res.lss } +// primitivesLSSBitsetSeries returns a copy of the bitset of added series from the lss. +func primitivesLSSBitsetSeries(lss uintptr) uintptr { + args := struct { + lss uintptr + }{lss} + var res struct { + bitset uintptr + } + + testGC() + fastcgo.UnsafeCall2( + C.prompp_primitives_lss_bitset_series, + uintptr(unsafe.Pointer(&args)), + uintptr(unsafe.Pointer(&res)), + ) + + return res.bitset +} + +// primitivesLSSBitsetDtor destroy bitset of added series. +func primitivesLSSBitsetDtor(bitset uintptr) { + args := struct { + bitset uintptr + }{bitset} + + testGC() + fastcgo.UnsafeCall1( + C.prompp_primitives_lss_bitset_dtor, + uintptr(unsafe.Pointer(&args)), + ) +} + +// primitivesReadonlyLSSCopyAddedSeries copy the label sets from the source lss to the destination lss +// that were added source lss. +func primitivesReadonlyLSSCopyAddedSeries(source, sourceBitset, destination uintptr) { + C.prompp_primitives_readonly_lss_copy_added_series( + C.uint64_t(source), + C.uint64_t(sourceBitset), + C.uint64_t(destination), + ) +} + func primitivesLSSCopyAddedSeries(source, destination uintptr) { C.prompp_primitives_lss_copy_added_series(C.uint64_t(source), C.uint64_t(destination)) } @@ -1530,18 +1738,6 @@ func prometheusRelabelStaleNansStateCtor() uintptr { return res.state } -func prometheusRelabelStaleNansStateReset(state uintptr) { - args := struct { - state uintptr - }{state} - - testGC() - fastcgo.UnsafeCall1( - C.prompp_prometheus_relabel_stalenans_state_reset, - uintptr(unsafe.Pointer(&args)), - ) -} - func prometheusRelabelStaleNansStateDtor(state uintptr) { args := struct { state uintptr @@ -2880,17 +3076,29 @@ func prometheusCacheAllocatedMemory(cache uintptr) uint64 { return res.cacheAllocatedMemory } -// prometheusCacheResetTo reset cache. -func prometheusCacheResetTo(cache uintptr) { +// prometheusCacheUpdate add to cache relabled data(third stage). +func prometheusCacheUpdate( + shardsRelabelerStateUpdate []*RelabelerStateUpdate, + cache uintptr, +) []byte { args := struct { - cache uintptr - }{cache} - + relabelerStateUpdates []*RelabelerStateUpdate + cache uintptr + }{shardsRelabelerStateUpdate, cache} + var res struct { + exception []byte + } + start := time.Now().UnixNano() testGC() - fastcgo.UnsafeCall1( - C.prompp_prometheus_cache_reset_to, + fastcgo.UnsafeCall2( + C.prompp_prometheus_cache_update, uintptr(unsafe.Pointer(&args)), + uintptr(unsafe.Pointer(&res)), ) + inputRelabelerUpdateRelabelerStateSum.Add(float64(time.Now().UnixNano() - start)) + inputRelabelerUpdateRelabelerStateCount.Inc() + + return res.exception } func headWalEncoderCtor(shardID uint16, logShards uint8, lss uintptr) uintptr { @@ -2914,14 +3122,14 @@ func headWalEncoderCtor(shardID uint16, logShards uint8, lss uintptr) uintptr { return res.encoder } -func headWalEncoderAddInnerSeries(encoder uintptr, innerSeries []*InnerSeries) (stats WALEncoderStats, err error) { +func headWalEncoderAddInnerSeries(encoder uintptr, innerSeries []*InnerSeries) (samples uint32, err error) { args := struct { innerSeries []*InnerSeries encoder uintptr }{innerSeries, encoder} var res struct { - WALEncoderStats exception []byte + samples uint32 } start := time.Now().UnixNano() @@ -2934,18 +3142,18 @@ func headWalEncoderAddInnerSeries(encoder uintptr, innerSeries []*InnerSeries) ( headWalEncoderAddInnerSeriesSum.Add(float64(time.Now().UnixNano() - start)) headWalEncoderAddInnerSeriesCount.Inc() - return res.WALEncoderStats, handleException(res.exception) + return res.samples, handleException(res.exception) } // headWalEncoderFinalize - finalize the encoded data in the C++ encoder to Segment. -func headWalEncoderFinalize(encoder uintptr) (stats WALEncoderStats, segment []byte, err error) { +func headWalEncoderFinalize(encoder uintptr) (samples uint32, segment []byte, err error) { args := struct { encoder uintptr }{encoder} var res struct { - WALEncoderStats segment []byte exception []byte + samples uint32 } start := time.Now().UnixNano() @@ -2958,7 +3166,7 @@ func headWalEncoderFinalize(encoder uintptr) (stats WALEncoderStats, segment []b headWalEncoderFinalizeSum.Add(float64(time.Now().UnixNano() - start)) headWalEncoderFinalizeCount.Inc() - return res.WALEncoderStats, res.segment, handleException(res.exception) + return res.samples, res.segment, handleException(res.exception) } func headWalEncoderDtor(encoder uintptr) { @@ -3195,3 +3403,318 @@ func LabelSetBytesWithLabels(lss uintptr, labelSetID uint32, bytes []byte, names func LabelSetBytesWithoutLabels(lss uintptr, labelSetID uint32, bytes []byte, names ...string) []byte { return labelSetBytesWithFilteredNames(C.prompp_label_set_bytes_without_labels, lss, labelSetID, bytes, names...) } + +// +// PerGoroutineRelabeler +// + +// prometheusPerGoroutineRelabelerCtor wrapper for constructor C-PerGoroutineRelabeler. +func prometheusPerGoroutineRelabelerCtor( + numberOfShards, shardID uint16, +) uintptr { + args := struct { + numberOfShards uint16 + shardID uint16 + }{numberOfShards, shardID} + var res struct { + perGoroutineRelabeler uintptr + } + + testGC() + fastcgo.UnsafeCall2( + C.prompp_prometheus_per_goroutine_relabeler_ctor, + uintptr(unsafe.Pointer(&args)), + uintptr(unsafe.Pointer(&res)), + ) + + return res.perGoroutineRelabeler +} + +// prometheusPerGoroutineRelabelerDtor wrapper for destructor C-PerGoroutineRelabeler. +func prometheusPerGoroutineRelabelerDtor(perGoroutineRelabeler uintptr) { + args := struct { + perGoroutineRelabeler uintptr + }{perGoroutineRelabeler} + + testGC() + fastcgo.UnsafeCall1( + C.prompp_prometheus_per_goroutine_relabeler_dtor, + uintptr(unsafe.Pointer(&args)), + ) +} + +// prometheusPerGoroutineRelabelerInputRelabeling wrapper for relabeling incoming hashdex(first stage). +func prometheusPerGoroutineRelabelerInputRelabeling( + perGoroutineRelabeler, statelessRelabeler, inputLss, targetLss, cache, hashdex uintptr, + options RelabelerOptions, + shardsInnerSeries []*InnerSeries, + shardsRelabeledSeries []*RelabeledSeries, +) (stats RelabelerStats, exception []byte, targetLssHasReallocations bool) { + args := struct { + shardsInnerSeries []*InnerSeries + shardsRelabeledSeries []*RelabeledSeries + options RelabelerOptions + perGoroutineRelabeler uintptr + statelessRelabeler uintptr + hashdex uintptr + cache uintptr + inputLss uintptr + targetLss uintptr + }{ + shardsInnerSeries, + shardsRelabeledSeries, + options, + perGoroutineRelabeler, + statelessRelabeler, + hashdex, + cache, + inputLss, + targetLss, + } + var res struct { + RelabelerStats + exception []byte + targetLssHasReallocations bool + } + start := time.Now().UnixNano() + testGC() + fastcgo.UnsafeCall2( + C.prompp_prometheus_per_goroutine_relabeler_input_relabeling, + uintptr(unsafe.Pointer(&args)), + uintptr(unsafe.Pointer(&res)), + ) + perGoroutineRelabelerInputRelabelingSum.Add(float64(time.Now().UnixNano() - start)) + perGoroutineRelabelerInputRelabelingCount.Inc() + + return res.RelabelerStats, res.exception, res.targetLssHasReallocations +} + +// prometheusPerGoroutineRelabelerInputRelabelingFromCache wrapper for relabeling +// incoming hashdex(first stage) from cache. +func prometheusPerGoroutineRelabelerInputRelabelingFromCache( + perGoroutineRelabeler, inputLss, targetLss, cache, hashdex uintptr, + options RelabelerOptions, + shardsInnerSeries []*InnerSeries, +) (stats RelabelerStats, exception []byte, ok bool) { + args := struct { + shardsInnerSeries []*InnerSeries + options RelabelerOptions + perGoroutineRelabeler uintptr + hashdex uintptr + cache uintptr + inputLss uintptr + targetLss uintptr + }{shardsInnerSeries, options, perGoroutineRelabeler, hashdex, cache, inputLss, targetLss} + var res struct { + RelabelerStats + ok bool + exception []byte + } + start := time.Now().UnixNano() + testGC() + fastcgo.UnsafeCall2( + C.prompp_prometheus_per_goroutine_relabeler_input_relabeling_from_cache, + uintptr(unsafe.Pointer(&args)), + uintptr(unsafe.Pointer(&res)), + ) + perGoroutineRelabelerInputRelabelingFromCacheSum.Add(float64(time.Now().UnixNano() - start)) + perGoroutineRelabelerInputRelabelingFromCacheCount.Inc() + + return res.RelabelerStats, res.exception, res.ok +} + +// prometheusPerGoroutineRelabelerInputRelabelingWithStalenans wrapper for relabeling incoming +// hashdex(first stage) with state stalenans. +func prometheusPerGoroutineRelabelerInputRelabelingWithStalenans( + perGoroutineRelabeler, statelessRelabeler, inputLss, targetLss, cache, hashdex, sourceState uintptr, + defTimestamp int64, + options RelabelerOptions, + shardsInnerSeries []*InnerSeries, + shardsRelabeledSeries []*RelabeledSeries, +) (stats RelabelerStats, exception []byte, targetLssHasReallocations bool) { + args := struct { + shardsInnerSeries []*InnerSeries + shardsRelabeledSeries []*RelabeledSeries + options RelabelerOptions + perGoroutineRelabeler uintptr + statelessRelabeler uintptr + hashdex uintptr + cache uintptr + inputLss uintptr + targetLss uintptr + state uintptr + defTimestamp int64 + }{ + shardsInnerSeries, + shardsRelabeledSeries, + options, + perGoroutineRelabeler, + statelessRelabeler, + hashdex, + cache, + inputLss, + targetLss, + sourceState, + defTimestamp, + } + var res struct { + RelabelerStats + exception []byte + targetLssHasReallocations bool + } + start := time.Now().UnixNano() + testGC() + fastcgo.UnsafeCall2( + C.prompp_prometheus_per_goroutine_relabeler_input_relabeling_with_stalenans, + uintptr(unsafe.Pointer(&args)), + uintptr(unsafe.Pointer(&res)), + ) + perGoroutineRelabelerInputRelabelingWithStalenansSum.Add(float64(time.Now().UnixNano() - start)) + perGoroutineRelabelerInputRelabelingWithStalenansCount.Inc() + + return res.RelabelerStats, res.exception, res.targetLssHasReallocations +} + +// prometheusPerGoroutineRelabelerInputRelabelingWithStalenansFromCache wrapper for relabeling incoming from cache +// hashdex(first stage) with state stalenans. +func prometheusPerGoroutineRelabelerInputRelabelingWithStalenansFromCache( + perGoroutineRelabeler, inputLss, targetLss, cache, hashdex, sourceState uintptr, + defTimestamp int64, + options RelabelerOptions, + shardsInnerSeries []*InnerSeries, +) (stats RelabelerStats, exception []byte, targetLssHasReallocations bool) { + args := struct { + shardsInnerSeries []*InnerSeries + options RelabelerOptions + perGoroutineRelabeler uintptr + hashdex uintptr + cache uintptr + inputLss uintptr + targetLss uintptr + state uintptr + defTimestamp int64 + }{ + shardsInnerSeries, + options, + perGoroutineRelabeler, + hashdex, + cache, + inputLss, + targetLss, + sourceState, + defTimestamp, + } + var res struct { + RelabelerStats + ok bool + exception []byte + } + start := time.Now().UnixNano() + testGC() + fastcgo.UnsafeCall2( + C.prompp_prometheus_per_goroutine_relabeler_input_relabeling_with_stalenans_from_cache, + uintptr(unsafe.Pointer(&args)), + uintptr(unsafe.Pointer(&res)), + ) + perGoroutineRelabelerInputRelabelingWithStalenansFromCacheSum.Add(float64(time.Now().UnixNano() - start)) + perGoroutineRelabelerInputRelabelingWithStalenansFromCacheCount.Inc() + + return res.RelabelerStats, res.exception, res.ok +} + +// prometheusPerGoroutineRelabelerInputTransitionRelabeling wrapper for +// transparent relabeling incoming hashdex(first stage). +func prometheusPerGoroutineRelabelerInputTransitionRelabeling( + perGoroutineRelabeler, targetLss, hashdex uintptr, + shardsInnerSeries []*InnerSeries, +) (stats RelabelerStats, exception []byte, targetLssHasReallocations bool) { + args := struct { + shardsInnerSeries []*InnerSeries + perGoroutineRelabeler uintptr + hashdex uintptr + targetLss uintptr + }{ + shardsInnerSeries, + perGoroutineRelabeler, + hashdex, + targetLss, + } + var res struct { + RelabelerStats + exception []byte + targetLssHasReallocations bool + } + start := time.Now().UnixNano() + testGC() + fastcgo.UnsafeCall2( + C.prompp_prometheus_per_goroutine_relabeler_input_transition_relabeling, + uintptr(unsafe.Pointer(&args)), + uintptr(unsafe.Pointer(&res)), + ) + perGoroutineRelabelerInputTransitionRelabelingSum.Add(float64(time.Now().UnixNano() - start)) + perGoroutineRelabelerInputTransitionRelabelingCount.Inc() + + return res.RelabelerStats, res.exception, res.targetLssHasReallocations +} + +// prometheusPerGoroutineRelabelerInputRelabelingOnlyRead wrapper for transparent relabeling +// incoming hashdex(first stage) from cache. +func prometheusPerGoroutineRelabelerInputRelabelingOnlyRead( + perGoroutineRelabeler, targetLss, hashdex uintptr, + shardsInnerSeries []*InnerSeries, +) (stats RelabelerStats, exception []byte, ok bool) { + args := struct { + shardsInnerSeries []*InnerSeries + perGoroutineRelabeler uintptr + hashdex uintptr + targetLss uintptr + }{shardsInnerSeries, perGoroutineRelabeler, hashdex, targetLss} + var res struct { + RelabelerStats + ok bool + exception []byte + } + start := time.Now().UnixNano() + testGC() + fastcgo.UnsafeCall2( + C.prompp_prometheus_per_goroutine_relabeler_input_transition_relabeling_only_read, + uintptr(unsafe.Pointer(&args)), + uintptr(unsafe.Pointer(&res)), + ) + perGoroutineRelabelerInputTransitionRelabelingOnlyReadSum.Add(float64(time.Now().UnixNano() - start)) + perGoroutineRelabelerInputTransitionRelabelingOnlyReadCount.Inc() + + return res.RelabelerStats, res.exception, res.ok +} + +// prometheusPerGoroutineRelabelerAppendRelabelerSeries wrapper for add relabeled ls to lss, +// add to result and add to cache update(second stage). +func prometheusPerGoroutineRelabelerAppendRelabelerSeries( + perGoroutineRelabeler, targetLss uintptr, + shardsInnerSeries []*InnerSeries, + shardsRelabeledSeries []*RelabeledSeries, + shardsRelabelerStateUpdate []*RelabelerStateUpdate, +) (exception []byte, targetLssHasReallocations bool) { + args := struct { + shardsInnerSeries []*InnerSeries + shardsRelabeledSeries []*RelabeledSeries + shardsRelabelerStateUpdate []*RelabelerStateUpdate + perGoroutineRelabeler uintptr + targetLss uintptr + }{shardsInnerSeries, shardsRelabeledSeries, shardsRelabelerStateUpdate, perGoroutineRelabeler, targetLss} + var res struct { + exception []byte + targetLssHasReallocations bool + } + start := time.Now().UnixNano() + testGC() + fastcgo.UnsafeCall2( + C.prompp_prometheus_per_goroutine_relabeler_append_relabeler_series, + uintptr(unsafe.Pointer(&args)), + uintptr(unsafe.Pointer(&res)), + ) + perGoroutineRelabelerAppendRelabelerSeriesSum.Add(float64(time.Now().UnixNano() - start)) + perGoroutineRelabelerAppendRelabelerSeriesCount.Inc() + + return res.exception, res.targetLssHasReallocations +} diff --git a/pp/go/cppbridge/entrypoint.h b/pp/go/cppbridge/entrypoint.h index a59048052c..d699fd2a53 100755 --- a/pp/go/cppbridge/entrypoint.h +++ b/pp/go/cppbridge/entrypoint.h @@ -150,13 +150,8 @@ void prompp_head_wal_encoder_dtor(void* args); * encoder uintptr // pointer to constructed encoder; * } * @param res { - * earliestTimestamp int64 // minimal sample timestamp in segment - * latestTimestamp int64 // maximal sample timestamp in segment - * allocatedMemory uint64 // size of allocated memory for label sets; - * samples uint32 // number of samples in segment - * series uint32 // number of series in segment - * remainderSize uint32 // rest of internal buffers capacity * error []byte // error string if thrown + * samples uint32 // number of samples in segment * } */ void prompp_head_wal_encoder_add_inner_series(void* args, void* res); @@ -168,14 +163,9 @@ void prompp_head_wal_encoder_add_inner_series(void* args, void* res); * encoder uintptr // pointer to constructed encoder * } * @param res { - * earliestTimestamp int64 // minimal sample timestamp in segment - * latestTimestamp int64 // maximal sample timestamp in segment - * allocatedMemory uint64 // size of allocated memory for label sets; - * samples uint32 // number of samples in segment - * series uint32 // number of series in segment - * remainderSize uint32 // rest of internal buffers capacity * segment []byte // segment content * error []byte // error string if thrown + * samples uint32 // number of samples in segment * } */ void prompp_head_wal_encoder_finalize(void* args, void* res); @@ -675,6 +665,41 @@ void prompp_create_readonly_lss(void* args, void* res); */ void prompp_primitives_lss_copy_added_series(uint64_t source_lss, uint64_t destination_lss); +/** + * @brief returns a copy of the bitset of added series from the lss. + * + * @param args { + * lss uintptr // pointer to constructed queryable lss; + * } + * + * @param res { + * bitset uintptr // bitset of added series; + * } + */ +void prompp_primitives_lss_bitset_series(void* args, void* res); + +/** + * @brief destroy bitset of added series. + * + * @param args { + * bitset uintptr // bitset of added series; + * } + * + */ +void prompp_primitives_lss_bitset_dtor(void* args); + +/** + * @brief Copy the label sets from the source lss to the destination lss that were added source lss. + * + * @param source_lss pointer to source label sets; + * @param source_bitset pointer to source bitset; + * @param destination_lss pointer to destination label sets; + * + * @attention This binding used as a CGO call!!! + * + */ +void prompp_primitives_readonly_lss_copy_added_series(uint64_t source_lss, uint64_t source_bitset, uint64_t destination_lss); + #ifdef __cplusplus } // extern "C" #endif @@ -861,15 +886,6 @@ void prompp_prometheus_relabel_stalenans_state_ctor(void* res); */ void prompp_prometheus_relabel_stalenans_state_dtor(void* args); -/** - * @brief Reset StaleNaNsState. - * - * @param args { - * state uintptr // pointer to StaleNaNsState; - * } - */ -void prompp_prometheus_relabel_stalenans_state_reset(void* args); - /** * @brief relabeling incomig hashdex(first stage) with state stalenans. * @@ -1079,13 +1095,204 @@ void prompp_prometheus_cache_dtor(void* args); void prompp_prometheus_cache_allocated_memory(void* args, void* res); /** - * @brief reset cache and store lss generation. + * @brief add to cache relabled data(third stage). * * @param args { - * cache uintptr // pointer to constructed Cache; + * shards_relabeler_state_update []*RelabelerStateUpdate // pointer to RelabelerStateUpdate per source shard; + * cache uintptr // pointer to constructed Cache; + * relabeled_shard_id uint16 // relabeled shard id; + * } + * + * @param res { + * error []byte // error string if thrown; + * } + */ +void prompp_prometheus_cache_update(void* args, void* res); + +// +// PerGoroutineRelabeler +// + +/** + * @brief Construct a new PerGoroutineRelabeler. + * + * @param args { + * number_of_shards uint16 // total shards count; + * shard_id uint16 // current shard id; + * } + * + * @param res { + * per_goroutine_relabeler uintptr // pointer to constructed PerGoroutineRelabeler; + * } + */ +void prompp_prometheus_per_goroutine_relabeler_ctor(void* args, void* res); + +/** + * @brief Destroy PerGoroutineRelabeler. + * + * @param args { + * per_goroutine_relabeler uintptr // pointer of PerGoroutineRelabeler; + * } + */ +void prompp_prometheus_per_goroutine_relabeler_dtor(void* args); + +/** + * @brief relabeling incomig hashdex(first stage). + * + * @param args { + * shards_inner_series []*InnerSeries // go slice with InnerSeries; + * shards_relabeled_series []*RelabeledSeries // go slice with RelabeledSeries; + * options RelabelerOptions // object RelabelerOptions; + * per_goroutine_relabeler uintptr // pointer to constructed per goroutine relabeler; + * stateless_relabeler uintptr // pointer to constructed stateless relabeler; + * hashdex uintptr // pointer to filled hashdex; + * cache uintptr // pointer to constructed Cache; + * input_lss uintptr // pointer to constructed input label sets; + * target_lss uintptr // pointer to constructed target label sets; + * } + * + * @param res { + * samples_added uint32 // number of added samples; + * series_added uint32 // number of added series; + * series_drop uint32 // number of dropped series; + * error []byte // error string if thrown; + * target_lss_has_reallocations bool // true if target lss has reallocations + * } + */ +void prompp_prometheus_per_goroutine_relabeler_input_relabeling(void* args, void* res); + +/** + * @brief relabeling incomig hashdex(first stage) from cache. + * + * @param args { + * shards_inner_series []*InnerSeries // go slice with InnerSeries; + * options RelabelerOptions // object RelabelerOptions; + * per_goroutine_relabeler uintptr // pointer to constructed per goroutine relabeler; + * hashdex uintptr // pointer to filled hashdex; + * cache uintptr // pointer to constructed Cache; + * input_lss uintptr // pointer to constructed input label sets; + * target_lss uintptr // pointer to constructed target label sets; + * } + * + * @param res { + * samples_added uint32 // number of added samples; + * series_added uint32 // number of added series; + * series_drop uint32 // number of dropped series; + * ok bool // true if all label set find in cache; + * error []byte // error string if thrown; + * } + */ +void prompp_prometheus_per_goroutine_relabeler_input_relabeling_from_cache(void* args, void* res); + +/** + * @brief relabeling incomig hashdex(first stage) with state stalenans. + * + * @param args { + * shards_inner_series []*InnerSeries // go slice with InnerSeries; + * shards_relabeled_series []*RelabeledSeries // go slice with RelabeledSeries; + * options RelabelerOptions // object RelabelerOptions; + * per_goroutine_relabeler uintptr // pointer to constructed per goroutine relabeler; + * stateless_relabeler uintptr // pointer to constructed stateless relabeler; + * hashdex uintptr // pointer to filled hashdex; + * cache uintptr // pointer to constructed Cache; + * input_lss uintptr // pointer to constructed input label sets; + * target_lss uintptr // pointer to constructed target label sets; + * state uintptr // pointer to source state + * def_timestamp int64 // timestamp for metrics and StaleNaNs + * } + * + * @param res { + * samples_added uint32 // number of added samples; + * series_added uint32 // number of added series; + * series_drop uint32 // number of dropped series; + * error []byte // error string if thrown; + * target_lss_has_reallocations bool // true if target lss has reallocations + * } + */ +void prompp_prometheus_per_goroutine_relabeler_input_relabeling_with_stalenans(void* args, void* res); + +/** + * @brief relabeling incomig hashdex(first stage) from cache with state stalenans. + * + * @param args { + * shards_inner_series []*InnerSeries // go slice with InnerSeries; + * options RelabelerOptions // object RelabelerOptions; + * per_goroutine_relabeler uintptr // pointer to constructed per goroutine relabeler; + * hashdex uintptr // pointer to filled hashdex; + * cache uintptr // pointer to constructed Cache; + * input_lss uintptr // pointer to constructed input label sets; + * target_lss uintptr // pointer to constructed target label sets; + * state uintptr // pointer to source state + * def_timestamp int64 // timestamp for metrics and StaleNaNs + * } + * + * @param res { + * samples_added uint32 // number of added samples; + * series_added uint32 // number of added series; + * series_drop uint32 // number of dropped series; + * ok bool // true if all label set find in cache; + * error []byte // error string if thrown; + * } + */ +void prompp_prometheus_per_goroutine_relabeler_input_relabeling_with_stalenans_from_cache(void* args, void* res); + +/** + * @brief transparent relabeling incomig hashdex(first stage). + * + * @param args { + * shards_inner_series []*InnerSeries // go slice with InnerSeries; + * per_goroutine_relabeler uintptr // pointer to constructed per goroutine relabeler; + * hashdex uintptr // pointer to filled hashdex; + * target_lss uintptr // pointer to constructed target label sets; + * } + * + * @param res { + * samples_added uint32 // number of added samples; + * series_added uint32 // number of added series; + * series_drop uint32 // number of dropped series; + * error []byte // error string if thrown; + * target_lss_has_reallocations bool // true if target lss has reallocations + * } + */ +void prompp_prometheus_per_goroutine_relabeler_input_transition_relabeling(void* args, void* res); + +/** + * @brief transparent relabeling incomig hashdex(first stage) from cache. + * + * @param args { + * shards_inner_series []*InnerSeries // go slice with InnerSeries; + * per_goroutine_relabeler uintptr // pointer to constructed per goroutine relabeler; + * hashdex uintptr // pointer to filled hashdex; + * target_lss uintptr // pointer to constructed target label sets; + * } + * + * @param res { + * samples_added uint32 // number of added samples; + * series_added uint32 // number of added series; + * series_drop uint32 // number of dropped series; + * ok bool // true if all label set find in cache; + * error []byte // error string if thrown; + * } + */ +void prompp_prometheus_per_goroutine_relabeler_input_transition_relabeling_only_read(void* args, void* res); + +/** + * @brief add relabeled ls to lss, add to result and add to cache update(second stage). + * + * @param args { + * shards_inner_series []*InnerSeries // go InnerSeries per source shard; + * shards_relabeled_series []*RelabeledSeries // go RelabeledSeries per source shard; + * shards_relabeler_state_update []*RelabelerStateUpdate // pointer to RelabelerStateUpdate per source shard; + * per_goroutine_relabeler uintptr // pointer to constructed per goroutine relabeler; + * target_lss uintptr // pointer to constructed label sets; + * } + * + * @param res { + * error []byte // error string if thrown + * target_lss_has_reallocations bool // true if target lss has reallocations * } */ -void prompp_prometheus_cache_reset_to(void* args); +void prompp_prometheus_per_goroutine_relabeler_append_relabeler_series(void* args, void* res); #ifdef __cplusplus } // extern "C" diff --git a/pp/go/cppbridge/head.go b/pp/go/cppbridge/head.go index 6aac5c876b..29c9716ee7 100644 --- a/pp/go/cppbridge/head.go +++ b/pp/go/cppbridge/head.go @@ -30,6 +30,17 @@ type TimeInterval struct { MaxT int64 } +func NewInvalidTimeInterval() TimeInterval { + return TimeInterval{ + MinT: math.MaxInt64, + MaxT: math.MinInt64, + } +} + +func (t *TimeInterval) IsInvalid() bool { + return t.MinT == math.MaxInt64 && t.MaxT == math.MinInt64 +} + type Sample struct { Timestamp int64 Value float64 @@ -37,13 +48,17 @@ type Sample struct { // HeadDataStorage is Go wrapper around series_data::Data_storage. type HeadDataStorage struct { - dataStorage uintptr + dataStorage uintptr + gcDestroyDetector *uint64 + timeInterval TimeInterval } // NewHeadDataStorage - constructor. func NewHeadDataStorage() *HeadDataStorage { ds := &HeadDataStorage{ - dataStorage: seriesDataDataStorageCtor(), + dataStorage: seriesDataDataStorageCtor(), + gcDestroyDetector: &gcDestroyDetector, + timeInterval: NewInvalidTimeInterval(), } runtime.SetFinalizer(ds, func(ds *HeadDataStorage) { @@ -56,12 +71,16 @@ func NewHeadDataStorage() *HeadDataStorage { // Reset - resets data storage. func (ds *HeadDataStorage) Reset() { seriesDataDataStorageReset(ds.dataStorage) + ds.timeInterval = NewInvalidTimeInterval() } -func (ds *HeadDataStorage) TimeInterval() TimeInterval { - res := seriesDataDataStorageTimeInterval(ds.dataStorage) - runtime.KeepAlive(ds) - return res +func (ds *HeadDataStorage) TimeInterval(invalidateCache bool) TimeInterval { + if invalidateCache || ds.timeInterval.IsInvalid() { + ds.timeInterval = seriesDataDataStorageTimeInterval(ds.dataStorage) + runtime.KeepAlive(ds) + } + + return ds.timeInterval } func (ds *HeadDataStorage) GetQueriedSeriesBitset() []byte { @@ -235,7 +254,7 @@ type HeadDataStorageSerializedChunks struct { type HeadDataStorageSerializedChunkMetadata [SerializedChunkMetadataSize]byte -func (cm *HeadDataStorageSerializedChunkMetadata) SeriesID() uint32 { +func (cm HeadDataStorageSerializedChunkMetadata) SeriesID() uint32 { return *(*uint32)(unsafe.Pointer(&cm[0])) } @@ -255,6 +274,11 @@ func (r *HeadDataStorageSerializedChunks) Data() []byte { return r.data } +func (r *HeadDataStorageSerializedChunks) Metadata(chunkIndex int) HeadDataStorageSerializedChunkMetadata { + offset := Uint32Size + chunkIndex*SerializedChunkMetadataSize + return HeadDataStorageSerializedChunkMetadata(r.data[offset : offset+SerializedChunkMetadataSize]) +} + type HeadDataStorageSerializedChunkIndex struct { m map[uint32][]int } diff --git a/pp/go/cppbridge/head_test.go b/pp/go/cppbridge/head_test.go index 468f3f43a1..17962d957d 100644 --- a/pp/go/cppbridge/head_test.go +++ b/pp/go/cppbridge/head_test.go @@ -164,10 +164,15 @@ func (s *HeadSuite) TestTimeInterval() { encoder.Encode(1, 3, 1.0) // Act - timeInterval := dataStorage.TimeInterval() + timeInterval := dataStorage.TimeInterval(false) + encoder.Encode(1, 4, 1.0) + cachedTimeInterval := dataStorage.TimeInterval(false) + actualTimeInterval := dataStorage.TimeInterval(true) // Assert s.Equal(cppbridge.TimeInterval{MinT: 1, MaxT: 3}, timeInterval) + s.Equal(cppbridge.TimeInterval{MinT: 1, MaxT: 3}, cachedTimeInterval) + s.Equal(cppbridge.TimeInterval{MinT: 1, MaxT: 4}, actualTimeInterval) } func (s *HeadSuite) TestInstantQuery() { diff --git a/pp/go/cppbridge/head_wal.go b/pp/go/cppbridge/head_wal.go index 039e95707b..e5d40fe823 100644 --- a/pp/go/cppbridge/head_wal.go +++ b/pp/go/cppbridge/head_wal.go @@ -1,12 +1,67 @@ package cppbridge -import "runtime" +import ( + "hash/crc32" + "io" + "runtime" +) + +// +// HeadEncodedSegment +// + +// HeadEncodedSegment the encoded segment from the head wal. +type HeadEncodedSegment struct { + buf []byte + samples uint32 +} + +// NewHeadEncodedSegment init new [HeadEncodedSegment]. +func NewHeadEncodedSegment(b []byte, samples uint32) *HeadEncodedSegment { + s := &HeadEncodedSegment{ + buf: b, + samples: samples, + } + + runtime.SetFinalizer(s, func(s *HeadEncodedSegment) { + freeBytes(s.buf) + }) + + return s +} + +// Samples returns count of samples in segment. +func (s HeadEncodedSegment) Samples() uint32 { + return s.samples +} +// Size returns len of bytes. +func (s *HeadEncodedSegment) Size() int64 { + return int64(len(s.buf)) +} + +// CRC32 the hash amount according to the data. +func (s *HeadEncodedSegment) CRC32() uint32 { + return crc32.ChecksumIEEE(s.buf) +} + +// WriteTo implements io.WriterTo inerface. +func (s *HeadEncodedSegment) WriteTo(w io.Writer) (int64, error) { + n, err := w.Write(s.buf) + return int64(n), err +} + +// +// HeadWalEncoder +// + +// HeadWalEncoder the encoder for the head wal. type HeadWalEncoder struct { lss *LabelSetStorage encoder uintptr } +// NewHeadWalEncoder initializes a new [HeadWalEncoder]. func NewHeadWalEncoder(shardID uint16, logShards uint8, lss *LabelSetStorage) *HeadWalEncoder { e := &HeadWalEncoder{ lss: lss, @@ -20,27 +75,36 @@ func NewHeadWalEncoder(shardID uint16, logShards uint8, lss *LabelSetStorage) *H return e } +// Version returns current encoder version. func (*HeadWalEncoder) Version() uint8 { return EncodersVersion() } -func (e *HeadWalEncoder) Encode(innerSeriesSlice []*InnerSeries) (WALEncoderStats, error) { - res, err := headWalEncoderAddInnerSeries(e.encoder, innerSeriesSlice) +// Encode encodes inner series into a segment. +func (e *HeadWalEncoder) Encode(innerSeriesSlice []*InnerSeries) (uint32, error) { + samples, err := headWalEncoderAddInnerSeries(e.encoder, innerSeriesSlice) runtime.KeepAlive(e) - return res, err + return samples, err } -func (e *HeadWalEncoder) Finalize() (*EncodedSegment, error) { - stats, segment, err := headWalEncoderFinalize(e.encoder) +// Finalize finalizes the encoder and returns the encoded segment. +func (e *HeadWalEncoder) Finalize() (*HeadEncodedSegment, error) { + samples, segment, err := headWalEncoderFinalize(e.encoder) runtime.KeepAlive(e) - return NewEncodedSegment(segment, stats), err + return NewHeadEncodedSegment(segment, samples), err } +// +// HeadWalDecoder +// + +// HeadWalDecoder the decoder for the head wal. type HeadWalDecoder struct { lss *LabelSetStorage decoder uintptr } +// NewHeadWalDecoder initializes a new [HeadWalDecoder]. func NewHeadWalDecoder(lss *LabelSetStorage, encoderVersion uint8) *HeadWalDecoder { d := &HeadWalDecoder{ lss: lss, @@ -54,12 +118,17 @@ func NewHeadWalDecoder(lss *LabelSetStorage, encoderVersion uint8) *HeadWalDecod return d } +// Decode decodes a segment into an inner series. func (d *HeadWalDecoder) Decode(segment []byte, innerSeries *InnerSeries) error { err := headWalDecoderDecode(d.decoder, segment, innerSeries) runtime.KeepAlive(d) return err } +// DecodeToDataStorage decodes a segment into a data storage. +// +//revive:disable-next-line:confusing-results // returns createTimestamp, encodeTimestamp, error. +//nolint:gocritic // unnamedResult // returns createTimestamp, encodeTimestamp, error. func (d *HeadWalDecoder) DecodeToDataStorage(segment []byte, headEncoder *HeadEncoder) (int64, int64, error) { createTimestamp, encodeTimestamp, err := headWalDecoderDecodeToDataStorage(d.decoder, segment, headEncoder.encoder) runtime.KeepAlive(d) @@ -67,6 +136,7 @@ func (d *HeadWalDecoder) DecodeToDataStorage(segment []byte, headEncoder *HeadEn return createTimestamp, encodeTimestamp, err } +// CreateEncoder creates a new [HeadWalEncoder] from the decoder. func (d *HeadWalDecoder) CreateEncoder() *HeadWalEncoder { e := &HeadWalEncoder{ lss: d.lss, diff --git a/pp/go/cppbridge/head_wal_test.go b/pp/go/cppbridge/head_wal_test.go index d400d01bd6..6698ac733d 100644 --- a/pp/go/cppbridge/head_wal_test.go +++ b/pp/go/cppbridge/head_wal_test.go @@ -4,16 +4,69 @@ import ( "testing" "github.com/gobuffalo/packr/v2/file/resolver/encoding/hex" - "github.com/prometheus/prometheus/pp/go/cppbridge" "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" + + "github.com/prometheus/prometheus/pp/go/cppbridge" ) -func TestHeadWalEncoder_Finalize(t *testing.T) { +const hexSegment = "04224D184040C025030000B1D62B2CEB28C11C18DFD92E08008601020000000001000100021200170D1200071700042900150A1200140A2000300000001100FD0A03000000040000000500000006000000070000000800000009320013080400F316190000002100000014000000350000001E000000530000000E0000006100000013000000744E00937A0000001700000091180055A4000000165A00F5AFBA0000002C2975312C3F7C6A3C47343C55453531246E5A673044594B562A3645216E6F3F4D4A6F6D7154273D696C6C5F465765725750304672722F246D3F2E216D4B64313678506C30356C6A4555403F5961284768315F7C35596D713F4D4C293B713D684B786368562F393B593C382E492F597200744A2B244F233B375A32362F703D5F30596E3E3D54235E4A7946675F6267704C5445492E5F66576E704D413C6D28465227784F5E495E64687937776A7462296E52282C43463B4762734A0114016601110D0E00000600E92E3852473464414D6547436F67012500111D0E00000600F90F2262376555745376402A3B4D542A3E6E33243F34242C48716D39792C2302350011170E00000600F90977456B216B512A6C6643244E4E67252C512D3362794B3A032F00110E0E00000600F900474D2D55766D53614B764D6C626D042600111F0E00000600F9116B002A41224D703A3F656A3C4D252B3D6C46246D554E242A623972644C742F05370011180E00000600F90A3C4B356F7724234E7233464B2D38686D5C504763636D495306300011190E00000600F90B65282D795533525F636B6E45693C23702E4749776723503C40073100111B0E00000600F90D764900705C6C32584E4F30725978686975624066253024624C6B21083300111A0E00000600F90C504A7A6377575549593F6D2262672D705E5C21432F2E325F492409320011110E00000600F10464544E562A642D4835534727507500396401019F02D0040901FFFFFFFF683D16B182012D00141C5C03B27D000001773742790135021100F03DAF1C5A643BDFC7FF1C96FFCFE8F5A65B4E3C827DD5EB4DB79C7804BBEE57252B06C1938D78BF329D216BC9D4C8F743B0ACE2AEC78B9C19FAB779E7FBA5CD20B899C395ED280C0901677407941" + +// +// HeadWalSuite +// + +type HeadWalSuite struct { + suite.Suite +} + +func TestHeadWalSuite(t *testing.T) { + suite.Run(t, new(HeadWalSuite)) +} + +func (s *HeadWalSuite) TestHeadWalEncoder_Encode() { + encoder := cppbridge.NewHeadWalEncoder(0, 1, cppbridge.NewQueryableLssStorage()) + + samples, err := encoder.Encode([]*cppbridge.InnerSeries{}) + s.Require().NoError(err) + + s.Empty(samples) +} + +func (s *HeadWalSuite) TestHeadWalEncoder_Finalize() { + encoder := cppbridge.NewHeadWalEncoder(0, 1, cppbridge.NewQueryableLssStorage()) + + segmentData, err := encoder.Finalize() + s.Require().NoError(err) + + s.NotNil(segmentData) + s.Empty(segmentData.Samples()) +} + +func (s *HeadWalSuite) TestHeadWalEncoder_EncodeAndFinalize() { + const kTestBufferVersion = 3 + + segment, _ := hex.DecodeString(hexSegment) lss := cppbridge.NewQueryableLssStorage() + decoder := cppbridge.NewHeadWalDecoder(lss, kTestBufferVersion) encoder := cppbridge.NewHeadWalEncoder(0, 1, lss) + innerSeries := cppbridge.NewInnerSeries() + err := decoder.Decode(segment, innerSeries) + s.Require().NoError(err) + s.NotNil(innerSeries) + + expectedSamples := innerSeries.Size() + samples, err := encoder.Encode([]*cppbridge.InnerSeries{innerSeries}) + s.Require().NoError(err) + + s.NotNil(samples) + s.Equal(expectedSamples, uint64(samples)) + segmentData, err := encoder.Finalize() - require.NoError(t, err) - _ = segmentData + s.Require().NoError(err) + + s.NotNil(segmentData) + s.Equal(expectedSamples, uint64(segmentData.Samples())) } func TestHeadWalDecoder_DecodeToDataStorage(t *testing.T) { @@ -23,12 +76,12 @@ func TestHeadWalDecoder_DecodeToDataStorage(t *testing.T) { dataStorage := cppbridge.NewHeadDataStorage() encoder := cppbridge.NewHeadEncoderWithDataStorage(dataStorage) decoder := cppbridge.NewHeadWalDecoder(cppbridge.NewQueryableLssStorage(), kTestBufferVersion) - segment, _ := hex.DecodeString("04224D184040C025030000B1D62B2CEB28C11C18DFD92E08008601020000000001000100021200170D1200071700042900150A1200140A2000300000001100FD0A03000000040000000500000006000000070000000800000009320013080400F316190000002100000014000000350000001E000000530000000E0000006100000013000000744E00937A0000001700000091180055A4000000165A00F5AFBA0000002C2975312C3F7C6A3C47343C55453531246E5A673044594B562A3645216E6F3F4D4A6F6D7154273D696C6C5F465765725750304672722F246D3F2E216D4B64313678506C30356C6A4555403F5961284768315F7C35596D713F4D4C293B713D684B786368562F393B593C382E492F597200744A2B244F233B375A32362F703D5F30596E3E3D54235E4A7946675F6267704C5445492E5F66576E704D413C6D28465227784F5E495E64687937776A7462296E52282C43463B4762734A0114016601110D0E00000600E92E3852473464414D6547436F67012500111D0E00000600F90F2262376555745376402A3B4D542A3E6E33243F34242C48716D39792C2302350011170E00000600F90977456B216B512A6C6643244E4E67252C512D3362794B3A032F00110E0E00000600F900474D2D55766D53614B764D6C626D042600111F0E00000600F9116B002A41224D703A3F656A3C4D252B3D6C46246D554E242A623972644C742F05370011180E00000600F90A3C4B356F7724234E7233464B2D38686D5C504763636D495306300011190E00000600F90B65282D795533525F636B6E45693C23702E4749776723503C40073100111B0E00000600F90D764900705C6C32584E4F30725978686975624066253024624C6B21083300111A0E00000600F90C504A7A6377575549593F6D2262672D705E5C21432F2E325F492409320011110E00000600F10464544E562A642D4835534727507500396401019F02D0040901FFFFFFFF683D16B182012D00141C5C03B27D000001773742790135021100F03DAF1C5A643BDFC7FF1C96FFCFE8F5A65B4E3C827DD5EB4DB79C7804BBEE57252B06C1938D78BF329D216BC9D4C8F743B0ACE2AEC78B9C19FAB779E7FBA5CD20B899C395ED280C0901677407941") + segment, _ := hex.DecodeString(hexSegment) // Act _, _, err := decoder.DecodeToDataStorage(segment, encoder) // Assert require.NoError(t, err) - require.Equal(t, cppbridge.TimeInterval{MinT: 1660828401000, MaxT: 1660828410000}, dataStorage.TimeInterval()) + require.Equal(t, cppbridge.TimeInterval{MinT: 1660828401000, MaxT: 1660828410000}, dataStorage.TimeInterval(false)) } diff --git a/pp/go/cppbridge/lss_snapshot.go b/pp/go/cppbridge/lss_snapshot.go index 9d7535accb..8b95d1aca0 100644 --- a/pp/go/cppbridge/lss_snapshot.go +++ b/pp/go/cppbridge/lss_snapshot.go @@ -76,3 +76,12 @@ func (lss *LabelSetSnapshot) Query(selector uintptr) *LSSQueryResult { runtime.KeepAlive(lss) return result } + +// CopyAddedSeries copy the label sets from the source lss to the destination lss +// that were added source lss. +func (lss *LabelSetSnapshot) CopyAddedSeries(bitsetSeries *BitsetSeries, destination *LabelSetStorage) { + primitivesReadonlyLSSCopyAddedSeries(lss.pointer, bitsetSeries.pointer, destination.pointer) + runtime.KeepAlive(lss) + runtime.KeepAlive(bitsetSeries) + runtime.KeepAlive(destination) +} diff --git a/pp/go/cppbridge/primitives_lss.go b/pp/go/cppbridge/primitives_lss.go index 1a5d6493a0..802dab6132 100644 --- a/pp/go/cppbridge/primitives_lss.go +++ b/pp/go/cppbridge/primitives_lss.go @@ -71,6 +71,14 @@ func (lss *LabelSetStorage) AllocatedMemory() uint64 { return res } +// BitsetSeries returns a copy of the bitset of added series from the lss. Read operation. +func (lss *LabelSetStorage) BitsetSeries() *BitsetSeries { + bsPointer := primitivesLSSBitsetSeries(lss.pointer) + runtime.KeepAlive(lss) + + return newBitsetSeriesFromPointer(bsPointer) +} + // FindOrEmplace find in lss LabelSet or emplace and return ls id. func (lss *LabelSetStorage) FindOrEmplace(labelSet model.LabelSet) FindOrEmplaceResult { res := primitivesLSSFindOrEmplace(lss.pointer, labelSet) @@ -291,3 +299,23 @@ type CppLabelSetBuilder struct { SortedAdd []Label SortedDel []string } + +// +// BitsetSeries +// + +// BitsetSeries copies of the bitset of added series from the lss. +type BitsetSeries struct { + pointer uintptr + gcDestroyDetector *uint64 +} + +// newBitsetSeriesFromPointer init new [BitsetSeries]. +func newBitsetSeriesFromPointer(bitsetSeriesPointer uintptr) *BitsetSeries { + bitsetSeries := &BitsetSeries{pointer: bitsetSeriesPointer, gcDestroyDetector: &gcDestroyDetector} + runtime.SetFinalizer(bitsetSeries, func(bs *BitsetSeries) { + primitivesLSSBitsetDtor(bs.pointer) + }) + + return bitsetSeries +} diff --git a/pp/go/cppbridge/primitives_lss_test.go b/pp/go/cppbridge/primitives_lss_test.go index 96d9736389..07918d903f 100644 --- a/pp/go/cppbridge/primitives_lss_test.go +++ b/pp/go/cppbridge/primitives_lss_test.go @@ -3,6 +3,7 @@ package cppbridge_test import ( "context" "runtime" + "sort" "testing" "github.com/prometheus/prometheus/pp/go/model" @@ -413,8 +414,15 @@ func (s *QueryableLSSSuite) TestCopyAddedSeries() { s.lss.CopyAddedSeries(lssCopy) lssCopy.CopyAddedSeries(lssCopyOfCopy) + // lssCopy will contain lexicographically sorted labels with new IDs. + expectedLabelSets := make([]model.LabelSet, len(s.labelSets)) + copy(expectedLabelSets, s.labelSets) + sort.Slice(expectedLabelSets, func(i, j int) bool { + return expectedLabelSets[i].String() < expectedLabelSets[j].String() + }) + // Assert - s.Equal(labelSetToCppBridgeLabels(s.labelSets), lssCopy.GetLabelSets(s.labelSetIDs).LabelsSets()) + s.Equal(labelSetToCppBridgeLabels(expectedLabelSets), lssCopy.GetLabelSets(s.labelSetIDs).LabelsSets()) s.Equal(emptyLabelsSets, lssCopyOfCopy.GetLabelSets(s.labelSetIDs).LabelsSets()) } @@ -478,3 +486,30 @@ func (s *QueryableLSSSuite) TestFindOrEmplaceBuilderWithoutReadonlyLss() { // Assert s.Equal(uint32(expectedLsId), existingLsId) } + +func (s *QueryableLSSSuite) TestCopyAddedSeriesFromSnapshot() { + // Arrange + emptyLabelsSets := make([]cppbridge.Labels, len(s.labelSetIDs)) + lssCopy := cppbridge.NewQueryableLssStorage() + lssCopyOfCopy := cppbridge.NewQueryableLssStorage() + + // Act + snapshot := s.lss.CreateLabelSetSnapshot() + bitsetSeries := s.lss.BitsetSeries() + snapshot.CopyAddedSeries(bitsetSeries, lssCopy) + + snapshotCopy := lssCopy.CreateLabelSetSnapshot() + bitsetSeriesCopy := lssCopy.BitsetSeries() + snapshotCopy.CopyAddedSeries(bitsetSeriesCopy, lssCopyOfCopy) + + // lssCopy will contain lexicographically sorted labels with new IDs. + expectedLabelSets := make([]model.LabelSet, len(s.labelSets)) + copy(expectedLabelSets, s.labelSets) + sort.Slice(expectedLabelSets, func(i, j int) bool { + return expectedLabelSets[i].String() < expectedLabelSets[j].String() + }) + + // Assert + s.Equal(labelSetToCppBridgeLabels(expectedLabelSets), lssCopy.GetLabelSets(s.labelSetIDs).LabelsSets()) + s.Equal(emptyLabelsSets, lssCopyOfCopy.GetLabelSets(s.labelSetIDs).LabelsSets()) +} diff --git a/pp/go/cppbridge/prometheus_relabeler.go b/pp/go/cppbridge/prometheus_relabeler.go index 1572b445b7..a7667dbe71 100644 --- a/pp/go/cppbridge/prometheus_relabeler.go +++ b/pp/go/cppbridge/prometheus_relabeler.go @@ -9,12 +9,14 @@ import ( "runtime" "strconv" "strings" + "sync" "time" "github.com/cespare/xxhash/v2" "github.com/prometheus/common/model" ) +// NullTimestamp the timestamp that is used as the nil value. const NullTimestamp = math.MinInt64 // ErrLSSNullPointer - error when lss is null pointer @@ -525,10 +527,6 @@ func NewStaleNansState() *StaleNansState { return s } -func (s *StaleNansState) Reset() { - prometheusRelabelStaleNansStateReset(s.state) -} - // RelabelerStats statistics return from relabeler. type RelabelerStats struct { SamplesAdded uint32 @@ -536,6 +534,15 @@ type RelabelerStats struct { SeriesDrop uint32 } +// Add another stats. +func (rs *RelabelerStats) Add(stats ...RelabelerStats) { + for _, s := range stats { + rs.SamplesAdded += s.SamplesAdded + rs.SeriesAdded += s.SeriesAdded + rs.SeriesDrop += s.SeriesDrop + } +} + // String serialize to string. func (rs RelabelerStats) String() string { return fmt.Sprintf( @@ -924,6 +931,7 @@ func (opsr *OutputPerShardRelabeler) UpdateRelabelerState( // cPointer - pointer to C-Cache; type Cache struct { cPointer uintptr + lock sync.RWMutex } // NewCache init new Cache. @@ -939,17 +947,31 @@ func NewCache() *Cache { // AllocatedMemory return size of allocated memory for caches. func (c *Cache) AllocatedMemory() uint64 { + c.lock.RLock() res := prometheusCacheAllocatedMemory(c.cPointer) + c.lock.RUnlock() runtime.KeepAlive(c) return res } -// ResetTo reset cache. -func (c *Cache) ResetTo() { - prometheusCacheResetTo(c.cPointer) +// Update add to cache relabled data(third stage). +func (c *Cache) Update(ctx context.Context, shardsRelabelerStateUpdate []*RelabelerStateUpdate) error { + if ctx.Err() != nil { + return ctx.Err() + } + + c.lock.Lock() + exception := prometheusCacheUpdate(shardsRelabelerStateUpdate, c.cPointer) + c.lock.Unlock() runtime.KeepAlive(c) + + return handleException(exception) } +// +// State +// + // State state of relabelers per shard. type State struct { caches []*Cache @@ -976,38 +998,9 @@ func NewState(numberOfShards uint16) *State { // CacheByShard return *Cache for shard. func (s *State) CacheByShard(shardID uint16) *Cache { - if int(shardID) >= len(s.caches) { - panic(fmt.Sprintf( - "shardID(%d) out of range in caches(%d)", - shardID, - len(s.caches), - )) - } - - if s.caches[shardID] == nil { - s.caches[shardID] = NewCache() - } - return s.caches[shardID] } -// StaleNansStateByShard return SourceStaleNansState for shard. -func (s *State) StaleNansStateByShard(shardID uint16) *StaleNansState { - if int(shardID) >= len(s.staleNansStates) { - panic(fmt.Sprintf( - "shardID(%d) out of range in staleNansStates(%d)", - shardID, - len(s.caches), - )) - } - - if s.staleNansStates[shardID] == nil { - s.staleNansStates[shardID] = NewStaleNansState() - } - - return s.staleNansStates[shardID] -} - // DefTimestamp return timestamp for scrape time and stalenan. func (s *State) DefTimestamp() int64 { if s.defTimestamp == 0 { @@ -1017,34 +1010,14 @@ func (s *State) DefTimestamp() int64 { return s.defTimestamp } -// SetDefTimestamp set timestamp for scrape time and stalenan. -func (s *State) SetDefTimestamp(ts int64) { - s.defTimestamp = ts -} - -// EnableTrackStaleness enable track stalenans. -func (s *State) EnableTrackStaleness() { - s.trackStaleness = true -} - // DisableTrackStaleness disable track stalenans. func (s *State) DisableTrackStaleness() { s.trackStaleness = false } -// TrackStaleness return state track stalenans. -func (s *State) TrackStaleness() bool { - return s.trackStaleness -} - -// RelabelerOptions return Options for relabeler. -func (s *State) RelabelerOptions() RelabelerOptions { - return s.options -} - -// SetRelabelerOptions set Options for relabeler. -func (s *State) SetRelabelerOptions(options *RelabelerOptions) { - s.options = *options +// EnableTrackStaleness enable track stalenans. +func (s *State) EnableTrackStaleness() { + s.trackStaleness = true } // Reconfigure recreate caches and stalenans states if need and set new generations. @@ -1061,6 +1034,31 @@ func (s *State) Reconfigure( s.generationHead = generationHead } +// RelabelerOptions return Options for relabeler. +func (s *State) RelabelerOptions() RelabelerOptions { + return s.options +} + +// SetDefTimestamp set timestamp for scrape time and stalenan. +func (s *State) SetDefTimestamp(ts int64) { + s.defTimestamp = ts +} + +// SetRelabelerOptions set Options for relabeler. +func (s *State) SetRelabelerOptions(options *RelabelerOptions) { + s.options = *options +} + +// StaleNansStateByShard return SourceStaleNansState for shard. +func (s *State) StaleNansStateByShard(shardID uint16) *StaleNansState { + return s.staleNansStates[shardID] +} + +// TrackStaleness return state track stalenans. +func (s *State) TrackStaleness() bool { + return s.trackStaleness +} + // resetCaches recreate Caches. // //revive:disable-next-line:flag-parameter this is a flag, but it's more convenient this way @@ -1069,21 +1067,21 @@ func (s *State) resetCaches(numberOfShards uint16, equaledGeneration bool) { return } - for shardID := range s.caches { - s.caches[shardID] = nil - } + switch { + case len(s.caches) > int(numberOfShards): + for shardID := range s.caches[numberOfShards:] { + s.caches[shardID] = nil + } - if len(s.caches) > int(numberOfShards) { // cut s.caches = s.caches[:numberOfShards] + case len(s.caches) < int(numberOfShards): + // grow + s.caches = make([]*Cache, numberOfShards) } - if len(s.caches) < int(numberOfShards) { - // grow - s.caches = append( - s.caches, - make([]*Cache, int(numberOfShards)-len(s.caches))..., - ) + for shardID := range s.caches { + s.caches[shardID] = NewCache() } } @@ -1099,23 +1097,603 @@ func (s *State) resetStaleNansStates(numberOfShards uint16, equaledGeneration bo return } + switch { + case len(s.staleNansStates) > int(numberOfShards): + for shardID := range s.staleNansStates[numberOfShards:] { + s.staleNansStates[shardID] = nil + } + + // cut + s.staleNansStates = s.staleNansStates[:numberOfShards] + case len(s.staleNansStates) < int(numberOfShards): + // grow + s.staleNansStates = make([]*StaleNansState, numberOfShards) + } + for shardID := range s.staleNansStates { - state := s.staleNansStates[shardID] - if state != nil { - state.Reset() + s.staleNansStates[shardID] = NewStaleNansState() + } +} + +// +// PerGoroutineRelabeler +// + +// PerGoroutineRelabeler go wrapper for C-PerGoroutineRelabeler, relabeler for shard goroutines. +type PerGoroutineRelabeler struct { + cptr uintptr + gcDestroyDetector *uint64 + shardID uint16 +} + +// NewPerGoroutineRelabeler init new [PerGoroutineRelabeler]. +func NewPerGoroutineRelabeler( + numberOfShards, shardID uint16, +) *PerGoroutineRelabeler { + pgr := &PerGoroutineRelabeler{ + cptr: prometheusPerGoroutineRelabelerCtor(numberOfShards, shardID), + gcDestroyDetector: &gcDestroyDetector, + shardID: shardID, + } + runtime.SetFinalizer(pgr, func(r *PerGoroutineRelabeler) { + prometheusPerGoroutineRelabelerDtor(r.cptr) + }) + + return pgr +} + +// AppendRelabelerSeries add relabeled ls to lss, add to result and add to cache update(second stage). +func (pgr *PerGoroutineRelabeler) AppendRelabelerSeries( + ctx context.Context, + targetLss *LabelSetStorage, + shardsInnerSeries []*InnerSeries, + shardsRelabeledSeries []*RelabeledSeries, + shardsRelabelerStateUpdate []*RelabelerStateUpdate, +) (bool, error) { + if ctx.Err() != nil { + return false, ctx.Err() + } + + exception, hasReallocations := prometheusPerGoroutineRelabelerAppendRelabelerSeries( + pgr.cptr, + targetLss.Pointer(), + shardsInnerSeries, + shardsRelabeledSeries, + shardsRelabelerStateUpdate, + ) + + return hasReallocations, handleException(exception) +} + +// Relabeling relabeling incoming hashdex(first stage). +func (pgr *PerGoroutineRelabeler) Relabeling( + ctx context.Context, + inputLss *LabelSetStorage, + targetLss *LabelSetStorage, + state *StateV2, + shardedData ShardedData, + shardsInnerSeries []*InnerSeries, + shardsRelabeledSeries []*RelabeledSeries, +) (RelabelerStats, bool, error) { + if ctx.Err() != nil { + return RelabelerStats{}, false, ctx.Err() + } + + cptrContainer, ok := shardedData.(cptrable) + if !ok { + return RelabelerStats{}, false, ErrMustImplementCptrable + } + + if state.TrackStaleness() { + return pgr.inputRelabelingWithStalenans( + inputLss, + targetLss, + state, + cptrContainer, + shardsInnerSeries, + shardsRelabeledSeries, + ) + } + + if state.IsTransition() { + return pgr.inputTransitionRelabeling( + targetLss, + state, + cptrContainer, + shardsInnerSeries, + ) + } + + return pgr.inputRelabeling( + inputLss, + targetLss, + state, + cptrContainer, + shardsInnerSeries, + shardsRelabeledSeries, + ) +} + +// RelabelingFromCache relabeling incoming hashdex(first stage) from cache. +func (pgr *PerGoroutineRelabeler) RelabelingFromCache( + ctx context.Context, + inputLss *LabelSetStorage, + targetLss *LabelSetStorage, + state *StateV2, + shardedData ShardedData, + shardsInnerSeries []*InnerSeries, +) (RelabelerStats, bool, error) { + if ctx.Err() != nil { + return RelabelerStats{}, false, ctx.Err() + } + + cptrContainer, ok := shardedData.(cptrable) + if !ok { + return RelabelerStats{}, false, ErrMustImplementCptrable + } + + if state.TrackStaleness() { + return pgr.inputRelabelingWithStalenansFromCache( + inputLss, + targetLss, + state, + cptrContainer, + shardsInnerSeries, + ) + } + + if state.IsTransition() { + return pgr.inputTransitionRelabelingOnlyRead( + targetLss, + state, + cptrContainer, + shardsInnerSeries, + ) + } + + return pgr.inputRelabelingFromCache( + inputLss, + targetLss, + state, + cptrContainer, + shardsInnerSeries, + ) +} + +// inputRelabeling relabeling incoming hashdex(first stage). +func (pgr *PerGoroutineRelabeler) inputRelabeling( + inputLss *LabelSetStorage, + targetLss *LabelSetStorage, + state *StateV2, + cptrContainer cptrable, + shardsInnerSeries []*InnerSeries, + shardsRelabeledSeries []*RelabeledSeries, +) (RelabelerStats, bool, error) { + cache := state.CacheByShard(pgr.shardID) + cache.lock.Lock() + stats, exception, hasReallocations := prometheusPerGoroutineRelabelerInputRelabeling( + pgr.cptr, + state.StatelessRelabeler().Pointer(), + inputLss.Pointer(), + targetLss.Pointer(), + cache.cPointer, + cptrContainer.cptr(), + state.RelabelerOptions(), + shardsInnerSeries, + shardsRelabeledSeries, + ) + cache.lock.Unlock() + + runtime.KeepAlive(pgr) + runtime.KeepAlive(inputLss) + runtime.KeepAlive(targetLss) + runtime.KeepAlive(state) + runtime.KeepAlive(cptrContainer) + + return stats, hasReallocations, handleException(exception) +} + +// InputRelabelingFromCache relabeling incoming hashdex(first stage) from cache. +func (pgr *PerGoroutineRelabeler) inputRelabelingFromCache( + inputLss *LabelSetStorage, + targetLss *LabelSetStorage, + state *StateV2, + cptrContainer cptrable, + shardsInnerSeries []*InnerSeries, +) (RelabelerStats, bool, error) { + cache := state.CacheByShard(pgr.shardID) + cache.lock.RLock() + stats, exception, ok := prometheusPerGoroutineRelabelerInputRelabelingFromCache( + pgr.cptr, + inputLss.Pointer(), + targetLss.Pointer(), + cache.cPointer, + cptrContainer.cptr(), + state.RelabelerOptions(), + shardsInnerSeries, + ) + cache.lock.RUnlock() + + runtime.KeepAlive(pgr) + runtime.KeepAlive(inputLss) + runtime.KeepAlive(targetLss) + runtime.KeepAlive(state) + runtime.KeepAlive(cptrContainer) + + return stats, ok, handleException(exception) +} + +// inputRelabelingWithStalenans relabeling incoming hashdex(first stage) with state stalenans. +func (pgr *PerGoroutineRelabeler) inputRelabelingWithStalenans( + inputLss *LabelSetStorage, + targetLss *LabelSetStorage, + state *StateV2, + cptrContainer cptrable, + shardsInnerSeries []*InnerSeries, + shardsRelabeledSeries []*RelabeledSeries, +) (RelabelerStats, bool, error) { + cache := state.CacheByShard(pgr.shardID) + cache.lock.Lock() + stats, exception, hasReallocations := prometheusPerGoroutineRelabelerInputRelabelingWithStalenans( + pgr.cptr, + state.StatelessRelabeler().Pointer(), + inputLss.Pointer(), + targetLss.Pointer(), + cache.cPointer, + cptrContainer.cptr(), + state.StaleNansStateByShard(pgr.shardID).state, + state.DefTimestamp(), + state.RelabelerOptions(), + shardsInnerSeries, + shardsRelabeledSeries, + ) + cache.lock.Unlock() + + runtime.KeepAlive(pgr) + runtime.KeepAlive(inputLss) + runtime.KeepAlive(targetLss) + runtime.KeepAlive(state) + runtime.KeepAlive(cptrContainer) + + return stats, hasReallocations, handleException(exception) +} + +// inputRelabelingWithStalenansFromCache relabeling incoming hashdex(first stage) from cache with state stalenans. +func (pgr *PerGoroutineRelabeler) inputRelabelingWithStalenansFromCache( + inputLss *LabelSetStorage, + targetLss *LabelSetStorage, + state *StateV2, + cptrContainer cptrable, + shardsInnerSeries []*InnerSeries, +) (RelabelerStats, bool, error) { + cache := state.CacheByShard(pgr.shardID) + cache.lock.RLock() + stats, exception, ok := prometheusPerGoroutineRelabelerInputRelabelingWithStalenansFromCache( + pgr.cptr, + inputLss.Pointer(), + targetLss.Pointer(), + cache.cPointer, + cptrContainer.cptr(), + state.StaleNansStateByShard(pgr.shardID).state, + state.DefTimestamp(), + state.RelabelerOptions(), + shardsInnerSeries, + ) + cache.lock.RUnlock() + + runtime.KeepAlive(pgr) + runtime.KeepAlive(inputLss) + runtime.KeepAlive(targetLss) + runtime.KeepAlive(state) + runtime.KeepAlive(cptrContainer) + + return stats, ok, handleException(exception) +} + +// inputTransitionRelabeling transparent relabeling incoming hashdex(first stage). +func (pgr *PerGoroutineRelabeler) inputTransitionRelabeling( + targetLss *LabelSetStorage, + state *StateV2, + cptrContainer cptrable, + shardsInnerSeries []*InnerSeries, +) (RelabelerStats, bool, error) { + stats, exception, hasReallocations := prometheusPerGoroutineRelabelerInputTransitionRelabeling( + pgr.cptr, + targetLss.Pointer(), + cptrContainer.cptr(), + shardsInnerSeries, + ) + + runtime.KeepAlive(pgr) + runtime.KeepAlive(targetLss) + runtime.KeepAlive(state) + runtime.KeepAlive(cptrContainer) + + return stats, hasReallocations, handleException(exception) +} + +// inputTransitionRelabelingOnlyRead transparent relabeling incoming hashdex(first stage) from cache. +func (pgr *PerGoroutineRelabeler) inputTransitionRelabelingOnlyRead( + targetLss *LabelSetStorage, + state *StateV2, + cptrContainer cptrable, + shardsInnerSeries []*InnerSeries, +) (RelabelerStats, bool, error) { + stats, exception, ok := prometheusPerGoroutineRelabelerInputRelabelingOnlyRead( + pgr.cptr, + targetLss.Pointer(), + cptrContainer.cptr(), + shardsInnerSeries, + ) + + runtime.KeepAlive(pgr) + runtime.KeepAlive(targetLss) + runtime.KeepAlive(state) + runtime.KeepAlive(cptrContainer) + + return stats, ok, handleException(exception) +} + +// +// TransitionLocker +// + +// TransitionLocker is an implementing [sync.Mutex] that, depending on the situation, does not block. +type TransitionLocker struct { + mx sync.Mutex + lock bool +} + +// NewTransitionLocker init new [TransitionLocker]. +func NewTransitionLocker() TransitionLocker { + return TransitionLocker{ + mx: sync.Mutex{}, + lock: true, + } +} + +// NewTransitionLockerWithoutLock init new [TransitionLocker], without locks. +func NewTransitionLockerWithoutLock() TransitionLocker { + return TransitionLocker{ + mx: sync.Mutex{}, + lock: false, + } +} + +// Lock locks rw for writing, if need. +func (l *TransitionLocker) Lock() { + if l.lock { + l.mx.Lock() + } +} + +// Unlock unlocks rw for writing, if need. +func (l *TransitionLocker) Unlock() { + if l.lock { + l.mx.Unlock() + } +} + +// +// StateV2 +// + +const ( + initStatus uint8 = 0 + inited uint8 = 15 + transitionStatus uint8 = 240 +) + +// StateV2 of relabelers per shard. +type StateV2 struct { + caches []*Cache + staleNansStates []*StaleNansState + statelessRelabeler *StatelessRelabeler + locker TransitionLocker + defTimestamp int64 + generationHead uint64 + options RelabelerOptions + status uint8 + trackStaleness bool +} + +// NewTransitionStateV2 init empty [StateV2], with locks. +func NewTransitionStateV2() *StateV2 { + return &StateV2{ + locker: NewTransitionLocker(), + generationHead: math.MaxUint64, + status: transitionStatus, + trackStaleness: false, + } +} + +// NewTransitionStateV2WithoutLock init empty [StateV2], without locks. +func NewTransitionStateV2WithoutLock() *StateV2 { + return &StateV2{ + locker: NewTransitionLockerWithoutLock(), + generationHead: math.MaxUint64, + status: transitionStatus, + trackStaleness: false, + } +} + +// NewStateV2 init empty [StateV2], with locks. +func NewStateV2() *StateV2 { + return &StateV2{ + locker: NewTransitionLocker(), + generationHead: math.MaxUint64, + status: initStatus, + trackStaleness: false, + } +} + +// NewStateV2WithoutLock init empty [StateV2], without locks. +func NewStateV2WithoutLock() *StateV2 { + return &StateV2{ + locker: NewTransitionLockerWithoutLock(), + generationHead: math.MaxUint64, + status: initStatus, + trackStaleness: false, + } +} + +// CacheByShard return *Cache for shard. +func (s *StateV2) CacheByShard(shardID uint16) *Cache { + if s.IsTransition() { + panic("CacheByShard: state is transition") + } + + return s.caches[shardID] +} + +// DefTimestamp return timestamp for scrape time and stalenan. +func (s *StateV2) DefTimestamp() int64 { + if s.defTimestamp == 0 { + return time.Now().UnixMilli() + } + + return s.defTimestamp +} + +// EnableTrackStaleness enable track stalenans. +func (s *StateV2) EnableTrackStaleness() { + if s.IsTransition() { + panic("EnableTrackStaleness: state is transition") + } + + s.trackStaleness = true +} + +// Reconfigure recreate caches and stalenans states if need and set new generations. +func (s *StateV2) Reconfigure( + generationHead uint64, + numberOfShards uint16, +) { + if s.status&inited == inited && generationHead == s.generationHead { + return + } + + // long way + s.locker.Lock() + + // we check it a second time, but under lock + if s.status&inited == inited && generationHead == s.generationHead { + s.locker.Unlock() + return + } + + // the transition state does not require caches and staleNaNs + if s.IsTransition() { + s.status |= inited + s.generationHead = generationHead + s.locker.Unlock() + return + } + + s.resetCaches(numberOfShards) + s.resetStaleNansStates(numberOfShards) + s.status |= inited + s.generationHead = generationHead + + s.locker.Unlock() +} + +// IsTransition indicates whether the state is transition. +func (s *StateV2) IsTransition() bool { + return s.status&transitionStatus == transitionStatus +} + +// RelabelerOptions return Options for relabeler. +func (s *StateV2) RelabelerOptions() RelabelerOptions { + return s.options +} + +// SetDefTimestamp set timestamp for scrape time and stalenan. +func (s *StateV2) SetDefTimestamp(ts int64) { + s.defTimestamp = ts +} + +// SetRelabelerOptions set Options for relabeler. +func (s *StateV2) SetRelabelerOptions(options *RelabelerOptions) { + s.options = *options +} + +// SetStatelessRelabeler sets [StatelessRelabeler] for [PerGoroutineRelabeler]. +func (s *StateV2) SetStatelessRelabeler(statelessRelabeler *StatelessRelabeler) { + if s.IsTransition() { + panic("SetStatelessRelabeler: state is transition") + } + + s.statelessRelabeler = statelessRelabeler +} + +// StaleNansStateByShard return SourceStaleNansState for shard. +func (s *StateV2) StaleNansStateByShard(shardID uint16) *StaleNansState { + if s.IsTransition() { + panic("StaleNansStateByShard: state is transition") + } + + return s.staleNansStates[shardID] +} + +// StatelessRelabeler returns [StatelessRelabeler] for [PerGoroutineRelabeler]. +func (s *StateV2) StatelessRelabeler() *StatelessRelabeler { + if s.IsTransition() { + panic("StatelessRelabeler: state is transition") + } + + if s.statelessRelabeler == nil { + panic("statelessRelabeler is nil") + } + + return s.statelessRelabeler +} + +// TrackStaleness return state track stalenans. +func (s *StateV2) TrackStaleness() bool { + return s.trackStaleness +} + +// resetCaches recreate Caches. +func (s *StateV2) resetCaches(numberOfShards uint16) { + switch { + case len(s.caches) > int(numberOfShards): + for shardID := range s.caches[numberOfShards:] { + s.caches[shardID] = nil } + + // cut + s.caches = s.caches[:numberOfShards] + case len(s.caches) < int(numberOfShards): + // grow + s.caches = make([]*Cache, numberOfShards) + } + + for shardID := range s.caches { + s.caches[shardID] = NewCache() } +} + +// resetStaleNansStates recreate StaleNansStates. +func (s *StateV2) resetStaleNansStates(numberOfShards uint16) { + if !s.trackStaleness { + return + } + + switch { + case len(s.staleNansStates) > int(numberOfShards): + for shardID := range s.staleNansStates[numberOfShards:] { + s.staleNansStates[shardID] = nil + } - if len(s.staleNansStates) > int(numberOfShards) { // cut s.staleNansStates = s.staleNansStates[:numberOfShards] + case len(s.staleNansStates) < int(numberOfShards): + // grow + s.staleNansStates = make([]*StaleNansState, numberOfShards) } - if len(s.staleNansStates) < int(numberOfShards) { - // grow - s.staleNansStates = append( - s.staleNansStates, - make([]*StaleNansState, int(numberOfShards)-len(s.staleNansStates))..., - ) + for shardID := range s.staleNansStates { + s.staleNansStates[shardID] = NewStaleNansState() } } diff --git a/pp/go/cppbridge/prometheus_relabeler_test.go b/pp/go/cppbridge/prometheus_relabeler_test.go index b195ded8f3..6f0c0c4c09 100644 --- a/pp/go/cppbridge/prometheus_relabeler_test.go +++ b/pp/go/cppbridge/prometheus_relabeler_test.go @@ -600,3 +600,1352 @@ func (s *RelabelerSuite) TestToHash_EmptyConfig() { s.Require().Equal(xxhash.Sum64String("0"+a.String()), cppbridge.ToHash(rCfgs)) } + +// +// PerGoroutineRelabelerSuite +// + +type PerGoroutineRelabelerSuite struct { + suite.Suite + baseCtx context.Context + options cppbridge.RelabelerOptions + hlimits cppbridge.WALHashdexLimits + rCfgs []*cppbridge.RelabelConfig + inputLss *cppbridge.LabelSetStorage + targetLss *cppbridge.LabelSetStorage + numberOfShards uint16 +} + +func TestPerGoroutineRelabelerSuite(t *testing.T) { + suite.Run(t, new(PerGoroutineRelabelerSuite)) +} + +func (s *PerGoroutineRelabelerSuite) SetupSuite() { + s.baseCtx = context.Background() + s.hlimits = cppbridge.DefaultWALHashdexLimits() + s.rCfgs = []*cppbridge.RelabelConfig{ + { + SourceLabels: []string{"job"}, + Regex: "abc", + Action: cppbridge.Keep, + }, + } + s.numberOfShards = 1 +} + +func (s *PerGoroutineRelabelerSuite) SetupTest() { + s.options = cppbridge.RelabelerOptions{} + s.inputLss = cppbridge.NewLssStorage() + s.targetLss = cppbridge.NewQueryableLssStorage() +} + +func (s *PerGoroutineRelabelerSuite) TestRelabeling() { + h, err := s.makeSnappyProtobufHashdex(&prompb.WriteRequest{ + Timeseries: []prompb.TimeSeries{ + { + Labels: []prompb.Label{ + {Name: "__name__", Value: "value"}, + {Name: "job", Value: "abc"}, + {Name: "instance", Value: "value1"}, + }, + Samples: []prompb.Sample{ + {Value: 0.1, Timestamp: time.Now().UnixMilli()}, + }, + }, + { + Labels: []prompb.Label{ + {Name: "__name__", Value: "value"}, + {Name: "instance", Value: "value1"}, + }, + Samples: []prompb.Sample{ + {Value: 0.1, Timestamp: time.Now().UnixMilli()}, + }, + }, + }, + }) + s.Require().NoError(err) + + shardsInnerSeries := cppbridge.NewShardsInnerSeries(s.numberOfShards) + shardsRelabeledSeries := cppbridge.NewShardsRelabeledSeries(s.numberOfShards) + + statelessRelabeler, err := cppbridge.NewStatelessRelabeler(s.rCfgs) + s.Require().NoError(err) + + state := cppbridge.NewStateV2WithoutLock() + state.SetRelabelerOptions(&s.options) + state.SetStatelessRelabeler(statelessRelabeler) + state.Reconfigure(0, s.numberOfShards) + + pgr := cppbridge.NewPerGoroutineRelabeler(s.numberOfShards, 0) + stats, hasReallocations, err := pgr.Relabeling( + s.baseCtx, + s.inputLss, + s.targetLss, + state, + h, + shardsInnerSeries, + shardsRelabeledSeries, + ) + s.Require().NoError(err) + s.Equal(cppbridge.RelabelerStats{1, 1, 1}, stats) + s.True(hasReallocations) +} + +func (s *PerGoroutineRelabelerSuite) TestRelabelingDrop() { + h, err := s.makeSnappyProtobufHashdex(&prompb.WriteRequest{ + Timeseries: []prompb.TimeSeries{ + { + Labels: []prompb.Label{ + {Name: "job", Value: "abc"}, + {Name: "instance", Value: "value1"}, + }, + Samples: []prompb.Sample{ + {Value: 0.1, Timestamp: time.Now().UnixMilli()}, + }, + }, + { + Labels: []prompb.Label{ + {Name: "instance", Value: "value1"}, + }, + Samples: []prompb.Sample{ + {Value: 0.1, Timestamp: time.Now().UnixMilli()}, + }, + }, + }, + }) + s.Require().NoError(err) + + shardsInnerSeries := cppbridge.NewShardsInnerSeries(s.numberOfShards) + shardsRelabeledSeries := cppbridge.NewShardsRelabeledSeries(s.numberOfShards) + + statelessRelabeler, err := cppbridge.NewStatelessRelabeler(s.rCfgs) + s.Require().NoError(err) + + state := cppbridge.NewStateV2WithoutLock() + state.SetRelabelerOptions(&s.options) + state.SetStatelessRelabeler(statelessRelabeler) + state.Reconfigure(0, s.numberOfShards) + + pgr := cppbridge.NewPerGoroutineRelabeler(s.numberOfShards, 0) + stats, hasReallocations, err := pgr.Relabeling( + s.baseCtx, + s.inputLss, + s.targetLss, + state, + h, + shardsInnerSeries, + shardsRelabeledSeries, + ) + s.Require().NoError(err) + s.Equal(cppbridge.RelabelerStats{0, 0, 2}, stats) + s.True(hasReallocations) +} + +func (s *PerGoroutineRelabelerSuite) TestRelabelingFromCacheTrue() { + h, err := s.makeSnappyProtobufHashdex(&prompb.WriteRequest{ + Timeseries: []prompb.TimeSeries{ + { + Labels: []prompb.Label{ + {Name: "__name__", Value: "value"}, + {Name: "job", Value: "abc"}, + {Name: "instance", Value: "value1"}, + }, + Samples: []prompb.Sample{ + {Value: 0.1, Timestamp: time.Now().UnixMilli()}, + }, + }, + }, + }) + s.Require().NoError(err) + + shardsInnerSeries := cppbridge.NewShardsInnerSeries(s.numberOfShards) + shardsRelabeledSeries := cppbridge.NewShardsRelabeledSeries(s.numberOfShards) + + statelessRelabeler, err := cppbridge.NewStatelessRelabeler(s.rCfgs) + s.Require().NoError(err) + + state := cppbridge.NewStateV2WithoutLock() + state.SetStatelessRelabeler(statelessRelabeler) + state.SetRelabelerOptions(&s.options) + state.Reconfigure(0, s.numberOfShards) + + pgr := cppbridge.NewPerGoroutineRelabeler(s.numberOfShards, 0) + stats, hasReallocations, err := pgr.Relabeling( + s.baseCtx, + s.inputLss, + s.targetLss, + state, + h, + shardsInnerSeries, + shardsRelabeledSeries, + ) + s.Require().NoError(err) + s.Equal(cppbridge.RelabelerStats{1, 1, 0}, stats) + s.True(hasReallocations) + + stats, ok, err := pgr.RelabelingFromCache( + s.baseCtx, + s.inputLss, + s.targetLss, + state, + h, + shardsInnerSeries, + ) + s.Require().NoError(err) + s.Equal(cppbridge.RelabelerStats{1, 0, 0}, stats) + s.True(ok) +} + +func (s *PerGoroutineRelabelerSuite) TestRelabelingFromCacheFalse() { + h, err := s.makeSnappyProtobufHashdex(&prompb.WriteRequest{ + Timeseries: []prompb.TimeSeries{ + { + Labels: []prompb.Label{ + {Name: "__name__", Value: "value"}, + {Name: "job", Value: "abc"}, + {Name: "instance", Value: "value1"}, + }, + Samples: []prompb.Sample{ + {Value: 0.1, Timestamp: time.Now().UnixMilli()}, + }, + }, + { + Labels: []prompb.Label{ + {Name: "__name__", Value: "value"}, + {Name: "instance", Value: "value1"}, + }, + Samples: []prompb.Sample{ + {Value: 0.1, Timestamp: time.Now().UnixMilli()}, + }, + }, + }, + }) + s.Require().NoError(err) + + shardsInnerSeries := cppbridge.NewShardsInnerSeries(s.numberOfShards) + state := cppbridge.NewStateV2WithoutLock() + state.SetRelabelerOptions(&s.options) + state.Reconfigure(0, s.numberOfShards) + + pgr := cppbridge.NewPerGoroutineRelabeler(s.numberOfShards, 0) + stats, ok, err := pgr.RelabelingFromCache( + s.baseCtx, + s.inputLss, + s.targetLss, + state, + h, + shardsInnerSeries, + ) + s.Require().NoError(err) + s.Equal(cppbridge.RelabelerStats{0, 0, 0}, stats) + s.False(ok) +} + +func (s *PerGoroutineRelabelerSuite) TestRelabelingFromCachePartially() { + ts := time.Now().UnixMilli() + h1, err := s.makeSnappyProtobufHashdex(&prompb.WriteRequest{ + Timeseries: []prompb.TimeSeries{ + { + Labels: []prompb.Label{ + {Name: "__name__", Value: "value0"}, + {Name: "job", Value: "abc"}, + {Name: "instance", Value: "value0"}, + }, + Samples: []prompb.Sample{ + {Value: 0.1, Timestamp: ts}, + }, + }, + { + Labels: []prompb.Label{ + {Name: "__name__", Value: "value1"}, + {Name: "job", Value: "abc"}, + {Name: "instance", Value: "value1"}, + }, + Samples: []prompb.Sample{ + {Value: 0.1, Timestamp: ts}, + }, + }, + { + Labels: []prompb.Label{ + {Name: "__name__", Value: "value2"}, + {Name: "job", Value: "abc"}, + {Name: "instance", Value: "value2"}, + }, + Samples: []prompb.Sample{ + {Value: 0.1, Timestamp: ts}, + }, + }, + { + Labels: []prompb.Label{ + {Name: "__name__", Value: "value3"}, + {Name: "job", Value: "abc"}, + {Name: "instance", Value: "value3"}, + }, + Samples: []prompb.Sample{ + {Value: 0.1, Timestamp: ts}, + }, + }, + { + Labels: []prompb.Label{ + {Name: "__name__", Value: "value3"}, + {Name: "instance", Value: "value3"}, + }, + Samples: []prompb.Sample{ + {Value: 0.1, Timestamp: ts}, + }, + }, + { + Labels: []prompb.Label{ + {Name: "__name__", Value: "value4"}, + {Name: "instance", Value: "value4"}, + }, + Samples: []prompb.Sample{ + {Value: 0.1, Timestamp: ts}, + }, + }, + { + Labels: []prompb.Label{ + {Name: "__name__", Value: "value5"}, + {Name: "instance", Value: "value5"}, + }, + Samples: []prompb.Sample{ + {Value: 0.1, Timestamp: ts}, + }, + }, + { + Labels: []prompb.Label{ + {Name: "__name__", Value: "value6"}, + {Name: "instance", Value: "value6"}, + }, + Samples: []prompb.Sample{ + {Value: 0.1, Timestamp: ts}, + }, + }, + }, + }) + s.Require().NoError(err) + + ts += 6000 + h2, err := s.makeSnappyProtobufHashdex(&prompb.WriteRequest{ + Timeseries: []prompb.TimeSeries{ + { + Labels: []prompb.Label{ + {Name: "__name__", Value: "value0"}, + {Name: "job", Value: "abc"}, + {Name: "instance", Value: "value0"}, + }, + Samples: []prompb.Sample{ + {Value: 0.1, Timestamp: ts}, + }, + }, + { + Labels: []prompb.Label{ + {Name: "__name__", Value: "value1"}, + {Name: "job", Value: "abc"}, + {Name: "instance", Value: "value1"}, + }, + Samples: []prompb.Sample{ + {Value: 0.1, Timestamp: ts}, + }, + }, + { + Labels: []prompb.Label{ + {Name: "__name__", Value: "value2"}, + {Name: "job", Value: "abc"}, + {Name: "instance", Value: "value2"}, + }, + Samples: []prompb.Sample{ + {Value: 0.1, Timestamp: ts}, + }, + }, + { + Labels: []prompb.Label{ + {Name: "__name__", Value: "value3"}, + {Name: "job", Value: "abc"}, + {Name: "instance", Value: "value3"}, + }, + Samples: []prompb.Sample{ + {Value: 0.1, Timestamp: ts}, + }, + }, + { + Labels: []prompb.Label{ + {Name: "__name__", Value: "value4"}, + {Name: "job", Value: "abc"}, + {Name: "instance", Value: "value4"}, + }, + Samples: []prompb.Sample{ + {Value: 0.1, Timestamp: ts}, + }, + }, + }, + }) + s.Require().NoError(err) + + shardsInnerSeries := cppbridge.NewShardsInnerSeries(s.numberOfShards) + shardsRelabeledSeries := cppbridge.NewShardsRelabeledSeries(s.numberOfShards) + + statelessRelabeler, err := cppbridge.NewStatelessRelabeler(s.rCfgs) + s.Require().NoError(err) + + state := cppbridge.NewStateV2WithoutLock() + state.SetRelabelerOptions(&s.options) + state.SetStatelessRelabeler(statelessRelabeler) + state.Reconfigure(0, s.numberOfShards) + + pgr := cppbridge.NewPerGoroutineRelabeler(s.numberOfShards, 0) + stats, hasReallocations, err := pgr.Relabeling( + s.baseCtx, + s.inputLss, + s.targetLss, + state, + h1, + shardsInnerSeries, + shardsRelabeledSeries, + ) + s.Require().NoError(err) + s.Equal(cppbridge.RelabelerStats{4, 4, 4}, stats) + s.True(hasReallocations) + + shardsInnerSeries = cppbridge.NewShardsInnerSeries(s.numberOfShards) + stats, ok, err := pgr.RelabelingFromCache( + s.baseCtx, + s.inputLss, + s.targetLss, + state, + h2, + shardsInnerSeries, + ) + s.Require().NoError(err) + s.Equal(cppbridge.RelabelerStats{4, 0, 0}, stats) + s.False(ok) + s.Equal(uint64(4), shardsInnerSeries[0].Size()) + + stats, _, err = pgr.Relabeling( + s.baseCtx, + s.inputLss, + s.targetLss, + state, + h2, + shardsInnerSeries, + shardsRelabeledSeries, + ) + s.Require().NoError(err) + s.Equal(cppbridge.RelabelerStats{1, 1, 0}, stats) + s.Equal(uint64(5), shardsInnerSeries[0].Size()) +} + +func (s *PerGoroutineRelabelerSuite) TestRelabelingTransition() { + h, err := s.makeSnappyProtobufHashdex(&prompb.WriteRequest{ + Timeseries: []prompb.TimeSeries{ + { + Labels: []prompb.Label{ + {Name: "__name__", Value: "value"}, + {Name: "job", Value: "abc"}, + {Name: "instance", Value: "value1"}, + }, + Samples: []prompb.Sample{ + {Value: 0.1, Timestamp: time.Now().UnixMilli()}, + }, + }, + { + Labels: []prompb.Label{ + {Name: "__name__", Value: "value"}, + {Name: "instance", Value: "value1"}, + }, + Samples: []prompb.Sample{ + {Value: 0.1, Timestamp: time.Now().UnixMilli()}, + }, + }, + }, + }) + s.Require().NoError(err) + + shardsInnerSeries := cppbridge.NewShardsInnerSeries(s.numberOfShards) + shardsRelabeledSeries := cppbridge.NewShardsRelabeledSeries(s.numberOfShards) + + state := cppbridge.NewTransitionStateV2() + state.Reconfigure(0, s.numberOfShards) + + pgr := cppbridge.NewPerGoroutineRelabeler(s.numberOfShards, 0) + stats, hasReallocations, err := pgr.Relabeling( + s.baseCtx, + s.inputLss, + s.targetLss, + state, + h, + shardsInnerSeries, + shardsRelabeledSeries, + ) + s.Require().NoError(err) + s.Equal(cppbridge.RelabelerStats{2, 2, 0}, stats) + s.True(hasReallocations) +} + +func (s *PerGoroutineRelabelerSuite) TestRelabelingFromCacheTrueTransition() { + h, err := s.makeSnappyProtobufHashdex(&prompb.WriteRequest{ + Timeseries: []prompb.TimeSeries{ + { + Labels: []prompb.Label{ + {Name: "__name__", Value: "value"}, + {Name: "job", Value: "abc"}, + {Name: "instance", Value: "value1"}, + }, + Samples: []prompb.Sample{ + {Value: 0.1, Timestamp: time.Now().UnixMilli()}, + }, + }, + }, + }) + s.Require().NoError(err) + + shardsInnerSeries := cppbridge.NewShardsInnerSeries(s.numberOfShards) + shardsRelabeledSeries := cppbridge.NewShardsRelabeledSeries(s.numberOfShards) + + state := cppbridge.NewTransitionStateV2() + state.Reconfigure(0, s.numberOfShards) + + pgr := cppbridge.NewPerGoroutineRelabeler(s.numberOfShards, 0) + stats, hasReallocations, err := pgr.Relabeling( + s.baseCtx, + s.inputLss, + s.targetLss, + state, + h, + shardsInnerSeries, + shardsRelabeledSeries, + ) + s.Require().NoError(err) + s.Equal(cppbridge.RelabelerStats{1, 1, 0}, stats) + s.True(hasReallocations) + + stats, ok, err := pgr.RelabelingFromCache( + s.baseCtx, + s.inputLss, + s.targetLss, + state, + h, + shardsInnerSeries, + ) + s.Require().NoError(err) + s.Equal(cppbridge.RelabelerStats{1, 0, 0}, stats) + s.True(ok) +} + +func (s *PerGoroutineRelabelerSuite) TestRelabelingFromCacheFalseTransition() { + h, err := s.makeSnappyProtobufHashdex(&prompb.WriteRequest{ + Timeseries: []prompb.TimeSeries{ + { + Labels: []prompb.Label{ + {Name: "__name__", Value: "value"}, + {Name: "job", Value: "abc"}, + {Name: "instance", Value: "value1"}, + }, + Samples: []prompb.Sample{ + {Value: 0.1, Timestamp: time.Now().UnixMilli()}, + }, + }, + { + Labels: []prompb.Label{ + {Name: "__name__", Value: "value"}, + {Name: "instance", Value: "value1"}, + }, + Samples: []prompb.Sample{ + {Value: 0.1, Timestamp: time.Now().UnixMilli()}, + }, + }, + }, + }) + s.Require().NoError(err) + + shardsInnerSeries := cppbridge.NewShardsInnerSeries(s.numberOfShards) + state := cppbridge.NewTransitionStateV2() + state.Reconfigure(0, s.numberOfShards) + + pgr := cppbridge.NewPerGoroutineRelabeler(s.numberOfShards, 0) + stats, ok, err := pgr.RelabelingFromCache( + s.baseCtx, + s.inputLss, + s.targetLss, + state, + h, + shardsInnerSeries, + ) + s.Require().NoError(err) + s.Equal(cppbridge.RelabelerStats{0, 0, 0}, stats) + s.False(ok) +} + +func (s *PerGoroutineRelabelerSuite) TestRelabelingFromCachePartiallyTransition() { + ts := time.Now().UnixMilli() + h1, err := s.makeSnappyProtobufHashdex(&prompb.WriteRequest{ + Timeseries: []prompb.TimeSeries{ + { + Labels: []prompb.Label{ + {Name: "__name__", Value: "value0"}, + {Name: "job", Value: "abc"}, + {Name: "instance", Value: "value0"}, + }, + Samples: []prompb.Sample{ + {Value: 0.1, Timestamp: ts}, + }, + }, + { + Labels: []prompb.Label{ + {Name: "__name__", Value: "value1"}, + {Name: "job", Value: "abc"}, + {Name: "instance", Value: "value1"}, + }, + Samples: []prompb.Sample{ + {Value: 0.1, Timestamp: ts}, + }, + }, + { + Labels: []prompb.Label{ + {Name: "__name__", Value: "value2"}, + {Name: "job", Value: "abc"}, + {Name: "instance", Value: "value2"}, + }, + Samples: []prompb.Sample{ + {Value: 0.1, Timestamp: ts}, + }, + }, + { + Labels: []prompb.Label{ + {Name: "__name__", Value: "value3"}, + {Name: "job", Value: "abc"}, + {Name: "instance", Value: "value3"}, + }, + Samples: []prompb.Sample{ + {Value: 0.1, Timestamp: ts}, + }, + }, + { + Labels: []prompb.Label{ + {Name: "__name__", Value: "value3"}, + {Name: "instance", Value: "value3"}, + }, + Samples: []prompb.Sample{ + {Value: 0.1, Timestamp: ts}, + }, + }, + { + Labels: []prompb.Label{ + {Name: "__name__", Value: "value4"}, + {Name: "instance", Value: "value4"}, + }, + Samples: []prompb.Sample{ + {Value: 0.1, Timestamp: ts}, + }, + }, + { + Labels: []prompb.Label{ + {Name: "__name__", Value: "value5"}, + {Name: "instance", Value: "value5"}, + }, + Samples: []prompb.Sample{ + {Value: 0.1, Timestamp: ts}, + }, + }, + { + Labels: []prompb.Label{ + {Name: "__name__", Value: "value6"}, + {Name: "instance", Value: "value6"}, + }, + Samples: []prompb.Sample{ + {Value: 0.1, Timestamp: ts}, + }, + }, + }, + }) + s.Require().NoError(err) + + ts += 6000 + h2, err := s.makeSnappyProtobufHashdex(&prompb.WriteRequest{ + Timeseries: []prompb.TimeSeries{ + { + Labels: []prompb.Label{ + {Name: "__name__", Value: "value0"}, + {Name: "job", Value: "abc"}, + {Name: "instance", Value: "value0"}, + }, + Samples: []prompb.Sample{ + {Value: 0.1, Timestamp: ts}, + }, + }, + { + Labels: []prompb.Label{ + {Name: "__name__", Value: "value1"}, + {Name: "job", Value: "abc"}, + {Name: "instance", Value: "value1"}, + }, + Samples: []prompb.Sample{ + {Value: 0.1, Timestamp: ts}, + }, + }, + { + Labels: []prompb.Label{ + {Name: "__name__", Value: "value2"}, + {Name: "job", Value: "abc"}, + {Name: "instance", Value: "value2"}, + }, + Samples: []prompb.Sample{ + {Value: 0.1, Timestamp: ts}, + }, + }, + { + Labels: []prompb.Label{ + {Name: "__name__", Value: "value3"}, + {Name: "job", Value: "abc"}, + {Name: "instance", Value: "value3"}, + }, + Samples: []prompb.Sample{ + {Value: 0.1, Timestamp: ts}, + }, + }, + { + Labels: []prompb.Label{ + {Name: "__name__", Value: "value4"}, + {Name: "job", Value: "abc"}, + {Name: "instance", Value: "value4"}, + }, + Samples: []prompb.Sample{ + {Value: 0.1, Timestamp: ts}, + }, + }, + }, + }) + s.Require().NoError(err) + + shardsInnerSeries := cppbridge.NewShardsInnerSeries(s.numberOfShards) + shardsRelabeledSeries := cppbridge.NewShardsRelabeledSeries(s.numberOfShards) + + state := cppbridge.NewTransitionStateV2() + state.Reconfigure(0, s.numberOfShards) + + pgr := cppbridge.NewPerGoroutineRelabeler(s.numberOfShards, 0) + stats, hasReallocations, err := pgr.Relabeling( + s.baseCtx, + s.inputLss, + s.targetLss, + state, + h1, + shardsInnerSeries, + shardsRelabeledSeries, + ) + s.Require().NoError(err) + s.Equal(cppbridge.RelabelerStats{8, 8, 0}, stats) + s.True(hasReallocations) + + shardsInnerSeries = cppbridge.NewShardsInnerSeries(s.numberOfShards) + stats, ok, err := pgr.RelabelingFromCache( + s.baseCtx, + s.inputLss, + s.targetLss, + state, + h2, + shardsInnerSeries, + ) + s.Require().NoError(err) + s.Equal(cppbridge.RelabelerStats{4, 0, 0}, stats) + s.False(ok) + s.Equal(uint64(4), shardsInnerSeries[0].Size()) + + stats, _, err = pgr.Relabeling( + s.baseCtx, + s.inputLss, + s.targetLss, + state, + h2, + shardsInnerSeries, + shardsRelabeledSeries, + ) + s.Require().NoError(err) + s.Equal(cppbridge.RelabelerStats{1, 1, 0}, stats) + s.Equal(uint64(5), shardsInnerSeries[0].Size()) +} + +func (s *PerGoroutineRelabelerSuite) TestRelabelingWithStalenans() { + h, err := s.makeSnappyProtobufHashdex(&prompb.WriteRequest{ + Timeseries: []prompb.TimeSeries{ + { + Labels: []prompb.Label{ + {Name: "__name__", Value: "value"}, + {Name: "job", Value: "abc"}, + {Name: "instance", Value: "value1"}, + }, + Samples: []prompb.Sample{ + {Value: 0.1, Timestamp: time.Now().UnixMilli()}, + }, + }, + { + Labels: []prompb.Label{ + {Name: "__name__", Value: "value"}, + {Name: "instance", Value: "value1"}, + }, + Samples: []prompb.Sample{ + {Value: 0.1, Timestamp: time.Now().UnixMilli()}, + }, + }, + }, + }) + s.Require().NoError(err) + + shardsInnerSeries := cppbridge.NewShardsInnerSeries(s.numberOfShards) + shardsRelabeledSeries := cppbridge.NewShardsRelabeledSeries(s.numberOfShards) + + statelessRelabeler, err := cppbridge.NewStatelessRelabeler(s.rCfgs) + s.Require().NoError(err) + + state := cppbridge.NewStateV2WithoutLock() + state.SetRelabelerOptions(&s.options) + state.SetStatelessRelabeler(statelessRelabeler) + state.EnableTrackStaleness() + state.Reconfigure(0, s.numberOfShards) + + pgr := cppbridge.NewPerGoroutineRelabeler(s.numberOfShards, 0) + stats, hasReallocations, err := pgr.Relabeling( + s.baseCtx, + s.inputLss, + s.targetLss, + state, + h, + shardsInnerSeries, + shardsRelabeledSeries, + ) + s.Require().NoError(err) + s.Equal(cppbridge.RelabelerStats{1, 1, 1}, stats) + s.True(hasReallocations) + s.Equal(uint64(1), shardsInnerSeries[0].Size()) + + h, err = s.makeSnappyProtobufHashdex(&prompb.WriteRequest{}) + s.Require().NoError(err) + + shardsInnerSeries = cppbridge.NewShardsInnerSeries(s.numberOfShards) + shardsRelabeledSeries = cppbridge.NewShardsRelabeledSeries(s.numberOfShards) + stats, hasReallocations, err = pgr.Relabeling( + s.baseCtx, + s.inputLss, + s.targetLss, + state, + h, + shardsInnerSeries, + shardsRelabeledSeries, + ) + s.Require().NoError(err) + s.Equal(cppbridge.RelabelerStats{0, 0, 0}, stats) + s.False(hasReallocations) + s.Equal(uint64(1), shardsInnerSeries[0].Size()) +} + +func (s *PerGoroutineRelabelerSuite) TestRelabelingWithStalenansFromCacheTrue() { + h, err := s.makeSnappyProtobufHashdex(&prompb.WriteRequest{ + Timeseries: []prompb.TimeSeries{ + { + Labels: []prompb.Label{ + {Name: "__name__", Value: "value"}, + {Name: "job", Value: "abc"}, + {Name: "instance", Value: "value1"}, + }, + Samples: []prompb.Sample{ + {Value: 0.1, Timestamp: time.Now().UnixMilli()}, + }, + }, + }, + }) + s.Require().NoError(err) + + shardsInnerSeries := cppbridge.NewShardsInnerSeries(s.numberOfShards) + shardsRelabeledSeries := cppbridge.NewShardsRelabeledSeries(s.numberOfShards) + + statelessRelabeler, err := cppbridge.NewStatelessRelabeler(s.rCfgs) + s.Require().NoError(err) + + state := cppbridge.NewStateV2WithoutLock() + state.SetStatelessRelabeler(statelessRelabeler) + state.SetRelabelerOptions(&s.options) + state.EnableTrackStaleness() + state.Reconfigure(0, s.numberOfShards) + + pgr := cppbridge.NewPerGoroutineRelabeler(s.numberOfShards, 0) + stats, hasReallocations, err := pgr.Relabeling( + s.baseCtx, + s.inputLss, + s.targetLss, + state, + h, + shardsInnerSeries, + shardsRelabeledSeries, + ) + s.Require().NoError(err) + s.Equal(cppbridge.RelabelerStats{1, 1, 0}, stats) + s.True(hasReallocations) + + stats, ok, err := pgr.RelabelingFromCache( + s.baseCtx, + s.inputLss, + s.targetLss, + state, + h, + shardsInnerSeries, + ) + s.Require().NoError(err) + s.Equal(cppbridge.RelabelerStats{1, 0, 0}, stats) + s.True(ok) + + h, err = s.makeSnappyProtobufHashdex(&prompb.WriteRequest{}) + s.Require().NoError(err) + + shardsInnerSeries = cppbridge.NewShardsInnerSeries(s.numberOfShards) + shardsRelabeledSeries = cppbridge.NewShardsRelabeledSeries(s.numberOfShards) + stats, hasReallocations, err = pgr.Relabeling( + s.baseCtx, + s.inputLss, + s.targetLss, + state, + h, + shardsInnerSeries, + shardsRelabeledSeries, + ) + s.Require().NoError(err) + s.Equal(cppbridge.RelabelerStats{0, 0, 0}, stats) + s.False(hasReallocations) + s.Equal(uint64(1), shardsInnerSeries[0].Size()) +} + +func (s *PerGoroutineRelabelerSuite) TestRelabelingWithStalenansFromCacheFalse() { + h, err := s.makeSnappyProtobufHashdex(&prompb.WriteRequest{ + Timeseries: []prompb.TimeSeries{ + { + Labels: []prompb.Label{ + {Name: "__name__", Value: "value"}, + {Name: "job", Value: "abc"}, + {Name: "instance", Value: "value1"}, + }, + Samples: []prompb.Sample{ + {Value: 0.1, Timestamp: time.Now().UnixMilli()}, + }, + }, + { + Labels: []prompb.Label{ + {Name: "__name__", Value: "value"}, + {Name: "instance", Value: "value1"}, + }, + Samples: []prompb.Sample{ + {Value: 0.1, Timestamp: time.Now().UnixMilli()}, + }, + }, + }, + }) + s.Require().NoError(err) + + shardsInnerSeries := cppbridge.NewShardsInnerSeries(s.numberOfShards) + state := cppbridge.NewStateV2WithoutLock() + state.SetRelabelerOptions(&s.options) + state.EnableTrackStaleness() + state.Reconfigure(0, s.numberOfShards) + + pgr := cppbridge.NewPerGoroutineRelabeler(s.numberOfShards, 0) + stats, ok, err := pgr.RelabelingFromCache( + s.baseCtx, + s.inputLss, + s.targetLss, + state, + h, + shardsInnerSeries, + ) + s.Require().NoError(err) + s.Equal(cppbridge.RelabelerStats{0, 0, 0}, stats) + s.False(ok) +} + +func (s *PerGoroutineRelabelerSuite) TestRelabelingWithStalenansFromCachePartially() { + ts := time.Now().UnixMilli() + h1, err := s.makeSnappyProtobufHashdex(&prompb.WriteRequest{ + Timeseries: []prompb.TimeSeries{ + { + Labels: []prompb.Label{ + {Name: "__name__", Value: "value0"}, + {Name: "job", Value: "abc"}, + {Name: "instance", Value: "value0"}, + }, + Samples: []prompb.Sample{ + {Value: 0.1, Timestamp: ts}, + }, + }, + { + Labels: []prompb.Label{ + {Name: "__name__", Value: "value1"}, + {Name: "job", Value: "abc"}, + {Name: "instance", Value: "value1"}, + }, + Samples: []prompb.Sample{ + {Value: 0.1, Timestamp: ts}, + }, + }, + { + Labels: []prompb.Label{ + {Name: "__name__", Value: "value2"}, + {Name: "job", Value: "abc"}, + {Name: "instance", Value: "value2"}, + }, + Samples: []prompb.Sample{ + {Value: 0.1, Timestamp: ts}, + }, + }, + { + Labels: []prompb.Label{ + {Name: "__name__", Value: "value3"}, + {Name: "job", Value: "abc"}, + {Name: "instance", Value: "value3"}, + }, + Samples: []prompb.Sample{ + {Value: 0.1, Timestamp: ts}, + }, + }, + { + Labels: []prompb.Label{ + {Name: "__name__", Value: "value3"}, + {Name: "instance", Value: "value3"}, + }, + Samples: []prompb.Sample{ + {Value: 0.1, Timestamp: ts}, + }, + }, + { + Labels: []prompb.Label{ + {Name: "__name__", Value: "value4"}, + {Name: "instance", Value: "value4"}, + }, + Samples: []prompb.Sample{ + {Value: 0.1, Timestamp: ts}, + }, + }, + { + Labels: []prompb.Label{ + {Name: "__name__", Value: "value5"}, + {Name: "instance", Value: "value5"}, + }, + Samples: []prompb.Sample{ + {Value: 0.1, Timestamp: ts}, + }, + }, + { + Labels: []prompb.Label{ + {Name: "__name__", Value: "value6"}, + {Name: "instance", Value: "value6"}, + }, + Samples: []prompb.Sample{ + {Value: 0.1, Timestamp: ts}, + }, + }, + }, + }) + s.Require().NoError(err) + + ts += 6000 + h2, err := s.makeSnappyProtobufHashdex(&prompb.WriteRequest{ + Timeseries: []prompb.TimeSeries{ + { + Labels: []prompb.Label{ + {Name: "__name__", Value: "value0"}, + {Name: "job", Value: "abc"}, + {Name: "instance", Value: "value0"}, + }, + Samples: []prompb.Sample{ + {Value: 0.1, Timestamp: ts}, + }, + }, + { + Labels: []prompb.Label{ + {Name: "__name__", Value: "value1"}, + {Name: "job", Value: "abc"}, + {Name: "instance", Value: "value1"}, + }, + Samples: []prompb.Sample{ + {Value: 0.1, Timestamp: ts}, + }, + }, + { + Labels: []prompb.Label{ + {Name: "__name__", Value: "value2"}, + {Name: "job", Value: "abc"}, + {Name: "instance", Value: "value2"}, + }, + Samples: []prompb.Sample{ + {Value: 0.1, Timestamp: ts}, + }, + }, + { + Labels: []prompb.Label{ + {Name: "__name__", Value: "value3"}, + {Name: "job", Value: "abc"}, + {Name: "instance", Value: "value3"}, + }, + Samples: []prompb.Sample{ + {Value: 0.1, Timestamp: ts}, + }, + }, + { + Labels: []prompb.Label{ + {Name: "__name__", Value: "value4"}, + {Name: "job", Value: "abc"}, + {Name: "instance", Value: "value4"}, + }, + Samples: []prompb.Sample{ + {Value: 0.1, Timestamp: ts}, + }, + }, + }, + }) + s.Require().NoError(err) + + shardsInnerSeries := cppbridge.NewShardsInnerSeries(s.numberOfShards) + shardsRelabeledSeries := cppbridge.NewShardsRelabeledSeries(s.numberOfShards) + + statelessRelabeler, err := cppbridge.NewStatelessRelabeler(s.rCfgs) + s.Require().NoError(err) + + state := cppbridge.NewStateV2WithoutLock() + state.SetRelabelerOptions(&s.options) + state.SetStatelessRelabeler(statelessRelabeler) + state.EnableTrackStaleness() + state.Reconfigure(0, s.numberOfShards) + + pgr := cppbridge.NewPerGoroutineRelabeler(s.numberOfShards, 0) + stats, hasReallocations, err := pgr.Relabeling( + s.baseCtx, + s.inputLss, + s.targetLss, + state, + h1, + shardsInnerSeries, + shardsRelabeledSeries, + ) + s.Require().NoError(err) + s.Equal(cppbridge.RelabelerStats{4, 4, 4}, stats) + s.True(hasReallocations) + + shardsInnerSeries = cppbridge.NewShardsInnerSeries(s.numberOfShards) + stats, ok, err := pgr.RelabelingFromCache( + s.baseCtx, + s.inputLss, + s.targetLss, + state, + h2, + shardsInnerSeries, + ) + s.Require().NoError(err) + s.Equal(cppbridge.RelabelerStats{4, 0, 0}, stats) + s.False(ok) + s.Equal(uint64(4), shardsInnerSeries[0].Size()) + + stats, _, err = pgr.Relabeling( + s.baseCtx, + s.inputLss, + s.targetLss, + state, + h2, + shardsInnerSeries, + shardsRelabeledSeries, + ) + s.Require().NoError(err) + s.Equal(cppbridge.RelabelerStats{1, 1, 0}, stats) + s.Equal(uint64(5), shardsInnerSeries[0].Size()) + + h, err := s.makeSnappyProtobufHashdex(&prompb.WriteRequest{}) + s.Require().NoError(err) + + shardsInnerSeries = cppbridge.NewShardsInnerSeries(s.numberOfShards) + shardsRelabeledSeries = cppbridge.NewShardsRelabeledSeries(s.numberOfShards) + stats, hasReallocations, err = pgr.Relabeling( + s.baseCtx, + s.inputLss, + s.targetLss, + state, + h, + shardsInnerSeries, + shardsRelabeledSeries, + ) + s.Require().NoError(err) + s.Equal(cppbridge.RelabelerStats{0, 0, 0}, stats) + s.False(hasReallocations) + s.Equal(uint64(5), shardsInnerSeries[0].Size()) +} + +func (s *PerGoroutineRelabelerSuite) makeSnappyProtobufHashdex( + wr *prompb.WriteRequest, +) (cppbridge.ShardedData, error) { + data, err := wr.Marshal() + if err != nil { + return nil, err + } + + return cppbridge.NewWALSnappyProtobufHashdex(snappy.Encode(nil, data), s.hlimits) +} + +// +// StateV2Suite +// + +type StateV2Suite struct { + suite.Suite +} + +func TestStateV2Suite(t *testing.T) { + suite.Run(t, new(StateV2Suite)) +} + +func (s *StateV2Suite) TestInitState() { + s.initState(cppbridge.NewStateV2()) + s.initState(cppbridge.NewStateV2WithoutLock()) +} + +func (s *StateV2Suite) initState(state *cppbridge.StateV2) { + s.Panics(func() { state.CacheByShard(0) }) + s.GreaterOrEqual(state.DefTimestamp(), time.Now().UnixMilli()) + + newDeftime := time.Now().Add(5 * time.Minute).UnixMilli() + state.SetDefTimestamp(newDeftime) + s.Equal(newDeftime, state.DefTimestamp()) + + s.False(state.IsTransition()) + s.Equal(cppbridge.RelabelerOptions{}, state.RelabelerOptions()) + s.Panics(func() { state.StaleNansStateByShard(0) }) + s.False(state.TrackStaleness()) +} + +func (s *StateV2Suite) TestStateReconfigure() { + s.stateReconfigure(cppbridge.NewStateV2()) + s.stateReconfigure(cppbridge.NewStateV2WithoutLock()) +} + +func (s *StateV2Suite) stateReconfigure(state *cppbridge.StateV2) { + state.Reconfigure(0, 1) + + s.NotNil(state.CacheByShard(0)) + s.False(state.TrackStaleness()) + s.Panics(func() { state.StaleNansStateByShard(0) }) +} + +func (s *StateV2Suite) TestStateReconfigureWithoutReconfigure() { + s.stateReconfigureWithoutReconfigure(cppbridge.NewStateV2()) + s.stateReconfigureWithoutReconfigure(cppbridge.NewStateV2WithoutLock()) +} + +func (s *StateV2Suite) stateReconfigureWithoutReconfigure(state *cppbridge.StateV2) { + state.Reconfigure(0, 1) + + cache1 := state.CacheByShard(0) + s.NotNil(cache1) + + state.Reconfigure(0, 1) + cache2 := state.CacheByShard(0) + s.NotNil(cache2) + s.Equal(cache1, cache2) +} + +func (s *StateV2Suite) TestStateReconfigureNumberOfShards() { + s.stateReconfigureNumberOfShards(cppbridge.NewStateV2()) + s.stateReconfigureNumberOfShards(cppbridge.NewStateV2WithoutLock()) +} + +func (s *StateV2Suite) stateReconfigureNumberOfShards(state *cppbridge.StateV2) { + state.EnableTrackStaleness() + state.Reconfigure(0, 2) + + cache0 := state.CacheByShard(0) + s.NotNil(cache0) + cache1 := state.CacheByShard(1) + s.NotNil(cache1) + + state.Reconfigure(1, 1) + newCache0 := state.CacheByShard(0) + s.NotNil(newCache0) + s.NotEqual(cache0, newCache0) + s.Panics(func() { state.CacheByShard(1) }) +} + +func (s *StateV2Suite) TestStateReconfigureTrackStaleness() { + s.stateReconfigureTrackStaleness(cppbridge.NewStateV2()) + s.stateReconfigureTrackStaleness(cppbridge.NewStateV2WithoutLock()) +} + +func (s *StateV2Suite) stateReconfigureTrackStaleness(state *cppbridge.StateV2) { + state.EnableTrackStaleness() + state.Reconfigure(0, 1) + + s.NotNil(state.CacheByShard(0)) + s.True(state.TrackStaleness()) + s.NotNil(state.StaleNansStateByShard(0)) +} + +func (s *StateV2Suite) TestStatelessRelabeler() { + s.statelessRelabeler(cppbridge.NewStateV2()) + s.statelessRelabeler(cppbridge.NewStateV2WithoutLock()) +} + +func (s *StateV2Suite) statelessRelabeler(state *cppbridge.StateV2) { + s.Panics(func() { state.StatelessRelabeler() }) + + statelessRelabeler, err := cppbridge.NewStatelessRelabeler([]*cppbridge.RelabelConfig{}) + s.Require().NoError(err) + + state.SetStatelessRelabeler(statelessRelabeler) + s.NotNil(state.StatelessRelabeler()) +} + +func (s *StateV2Suite) TestInitTransitionStateV2() { + s.initTransitionState(cppbridge.NewTransitionStateV2()) + s.initTransitionState(cppbridge.NewTransitionStateV2WithoutLock()) +} + +func (s *StateV2Suite) initTransitionState(state *cppbridge.StateV2) { + s.True(state.IsTransition()) + s.Equal(cppbridge.RelabelerOptions{}, state.RelabelerOptions()) + s.Panics(func() { state.CacheByShard(0) }) + s.Panics(func() { state.StaleNansStateByShard(0) }) + s.False(state.TrackStaleness()) +} + +func (s *StateV2Suite) TestStateTransitionReconfigure() { + s.stateTransitionReconfigure(cppbridge.NewTransitionStateV2()) + s.stateTransitionReconfigure(cppbridge.NewTransitionStateV2WithoutLock()) +} + +func (s *StateV2Suite) stateTransitionReconfigure(state *cppbridge.StateV2) { + state.Reconfigure(0, 1) + + s.False(state.TrackStaleness()) + s.Panics(func() { state.CacheByShard(0) }) + s.Panics(func() { state.StaleNansStateByShard(0) }) +} + +func (s *StateV2Suite) TestStateTransitionReconfigureTrackStaleness() { + s.stateTransitionReconfigureTrackStaleness(cppbridge.NewTransitionStateV2()) + s.stateTransitionReconfigureTrackStaleness(cppbridge.NewTransitionStateV2WithoutLock()) +} + +func (s *StateV2Suite) stateTransitionReconfigureTrackStaleness(state *cppbridge.StateV2) { + s.Panics(func() { state.EnableTrackStaleness() }) + state.Reconfigure(0, 1) + + s.False(state.TrackStaleness()) + s.Panics(func() { state.CacheByShard(0) }) + s.Panics(func() { state.StaleNansStateByShard(0) }) +} + +func (s *StateV2Suite) TestStatelessRelabelerTransition() { + s.statelessRelabelerTransition(cppbridge.NewTransitionStateV2()) + s.statelessRelabelerTransition(cppbridge.NewTransitionStateV2WithoutLock()) +} + +func (s *StateV2Suite) statelessRelabelerTransition(state *cppbridge.StateV2) { + s.Panics(func() { state.StatelessRelabeler() }) + + statelessRelabeler, err := cppbridge.NewStatelessRelabeler([]*cppbridge.RelabelConfig{}) + s.Require().NoError(err) + + s.Panics(func() { state.SetStatelessRelabeler(statelessRelabeler) }) + s.Panics(func() { state.StatelessRelabeler() }) +} diff --git a/pp/go/hatracker/high_availability_tracker.go b/pp/go/hatracker/high_availability_tracker.go new file mode 100644 index 0000000000..6bb557de96 --- /dev/null +++ b/pp/go/hatracker/high_availability_tracker.go @@ -0,0 +1,164 @@ +package hatracker + +import ( + "sync" + "time" + + "github.com/jonboulle/clockwork" + "github.com/prometheus/client_golang/prometheus" + + "github.com/prometheus/prometheus/pp/go/util" +) + +var ( + defaultHAOverTime = 30 * time.Second + defaultHACleanup = 10 * time.Minute +) + +// HighAvailabilityTracker track the replica we're accepting samples from +// for each HA cluster we know about. +type HighAvailabilityTracker struct { + stop chan struct{} + storage *sync.Map + overTime int64 + clock clockwork.Clock + // stat + electedReplicaChanges *prometheus.CounterVec + electedReplicaTimestamp *prometheus.GaugeVec + lastElectionTimestamp *prometheus.GaugeVec + deletedReplicas prometheus.Counter + dropedReplicas prometheus.Counter +} + +// NewHighAvailabilityTracker init new [HighAvailabilityTracker]. +func NewHighAvailabilityTracker( + clock clockwork.Clock, + registerer prometheus.Registerer, +) *HighAvailabilityTracker { + factory := util.NewUnconflictRegisterer(registerer) + ha := &HighAvailabilityTracker{ + storage: new(sync.Map), + overTime: int64(defaultHAOverTime.Seconds()), + clock: clock, + stop: make(chan struct{}), + electedReplicaChanges: factory.NewCounterVec( + prometheus.CounterOpts{ + Name: "prompp_high_availability_tracker_elected_replica_changes", + Help: "The total number of times the elected replica has changed for cluster.", + }, + []string{"cluster", "replica"}, + ), + electedReplicaTimestamp: factory.NewGaugeVec( + prometheus.GaugeOpts{ + Name: "prompp_high_availability_tracker_elected_replica_timestamp_seconds", + Help: "The timestamp stored for the currently elected replica.", + }, + []string{"cluster", "replica"}, + ), + lastElectionTimestamp: factory.NewGaugeVec( + prometheus.GaugeOpts{ + Name: "prompp_high_availability_tracker_last_election_timestamp_seconds", + Help: "The timestamp stored for the currently elected replica.", + }, + []string{"cluster", "replica"}, + ), + deletedReplicas: factory.NewCounter( + prometheus.CounterOpts{ + Name: "prompp_high_availability_tracker_deleted_total", + Help: "Number of elected replicas deleted from store.", + }, + ), + dropedReplicas: factory.NewCounter( + prometheus.CounterOpts{ + Name: "prompp_high_availability_tracker_droped_total", + Help: "Number of elected replicas droped.", + }, + ), + } + + go ha.cleanup() + + return ha +} + +// cleanup delete old replicas. +func (ha *HighAvailabilityTracker) cleanup() { + ticker := ha.clock.NewTicker(defaultHACleanup) + defer ticker.Stop() + + for { + select { + case <-ticker.Chan(): + now := ha.clock.Now().Unix() + ha.storage.Range(func(key, value any) bool { + hv := value.(*haValue) + hv.mx.Lock() + if now-hv.electedAt >= int64(defaultHACleanup.Seconds()) { + ha.storage.Delete(key) + ha.electedReplicaChanges.DeleteLabelValues(key.(string), hv.value) + ha.electedReplicaTimestamp.DeleteLabelValues(key.(string), hv.value) + ha.deletedReplicas.Inc() + } + hv.mx.Unlock() + return true + }) + case <-ha.stop: + return + } + } +} + +// IsDrop check whether data needs to be sent or discarded immediately. +func (ha *HighAvailabilityTracker) IsDrop(cluster, replica string) bool { + if replica == "" { + return false + } + now := ha.clock.Now().Unix() + val, ok := ha.storage.LoadOrStore( + cluster, + &haValue{electedAt: now, value: replica, mx: new(sync.Mutex)}, + ) + if !ok { + ha.electedReplicaChanges.With(prometheus.Labels{"cluster": cluster, "replica": replica}).Inc() + ha.electedReplicaTimestamp.With(prometheus.Labels{"cluster": cluster, "replica": replica}).Set(float64(now)) + return false + } + + hv := val.(*haValue) + hv.mx.Lock() + if hv.value == replica { + hv.electedAt = now + ha.electedReplicaTimestamp.With(prometheus.Labels{"cluster": cluster, "replica": replica}).Set(float64(now)) + hv.mx.Unlock() + return false + } + + if (now - hv.electedAt) >= ha.overTime { + ha.lastElectionTimestamp.With(prometheus.Labels{"cluster": cluster, "replica": hv.value}).Set(float64(now)) + hv.value = replica + hv.electedAt = now + ha.electedReplicaChanges.With(prometheus.Labels{"cluster": cluster, "replica": replica}).Inc() + ha.electedReplicaTimestamp.With(prometheus.Labels{"cluster": cluster, "replica": replica}).Set(float64(now)) + hv.mx.Unlock() + return true + } + hv.mx.Unlock() + ha.dropedReplicas.Inc() + return true +} + +// Destroy clear all clusters and stop work. +func (ha *HighAvailabilityTracker) Destroy() { + close(ha.stop) + ha.storage.Range(func(key, _ any) bool { + ha.storage.Delete(key) + return true + }) +} + +// haValue value for HighAvailabilityTracker. +type haValue struct { + mx *sync.Mutex + value string + electedAt int64 +} diff --git a/pp/go/logger/logger.go b/pp/go/logger/logger.go new file mode 100644 index 0000000000..548517806c --- /dev/null +++ b/pp/go/logger/logger.go @@ -0,0 +1,19 @@ +package logger + +func noop(string, ...any) {} + +// These variables are set by the common log package. +var ( + Errorf = noop + Warnf = noop + Infof = noop + Debugf = noop +) + +// Unset logger funcs to NoOp +func Unset() { + Errorf = noop + Warnf = noop + Infof = noop + Debugf = noop +} diff --git a/pp/go/relabeler/appender/appender.go b/pp/go/relabeler/appender/appender.go index b7e10d370a..90c56fdcd5 100644 --- a/pp/go/relabeler/appender/appender.go +++ b/pp/go/relabeler/appender/appender.go @@ -6,10 +6,12 @@ import ( "fmt" "time" + "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/prometheus/pp/go/cppbridge" "github.com/prometheus/prometheus/pp/go/relabeler" "github.com/prometheus/prometheus/pp/go/relabeler/logger" "github.com/prometheus/prometheus/pp/go/relabeler/querier" + "github.com/prometheus/prometheus/pp/go/util" "github.com/prometheus/prometheus/pp/go/util/locker" "github.com/prometheus/prometheus/storage" ) @@ -20,6 +22,10 @@ type QueryableAppender struct { head relabeler.Head distributor relabeler.Distributor querierMetrics *querier.Metrics + + appendDuration prometheus.Histogram + waitLockRotateDuration prometheus.Gauge + rotationDuration prometheus.Gauge } func NewQueryableAppender( @@ -27,13 +33,41 @@ func NewQueryableAppender( head relabeler.Head, distributor relabeler.Distributor, querierMetrics *querier.Metrics, + registerer prometheus.Registerer, ) *QueryableAppender { + factory := util.NewUnconflictRegisterer(registerer) return &QueryableAppender{ ctx: ctx, wlocker: locker.NewWeighted(2 * head.Concurrency()), // x2 for back pressure head: head, distributor: distributor, querierMetrics: querierMetrics, + + appendDuration: factory.NewHistogram( + prometheus.HistogramOpts{ + Name: "prompp_head_append_duration", + Help: "Append to head duration in microseconds", + Buckets: []float64{ + 50, 100, 250, 500, 750, + 1000, 2500, 5000, 7500, + 10000, 25000, 50000, 75000, + 100000, 500000, + }, + }, + ), + + waitLockRotateDuration: factory.NewGauge( + prometheus.GaugeOpts{ + Name: "prompp_head_wait_lock_rotate_duration", + Help: "The duration of the lock wait for rotation in nanoseconds", + }, + ), + rotationDuration: factory.NewGauge( + prometheus.GaugeOpts{ + Name: "prompp_head_rotate_duration", + Help: "The duration of the rotate in nanoseconds", + }, + ), } } @@ -63,7 +97,7 @@ func (qa *QueryableAppender) AppendWithStaleNans( defer runlock() defer func() { - qa.querierMetrics.AppendDuration.Observe(float64(time.Since(start).Microseconds())) + qa.appendDuration.Observe(float64(time.Since(start).Microseconds())) }() data, stats, err := qa.head.Append(ctx, incomingData, state, relabelerID, commitToWal) @@ -141,11 +175,11 @@ func (qa *QueryableAppender) Rotate(ctx context.Context) error { if err != nil { return fmt.Errorf("Rotate: weighted locker: %w", err) } - qa.querierMetrics.WaitLockRotateDuration.Set(float64(time.Since(start).Nanoseconds())) + qa.waitLockRotateDuration.Set(float64(time.Since(start).Nanoseconds())) defer unlock() defer func() { - qa.querierMetrics.RotationDuration.Set(float64(time.Since(start).Nanoseconds())) + qa.rotationDuration.Set(float64(time.Since(start).Nanoseconds())) }() qa.head.MergeOutOfOrderChunks() diff --git a/pp/go/relabeler/appender/appender_test.go b/pp/go/relabeler/appender/appender_test.go index 0339de94ff..46f8c679f1 100644 --- a/pp/go/relabeler/appender/appender_test.go +++ b/pp/go/relabeler/appender/appender_test.go @@ -126,7 +126,7 @@ func (s *AppenderSuite) TestManagerRelabelerKeep() { defer func() { _ = hd.Close() }() s.T().Log("make appender") - app := appender.NewQueryableAppender(s.baseCtx, hd, dstrb, s.metrics) + app := appender.NewQueryableAppender(s.baseCtx, hd, dstrb, s.metrics, nil) hlimits := cppbridge.DefaultWALHashdexLimits() @@ -269,7 +269,7 @@ func (s *AppenderSuite) TestManagerRelabelerRelabeling() { s.Require().NoError(err) defer func() { _ = hd.Close() }() s.T().Log("make appender") - app := appender.NewQueryableAppender(s.baseCtx, hd, dstrb, s.metrics) + app := appender.NewQueryableAppender(s.baseCtx, hd, dstrb, s.metrics, nil) hlimits := cppbridge.DefaultWALHashdexLimits() @@ -423,7 +423,7 @@ func (s *AppenderSuite) TestManagerRelabelerRelabelingAddNewLabel() { s.Require().NoError(err) defer func() { _ = hd.Close() }() s.T().Log("make appender") - app := appender.NewQueryableAppender(s.baseCtx, hd, dstrb, s.metrics) + app := appender.NewQueryableAppender(s.baseCtx, hd, dstrb, s.metrics, nil) hlimits := cppbridge.DefaultWALHashdexLimits() @@ -582,7 +582,7 @@ func (s *AppenderSuite) TestManagerRelabelerRelabelingWithExternalLabelsEnd() { defer func() { _ = hd.Close() }() s.Require().NoError(err) s.T().Log("make appender") - app := appender.NewQueryableAppender(s.baseCtx, hd, dstrb, s.metrics) + app := appender.NewQueryableAppender(s.baseCtx, hd, dstrb, s.metrics, nil) hlimits := cppbridge.DefaultWALHashdexLimits() s.T().Log("append first data") @@ -739,7 +739,7 @@ func (s *AppenderSuite) TestManagerRelabelerRelabelingWithExternalLabelsRelabel( s.Require().NoError(err) defer func() { _ = hd.Close() }() s.T().Log("make appender") - app := appender.NewQueryableAppender(s.baseCtx, hd, dstrb, s.metrics) + app := appender.NewQueryableAppender(s.baseCtx, hd, dstrb, s.metrics, nil) hlimits := cppbridge.DefaultWALHashdexLimits() @@ -901,7 +901,7 @@ func (s *AppenderSuite) TestManagerRelabelerRelabelingWithTargetLabels() { s.Require().NoError(err) defer func() { _ = hd.Close() }() s.T().Log("make appender") - app := appender.NewQueryableAppender(s.baseCtx, hd, dstrb, s.metrics) + app := appender.NewQueryableAppender(s.baseCtx, hd, dstrb, s.metrics, nil) hlimits := cppbridge.DefaultWALHashdexLimits() @@ -1093,7 +1093,7 @@ func (s *AppenderSuite) TestManagerRelabelerRelabelingWithTargetLabels_Conflicti s.Require().NoError(err) defer func() { _ = hd.Close() }() s.T().Log("make appender") - app := appender.NewQueryableAppender(s.baseCtx, hd, dstrb, s.metrics) + app := appender.NewQueryableAppender(s.baseCtx, hd, dstrb, s.metrics, nil) hlimits := cppbridge.DefaultWALHashdexLimits() @@ -1286,7 +1286,7 @@ func (s *AppenderSuite) TestManagerRelabelerRelabelingWithTargetLabels_Conflicti s.Require().NoError(err) defer func() { _ = hd.Close() }() s.T().Log("make appender") - app := appender.NewQueryableAppender(s.baseCtx, hd, dstrb, s.metrics) + app := appender.NewQueryableAppender(s.baseCtx, hd, dstrb, s.metrics, nil) hlimits := cppbridge.DefaultWALHashdexLimits() @@ -1516,7 +1516,7 @@ func (s *AppenderSuite) TestManagerRelabelerRelabelingWithRotate() { rotatableHead := appender.NewRotatableHead(hd, noOpStorage{}, builder, appender.NoOpHeadActivator{}) s.T().Log("make appender") - app := appender.NewQueryableAppender(s.baseCtx, rotatableHead, dstrb, s.metrics) + app := appender.NewQueryableAppender(s.baseCtx, rotatableHead, dstrb, s.metrics, nil) rotationTimer := relabeler.NewRotateTimer(clock, appender.DefaultRotateDuration) commitTimer := appender.NewConstantIntervalTimer(clock, appender.DefaultCommitTimeout) @@ -1978,7 +1978,7 @@ func (s *AppenderSuite) TestManagerRelabelerKeepWithStaleNans() { s.Require().NoError(err) defer func() { _ = hd.Close() }() s.T().Log("make appender") - app := appender.NewQueryableAppender(s.baseCtx, hd, dstrb, s.metrics) + app := appender.NewQueryableAppender(s.baseCtx, hd, dstrb, s.metrics, nil) hlimits := cppbridge.DefaultWALHashdexLimits() @@ -2112,7 +2112,7 @@ func (s *AppenderSuite) TestManagerRelabelerKeepWithStaleNans_WithNullTimestamp( s.Require().NoError(err) defer func() { _ = hd.Close() }() s.T().Log("make appender") - app := appender.NewQueryableAppender(s.baseCtx, hd, dstrb, s.metrics) + app := appender.NewQueryableAppender(s.baseCtx, hd, dstrb, s.metrics, nil) hlimits := cppbridge.DefaultWALHashdexLimits() @@ -2243,7 +2243,7 @@ func (s *AppenderSuite) TestManagerRelabelerKeepWithStaleNans_HonorTimestamps() s.Require().NoError(err) defer func() { _ = hd.Close() }() s.T().Log("make appender") - app := appender.NewQueryableAppender(s.baseCtx, hd, dstrb, s.metrics) + app := appender.NewQueryableAppender(s.baseCtx, hd, dstrb, s.metrics, nil) hlimits := cppbridge.DefaultWALHashdexLimits() @@ -2423,7 +2423,7 @@ func (s *AppenderSuite) TestManagerRelabelerRelabelingWithRotateWithStaleNans() rotatableHead := appender.NewRotatableHead(hd, noOpStorage{}, builder, appender.NoOpHeadActivator{}) s.T().Log("make appender") - app := appender.NewQueryableAppender(s.baseCtx, rotatableHead, dstrb, s.metrics) + app := appender.NewQueryableAppender(s.baseCtx, rotatableHead, dstrb, s.metrics, nil) rotationTimer := relabeler.NewRotateTimer(clock, appender.DefaultRotateDuration) commitTimer := appender.NewConstantIntervalTimer(clock, appender.DefaultCommitTimeout) diff --git a/pp/go/relabeler/block/writer.go b/pp/go/relabeler/block/writer.go index 6c45d7c8d5..9cb1f98b4e 100644 --- a/pp/go/relabeler/block/writer.go +++ b/pp/go/relabeler/block/writer.go @@ -316,7 +316,7 @@ func (w *Writer) createWriters(shard relabeler.Shard) (blockWriters, error) { var writers blockWriters shard.DataStorageRLock() - timeInterval := shard.DataStorage().TimeInterval() + timeInterval := shard.DataStorage().TimeInterval(false) shard.DataStorageRUnlock() quantStart := (timeInterval.MinT / w.blockDurationMs) * w.blockDurationMs diff --git a/pp/go/relabeler/head/headtest/fixtures.go b/pp/go/relabeler/head/headtest/fixtures.go index 33d94c6a02..43ae38b935 100644 --- a/pp/go/relabeler/head/headtest/fixtures.go +++ b/pp/go/relabeler/head/headtest/fixtures.go @@ -23,9 +23,9 @@ func (s *TimeSeries) AppendSamples(samples ...cppbridge.Sample) { func (s *TimeSeries) toModelTimeSeries() []model.TimeSeries { lsBuilder := model.NewLabelSetBuilder() - for i := range s.Labels { - lsBuilder.Add(s.Labels[i].Name, s.Labels[i].Value) - } + s.Labels.Range(func(l labels.Label) { + lsBuilder.Add(l.Name, l.Value) + }) ls := lsBuilder.Build() diff --git a/pp/go/relabeler/head/load_test.go b/pp/go/relabeler/head/load_test.go index 6bff0360da..8bf4d37350 100644 --- a/pp/go/relabeler/head/load_test.go +++ b/pp/go/relabeler/head/load_test.go @@ -173,9 +173,10 @@ func (s *HeadLoadSuite) TestErrorOpenShardFileInAllShards() { func (s *HeadLoadSuite) TestLoadWithDisabledDataUnloading() { // Arrange sourceHead := s.mustCreateHead(0) + series := []headtest.TimeSeries{ { - Labels: labels.Labels{{Name: "__name__", Value: "wal_metric"}}, + Labels: labels.FromStrings("__name__", "wal_metric"), Samples: []cppbridge.Sample{ {Timestamp: 0, Value: 1}, {Timestamp: 1, Value: 2}, @@ -208,7 +209,7 @@ func (s *HeadLoadSuite) TestAppendAfterLoad() { sourceHead := s.mustCreateHead(0) series := []headtest.TimeSeries{ { - Labels: labels.Labels{{Name: "__name__", Value: "wal_metric"}}, + Labels: labels.FromStrings("__name__", "wal_metric"), Samples: []cppbridge.Sample{ {Timestamp: 0, Value: 1}, {Timestamp: 1, Value: 2}, @@ -224,7 +225,7 @@ func (s *HeadLoadSuite) TestAppendAfterLoad() { loadedHead := s.mustLoadHead(0) s.appendTimeSeries(loadedHead, []headtest.TimeSeries{ { - Labels: labels.Labels{{Name: "__name__", Value: "wal_metric"}}, + Labels: labels.FromStrings("__name__", "wal_metric"), Samples: []cppbridge.Sample{ {Timestamp: 3, Value: 4}, }, @@ -239,7 +240,6 @@ func (s *HeadLoadSuite) TestAppendAfterLoad() { series[0].Samples = append(series[0].Samples, cppbridge.Sample{Timestamp: 3, Value: 4}) s.Equal(series, actual) s.Require().NoError(err) - } func (s *HeadLoadSuite) TestLoadWithEnabledDataUnloading() { @@ -247,7 +247,7 @@ func (s *HeadLoadSuite) TestLoadWithEnabledDataUnloading() { sourceHead := s.mustCreateHead(0) series1 := []headtest.TimeSeries{ { - Labels: labels.Labels{{Name: "__name__", Value: "wal_metric"}}, + Labels: labels.FromStrings("__name__", "wal_metric"), Samples: []cppbridge.Sample{ {Timestamp: 0, Value: 1}, {Timestamp: 1, Value: 2}, @@ -258,7 +258,7 @@ func (s *HeadLoadSuite) TestLoadWithEnabledDataUnloading() { s.appendTimeSeries(sourceHead, series1) series2 := []headtest.TimeSeries{ { - Labels: labels.Labels{{Name: "__name__", Value: "wal_metric"}}, + Labels: labels.FromStrings("__name__", "wal_metric"), Samples: []cppbridge.Sample{ {Timestamp: 100, Value: 1}, {Timestamp: 101, Value: 2}, @@ -290,7 +290,7 @@ func (s *HeadLoadSuite) TestLoadWithDataUnloading() { sourceHead := s.mustCreateHead(unloadDataStorageInterval) series1 := []headtest.TimeSeries{ { - Labels: labels.Labels{{Name: "__name__", Value: "wal_metric"}}, + Labels: labels.FromStrings("__name__", "wal_metric"), Samples: []cppbridge.Sample{ {Timestamp: 0, Value: 1}, {Timestamp: 1, Value: 2}, @@ -301,7 +301,7 @@ func (s *HeadLoadSuite) TestLoadWithDataUnloading() { s.appendTimeSeries(sourceHead, series1) series2 := []headtest.TimeSeries{ { - Labels: labels.Labels{{Name: "__name__", Value: "wal_metric"}}, + Labels: labels.FromStrings("__name__", "wal_metric"), Samples: []cppbridge.Sample{ {Timestamp: 100, Value: 1}, {Timestamp: 101, Value: 2}, @@ -334,7 +334,7 @@ func (s *HeadLoadSuite) TestErrorDataUnloading() { sourceHead := s.mustCreateHead(unloadDataStorageInterval) series1 := []headtest.TimeSeries{ { - Labels: labels.Labels{{Name: "__name__", Value: "wal_metric"}}, + Labels: labels.FromStrings("__name__", "wal_metric"), Samples: []cppbridge.Sample{ {Timestamp: 0, Value: 1}, {Timestamp: 1, Value: 2}, @@ -345,7 +345,7 @@ func (s *HeadLoadSuite) TestErrorDataUnloading() { s.appendTimeSeries(sourceHead, series1) series2 := []headtest.TimeSeries{ { - Labels: labels.Labels{{Name: "__name__", Value: "wal_metric"}}, + Labels: labels.FromStrings("__name__", "wal_metric"), Samples: []cppbridge.Sample{ {Timestamp: 100, Value: 1}, {Timestamp: 101, Value: 2}, diff --git a/pp/go/relabeler/head/manager/manager.go b/pp/go/relabeler/head/manager/manager.go index 77c9d7e332..f3fdcc536a 100644 --- a/pp/go/relabeler/head/manager/manager.go +++ b/pp/go/relabeler/head/manager/manager.go @@ -14,6 +14,7 @@ import ( "github.com/prometheus/prometheus/pp/go/relabeler/config" "github.com/prometheus/prometheus/pp/go/relabeler/head" "github.com/prometheus/prometheus/pp/go/relabeler/head/catalog" + "github.com/prometheus/prometheus/pp/go/relabeler/headcontainer" "github.com/prometheus/prometheus/pp/go/relabeler/logger" "github.com/prometheus/prometheus/pp/go/util" ) @@ -300,9 +301,9 @@ func (m *Manager) BuildWithConfig( } // createDiscardableRotatableHead create discardable and rotatable head. -func (m *Manager) createDiscardableRotatableHead(h relabeler.Head, releaseHeadFn func()) *DiscardableRotatableHead { +func (m *Manager) createDiscardableRotatableHead(h relabeler.Head, releaseHeadFn func()) relabeler.Head { m.counter.With(prometheus.Labels{"type": "created"}).Inc() - return NewDiscardableRotatableHead( + return headcontainer.NewDiscardableRotatable( h, func(id string, err error) error { if _, rotateErr := m.catalog.SetStatus(id, catalog.StatusRotated); rotateErr != nil { diff --git a/pp/go/relabeler/head/shard.go b/pp/go/relabeler/head/shard.go index f04302323a..c195b636c2 100644 --- a/pp/go/relabeler/head/shard.go +++ b/pp/go/relabeler/head/shard.go @@ -133,8 +133,8 @@ func (ds *DataStorage) CreateRevertableLoader(lss *cppbridge.LabelSetStorage, ls return ds.dataStorage.CreateRevertableLoader(lss, lsIdBatchSize) } -func (ds *DataStorage) TimeInterval() cppbridge.TimeInterval { - return ds.dataStorage.TimeInterval() +func (ds *DataStorage) TimeInterval(invalidateCache bool) cppbridge.TimeInterval { + return ds.dataStorage.TimeInterval(invalidateCache) } func (ds *DataStorage) GetQueriedSeriesBitset() []byte { diff --git a/pp/go/relabeler/head/wal.go b/pp/go/relabeler/head/wal.go index b930cb1eff..2a4ca3f5c2 100644 --- a/pp/go/relabeler/head/wal.go +++ b/pp/go/relabeler/head/wal.go @@ -87,7 +87,7 @@ func (w *ShardWal) Write(innerSeriesSlice []*cppbridge.InnerSeries) (bool, error return false, fmt.Errorf("writing in corrupted wal") } - stats, err := w.encoder.Encode(innerSeriesSlice) + samples, err := w.encoder.Encode(innerSeriesSlice) if err != nil { return false, fmt.Errorf("failed to encode inner series: %w", err) } @@ -97,7 +97,7 @@ func (w *ShardWal) Write(innerSeriesSlice []*cppbridge.InnerSeries) (bool, error } // memoize reaching of limits to deduplicate triggers - if !w.limitExhausted && stats.Samples() >= w.maxSegmentSize { + if !w.limitExhausted && samples >= w.maxSegmentSize { w.limitExhausted = true return true, nil } @@ -205,8 +205,8 @@ func ReadHeader(reader io.Reader) (fileFormatVersion uint8, encoderVersion uint8 type EncodedSegment interface { Size() int64 CRC32() uint32 + Samples() uint32 io.WriterTo - cppbridge.SegmentStats } func WriteSegment(writer io.Writer, segment EncodedSegment) (n int, err error) { diff --git a/pp/go/relabeler/headcontainer/active.go b/pp/go/relabeler/headcontainer/active.go new file mode 100644 index 0000000000..4297c0d771 --- /dev/null +++ b/pp/go/relabeler/headcontainer/active.go @@ -0,0 +1,235 @@ +package headcontainer + +import ( + "context" + "errors" + "fmt" + "time" + + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/prometheus/pp/go/cppbridge" + "github.com/prometheus/prometheus/pp/go/relabeler" + "github.com/prometheus/prometheus/pp/go/relabeler/logger" + "github.com/prometheus/prometheus/pp/go/relabeler/querier" + "github.com/prometheus/prometheus/pp/go/util" + "github.com/prometheus/prometheus/pp/go/util/locker" + "github.com/prometheus/prometheus/storage" +) + +type Active struct { + wlocker *locker.Weighted + head relabeler.Head + + appendDuration prometheus.Histogram + waitLockRotateDuration prometheus.Gauge + rotationDuration prometheus.Gauge +} + +func NewActive( + head relabeler.Head, + registerer prometheus.Registerer, +) *Active { + factory := util.NewUnconflictRegisterer(registerer) + return &Active{ + wlocker: locker.NewWeighted(2 * head.Concurrency()), // x2 for back pressure + head: head, + + appendDuration: factory.NewHistogram( + prometheus.HistogramOpts{ + Name: "prompp_head_append_duration", + Help: "Append to head duration in microseconds", + Buckets: []float64{ + 50, 100, 250, 500, 750, + 1000, 2500, 5000, 7500, + 10000, 25000, 50000, 75000, + 100000, 500000, + }, + }, + ), + + waitLockRotateDuration: factory.NewGauge( + prometheus.GaugeOpts{ + Name: "prompp_head_wait_lock_rotate_duration", + Help: "The duration of the lock wait for rotation in nanoseconds", + }, + ), + rotationDuration: factory.NewGauge( + prometheus.GaugeOpts{ + Name: "prompp_head_rotate_duration", + Help: "The duration of the rotate in nanoseconds", + }, + ), + } +} + +func (h *Active) Append( + ctx context.Context, + incomingData *relabeler.IncomingData, + state *cppbridge.State, + relabelerID string, + commitToWal bool, +) (cppbridge.RelabelerStats, error) { + start := time.Now() + + runlock, err := h.wlocker.RLock(ctx) + if err != nil { + return cppbridge.RelabelerStats{}, fmt.Errorf("Append: weighted locker: %w", err) + } + defer runlock() + + defer func() { + h.appendDuration.Observe(float64(time.Since(start).Microseconds())) + }() + + _, stats, err := h.head.Append(ctx, incomingData, state, relabelerID, commitToWal) + if err != nil { + return cppbridge.RelabelerStats{}, err + } + + return stats, nil +} + +func (h *Active) ChunkQuerier(ctx context.Context, mint, maxt int64) (storage.ChunkQuerier, error) { + runlock, err := h.wlocker.RLock(ctx) + if err != nil { + return nil, fmt.Errorf("ChunkQuerier: weighted locker: %w", err) + } + head := h.head.Raw() + runlock() + return querier.NewChunkQuerier( + head, + querier.NoOpShardedDeduplicatorFactory(), + mint, + maxt, + nil, + ), nil +} + +func (h *Active) Close(ctx context.Context) error { + unlock, err := h.wlocker.LockWithPriority(ctx) + if err != nil { + return fmt.Errorf("Close: weighted locker: %w", err) + } + defer unlock() + + return errors.Join(h.head.CommitToWal(), h.head.Flush(), h.head.Close()) +} + +func (h *Active) CommitToWal(ctx context.Context) error { + runlock, err := h.wlocker.RLock(ctx) + if err != nil { + return fmt.Errorf("CommitToWal: weighted locker: %w", err) + } + defer runlock() + + return h.head.CommitToWal() +} + +func (h *Active) HeadStatus(ctx context.Context, limit int) relabeler.HeadStatus { + runlock, err := h.wlocker.RLock(ctx) + if err != nil { + logger.Warnf("[ActiveHead] HeadStatus: weighted locker: %s", err) + return relabeler.HeadStatus{} + } + defer runlock() + + return h.head.Status(limit) +} + +// MergeOutOfOrderChunks merge chunks with out of order data chunks. +func (h *Active) MergeOutOfOrderChunks(ctx context.Context) { + runlock, err := h.wlocker.RLock(ctx) + if err != nil { + logger.Warnf("[ActiveHead] MergeOutOfOrderChunks: weighted locker: %s", err) + return + } + defer runlock() + + h.head.MergeOutOfOrderChunks() +} + +func (h *Active) Querier( + ctx context.Context, + querierMetrics *querier.Metrics, + mint, maxt int64, +) (storage.Querier, error) { + runlock, err := h.wlocker.RLock(ctx) + if err != nil { + return nil, fmt.Errorf("Querier: weighted locker: %w", err) + } + head := h.head.Raw() + runlock() + + return querier.NewQuerier( + head, + querier.NoOpShardedDeduplicatorFactory(), + mint, + maxt, + nil, + querierMetrics, + ), nil +} + +func (h *Active) Reconfigure( + ctx context.Context, + headConfigurator relabeler.HeadConfigurator, +) error { + unlock, err := h.wlocker.LockWithPriority(ctx) + if err != nil { + return fmt.Errorf("Reconfigure: weighted locker: %w", err) + } + defer unlock() + + if err := headConfigurator.Configure(h.head); err != nil { + return fmt.Errorf("failed to reconfigure head: %w", err) + } + + h.wlocker.Resize(2 * h.head.Concurrency()) // x2 for back pressure + + return nil +} + +func (h *Active) Rotate(ctx context.Context) error { + start := time.Now() + + unlock, err := h.wlocker.LockWithPriority(ctx) + if err != nil { + return fmt.Errorf("Rotate: weighted locker: %w", err) + } + h.waitLockRotateDuration.Set(float64(time.Since(start).Nanoseconds())) + defer unlock() + + defer func() { + h.rotationDuration.Set(float64(time.Since(start).Nanoseconds())) + }() + + if err := h.head.Rotate(); err != nil { + return fmt.Errorf("failed to rotate head: %w", err) + } + + h.wlocker.Resize(2 * h.head.Concurrency()) // x2 for back pressure + + return nil +} + +func (h *Active) UnloadUnusedSeriesData(ctx context.Context) { + runlock, err := h.wlocker.RLock(ctx) + if err != nil { + logger.Warnf("[ActiveHead] UnloadUnusedSeriesData: weighted locker: %s", err) + return + } + defer runlock() + + h.head.UnloadUnusedSeriesData() +} + +func (h *Active) WriteMetrics(ctx context.Context) { + runlock, err := h.wlocker.RLock(ctx) + if err != nil { + logger.Warnf("[ActiveHead] writeMetrics: weighted locker: %s", err) + return + } + defer runlock() + + h.head.WriteMetrics(ctx) +} diff --git a/pp/go/relabeler/headcontainer/discardable_rotatable.go b/pp/go/relabeler/headcontainer/discardable_rotatable.go new file mode 100644 index 0000000000..ff1fa51550 --- /dev/null +++ b/pp/go/relabeler/headcontainer/discardable_rotatable.go @@ -0,0 +1,61 @@ +package headcontainer + +import ( + "errors" + + "github.com/prometheus/prometheus/pp/go/relabeler" +) + +// +// DiscardableRotatable +// + +type DiscardableRotatable struct { + onRotate func(id string, err error) error + onDiscard func(id string) error + afterClose func(id string) error + relabeler.Head +} + +func NewDiscardableRotatable( + head relabeler.Head, + onRotate func(id string, err error) error, + onDiscard func(id string) error, + afterClose func(id string) error, +) *DiscardableRotatable { + return &DiscardableRotatable{ + onRotate: onRotate, + onDiscard: onDiscard, + afterClose: afterClose, + Head: head, + } +} + +func (h *DiscardableRotatable) Rotate() error { + err := h.Head.Rotate() + if h.onRotate != nil { + err = errors.Join(err, h.onRotate(h.ID(), err)) + h.onRotate = nil + } + + return err +} + +func (h *DiscardableRotatable) Close() error { + err := h.Head.Close() + if h.afterClose != nil { + err = errors.Join(err, h.afterClose(h.ID())) + } + + return err +} + +func (h *DiscardableRotatable) Discard() (err error) { + err = h.Head.Discard() + if h.onDiscard != nil { + err = errors.Join(err, h.onDiscard(h.ID())) + h.onDiscard = nil + } + + return err +} diff --git a/pp/go/relabeler/headcontainer/heap_profile_writable.go b/pp/go/relabeler/headcontainer/heap_profile_writable.go new file mode 100644 index 0000000000..59a2e5fe7a --- /dev/null +++ b/pp/go/relabeler/headcontainer/heap_profile_writable.go @@ -0,0 +1,30 @@ +package headcontainer + +import ( + "github.com/prometheus/prometheus/pp/go/relabeler" +) + +// +// HeapProfileWritable +// + +type HeapProfileWriter interface { + WriteHeapProfile() error +} + +type HeapProfileWritable struct { + heapProfileWriter HeapProfileWriter + relabeler.Head +} + +func NewHeapProfileWritable(head relabeler.Head, heapProfileWriter HeapProfileWriter) *HeapProfileWritable { + return &HeapProfileWritable{Head: head, heapProfileWriter: heapProfileWriter} +} + +func (h *HeapProfileWritable) Rotate() error { + if err := h.Head.Rotate(); err != nil { + return err + } + + return h.heapProfileWriter.WriteHeapProfile() +} diff --git a/pp/go/relabeler/headcontainer/rotatable.go b/pp/go/relabeler/headcontainer/rotatable.go new file mode 100644 index 0000000000..68f0691dce --- /dev/null +++ b/pp/go/relabeler/headcontainer/rotatable.go @@ -0,0 +1,118 @@ +package headcontainer + +import ( + "context" + + "github.com/prometheus/prometheus/pp/go/relabeler" + "github.com/prometheus/prometheus/pp/go/relabeler/config" + "github.com/prometheus/prometheus/pp/go/relabeler/logger" +) + +// CopySeriesOnRotate copy active series from the current head to the new head during rotation. +var CopySeriesOnRotate = false + +// Storage - head storage. +type Storage interface { + Add(head relabeler.Head) +} + +// HeadBuilder - head builder. +type HeadBuilder interface { + Build() (relabeler.Head, error) + BuildWithConfig(inputRelabelerConfigs []*config.InputRelabelerConfig, numberOfShards uint16) (relabeler.Head, error) +} + +type HeadActivator interface { + Activate(headID string) error +} + +type NoOpHeadActivator struct{} + +func (NoOpHeadActivator) Activate(headID string) error { return nil } + +// Rotatable head wrapper, allows rotations. +type Rotatable struct { + storage Storage + builder HeadBuilder + headActivator HeadActivator + relabeler.Head +} + +// NewRotatable init new [*Rotatable] container head. +func NewRotatable( + head relabeler.Head, + storage Storage, + builder HeadBuilder, + headActivator HeadActivator, +) *Rotatable { + return &Rotatable{ + storage: storage, + builder: builder, + headActivator: headActivator, + Head: head, + } +} + +// Reconfigure relabeler.Head interface implementation. +func (h *Rotatable) Reconfigure( + ctx context.Context, + inputRelabelerConfigs []*config.InputRelabelerConfig, + numberOfShards uint16, +) error { + if h.Head.NumberOfShards() != numberOfShards { + return h.RotateWithConfig(inputRelabelerConfigs, numberOfShards) + } + return h.Head.Reconfigure(ctx, inputRelabelerConfigs, numberOfShards) +} + +// Rotate - relabeler.Head interface implementation. +func (h *Rotatable) Rotate() error { + h.Head.MergeOutOfOrderChunks() + + newHead, err := h.builder.Build() + if err != nil { + return err + } + + if CopySeriesOnRotate { + newHead.CopySeriesFrom(h.Head) + } + + if err = h.headActivator.Activate(newHead.ID()); err != nil { + return err + } + + if err = h.Head.CommitToWal(); err != nil { + logger.Errorf("failed to commit wal on rotation: %v", err) + } + h.Head.Stop() + + h.storage.Add(h.Head) + h.Head = newHead + return nil +} + +func (h *Rotatable) RotateWithConfig( + inputRelabelerConfigs []*config.InputRelabelerConfig, + numberOfShards uint16, +) error { + h.Head.MergeOutOfOrderChunks() + + newHead, err := h.builder.BuildWithConfig(inputRelabelerConfigs, numberOfShards) + if err != nil { + return err + } + + if err = h.headActivator.Activate(newHead.ID()); err != nil { + return err + } + + if err = h.Head.CommitToWal(); err != nil { + logger.Errorf("failed to commit wal on rotation: %v", err) + } + h.Head.Stop() + + h.storage.Add(h.Head) + h.Head = newHead + return nil +} diff --git a/pp/go/relabeler/interface.go b/pp/go/relabeler/interface.go index 945997b798..fc0b30b87f 100644 --- a/pp/go/relabeler/interface.go +++ b/pp/go/relabeler/interface.go @@ -23,7 +23,7 @@ type DataStorage interface { CreateUnusedSeriesDataUnloader() *cppbridge.UnusedSeriesDataUnloader CreateLoader(queriers []uintptr) *cppbridge.UnloadedDataLoader CreateRevertableLoader(lss *cppbridge.LabelSetStorage, lsIdBatchSize uint32) *cppbridge.UnloadedDataRevertableLoader - TimeInterval() cppbridge.TimeInterval + TimeInterval(invalidateCache bool) cppbridge.TimeInterval GetQueriedSeriesBitset() []byte } diff --git a/pp/go/relabeler/querier/chunk_querier_test.go b/pp/go/relabeler/querier/chunk_querier_test.go index 59f6fc992b..337959b56e 100644 --- a/pp/go/relabeler/querier/chunk_querier_test.go +++ b/pp/go/relabeler/querier/chunk_querier_test.go @@ -50,29 +50,20 @@ func (s *ChunkQuerierTestSuite) TestSelect() { // Arrange timeSeries := []headtest.TimeSeries{ { - Labels: labels.Labels{ - {Name: "__name__", Value: "metric"}, - {Name: "job", Value: "test"}, - }, + Labels: labels.FromStrings("__name__", "metric", "job", "test"), Samples: []cppbridge.Sample{ {Timestamp: 0, Value: 1}, {Timestamp: 1, Value: 1}, }, }, { - Labels: labels.Labels{ - {Name: "__name__", Value: "metric"}, - {Name: "job", Value: "test2"}, - }, + Labels: labels.FromStrings("__name__", "metric", "job", "test2"), Samples: []cppbridge.Sample{ {Timestamp: 0, Value: 10}, }, }, { - Labels: labels.Labels{ - {Name: "__name__", Value: "metric"}, - {Name: "job", Value: "test3"}, - }, + Labels: labels.FromStrings("__name__", "metric", "job", "test3"), Samples: []cppbridge.Sample{ {Timestamp: 10, Value: 10}, }, @@ -98,10 +89,7 @@ func (s *ChunkQuerierTestSuite) TestSelectWithDataStorageLoading() { // Arrange timeSeries := []headtest.TimeSeries{ { - Labels: labels.Labels{ - {Name: "__name__", Value: "metric"}, - {Name: "job", Value: "test"}, - }, + Labels: labels.FromStrings("__name__", "metric", "job", "test"), Samples: []cppbridge.Sample{ {Timestamp: 0, Value: 0}, {Timestamp: 1, Value: 1}, @@ -110,10 +98,7 @@ func (s *ChunkQuerierTestSuite) TestSelectWithDataStorageLoading() { }, }, { - Labels: labels.Labels{ - {Name: "__name__", Value: "metric"}, - {Name: "job", Value: "test2"}, - }, + Labels: labels.FromStrings("__name__", "metric", "job", "test2"), Samples: []cppbridge.Sample{ {Timestamp: 0, Value: 10}, {Timestamp: 1, Value: 11}, diff --git a/pp/go/relabeler/querier/metrics.go b/pp/go/relabeler/querier/metrics.go index 493ee255fa..530ee752ca 100644 --- a/pp/go/relabeler/querier/metrics.go +++ b/pp/go/relabeler/querier/metrics.go @@ -16,10 +16,6 @@ type Metrics struct { LabelNamesDuration prometheus.Histogram LabelValuesDuration prometheus.Histogram SelectDuration *prometheus.HistogramVec - AppendDuration prometheus.Histogram - - WaitLockRotateDuration prometheus.Gauge - RotationDuration prometheus.Gauge } func NewMetrics(registerer prometheus.Registerer, source string) *Metrics { @@ -65,30 +61,5 @@ func NewMetrics(registerer prometheus.Registerer, source string) *Metrics { }, []string{"query_type"}, ), - AppendDuration: factory.NewHistogram( - prometheus.HistogramOpts{ - Name: "prompp_head_append_duration", - Help: "Append to head duration in microseconds", - Buckets: []float64{ - 50, 100, 250, 500, 750, - 1000, 2500, 5000, 7500, - 10000, 25000, 50000, 75000, - 100000, 500000, - }, - }, - ), - - WaitLockRotateDuration: factory.NewGauge( - prometheus.GaugeOpts{ - Name: "prompp_head_wait_lock_rotate_duration", - Help: "The duration of the lock wait for rotation in nanoseconds", - }, - ), - RotationDuration: factory.NewGauge( - prometheus.GaugeOpts{ - Name: "prompp_head_rotate_duration", - Help: "The duration of the rotate in nanoseconds", - }, - ), } } diff --git a/pp/go/relabeler/querier/multi.go b/pp/go/relabeler/querier/multi.go index f2433490ba..169b6f1180 100644 --- a/pp/go/relabeler/querier/multi.go +++ b/pp/go/relabeler/querier/multi.go @@ -12,15 +12,23 @@ import ( ) type MultiQuerier struct { - mint int64 - maxt int64 queriers []storage.Querier closer func() error } func NewMultiQuerier(queriers []storage.Querier, closer func() error) *MultiQuerier { + qs := make([]storage.Querier, 0, len(queriers)) + for _, q := range queriers { + if rawQ, ok := q.(*MultiQuerier); ok { + qs = append(qs, rawQ.queriers...) + continue + } + + qs = append(qs, q) + } + return &MultiQuerier{ - queriers: queriers, + queriers: qs, closer: closer, } } @@ -31,6 +39,10 @@ func (q *MultiQuerier) LabelValues( hints *storage.LabelHints, matchers ...*labels.Matcher, ) ([]string, annotations.Annotations, error) { + if len(q.queriers) == 1 { + return q.queriers[0].LabelValues(ctx, name, hints, matchers...) + } + labelValuesResults := make([][]string, len(q.queriers)) annotationResults := make([]annotations.Annotations, len(q.queriers)) errs := make([]error, len(q.queriers)) @@ -60,6 +72,10 @@ func (q *MultiQuerier) LabelNames( hints *storage.LabelHints, matchers ...*labels.Matcher, ) ([]string, annotations.Annotations, error) { + if len(q.queriers) == 1 { + return q.queriers[0].LabelNames(ctx, hints, matchers...) + } + labelNamesResults := make([][]string, len(q.queriers)) annotationResults := make([]annotations.Annotations, len(q.queriers)) errs := make([]error, len(q.queriers)) @@ -101,6 +117,10 @@ func (q *MultiQuerier) Select( hints *storage.SelectHints, matchers ...*labels.Matcher, ) storage.SeriesSet { + if len(q.queriers) == 1 { + return q.queriers[0].Select(ctx, sortSeries, hints, matchers...) + } + seriesSets := make([]storage.SeriesSet, len(q.queriers)) wg := &sync.WaitGroup{} diff --git a/pp/go/relabeler/querier/querier_test.go b/pp/go/relabeler/querier/querier_test.go index 259fe913b1..282003da7a 100644 --- a/pp/go/relabeler/querier/querier_test.go +++ b/pp/go/relabeler/querier/querier_test.go @@ -66,19 +66,13 @@ func (s *QuerierTestSuite) TestRangeQuery() { // Arrange timeSeries := []headtest.TimeSeries{ { - Labels: labels.Labels{ - {Name: "__name__", Value: "metric"}, - {Name: "job", Value: "test"}, - }, + Labels: labels.FromStrings("__name__", "metric", "job", "test"), Samples: []cppbridge.Sample{ {Timestamp: 0, Value: 1}, }, }, { - Labels: labels.Labels{ - {Name: "__name__", Value: "metric"}, - {Name: "job", Value: "test2"}, - }, + Labels: labels.FromStrings("__name__", "metric", "job", "test2"), Samples: []cppbridge.Sample{ {Timestamp: 0, Value: 10}, }, @@ -101,10 +95,7 @@ func (s *QuerierTestSuite) TestRangeQueryWithDataStorageLoading() { // Arrange timeSeries := []headtest.TimeSeries{ { - Labels: labels.Labels{ - {Name: "__name__", Value: "metric"}, - {Name: "job", Value: "test"}, - }, + Labels: labels.FromStrings("__name__", "metric", "job", "test"), Samples: []cppbridge.Sample{ {Timestamp: 0, Value: 0}, {Timestamp: 1, Value: 1}, @@ -112,10 +103,7 @@ func (s *QuerierTestSuite) TestRangeQueryWithDataStorageLoading() { }, }, { - Labels: labels.Labels{ - {Name: "__name__", Value: "metric"}, - {Name: "job", Value: "test2"}, - }, + Labels: labels.FromStrings("__name__", "metric", "job", "test2"), Samples: []cppbridge.Sample{ {Timestamp: 0, Value: 10}, {Timestamp: 1, Value: 11}, @@ -159,19 +147,13 @@ func (s *QuerierTestSuite) TestInstantQuery() { // Arrange timeSeries := []headtest.TimeSeries{ { - Labels: labels.Labels{ - {Name: "__name__", Value: "metric"}, - {Name: "job", Value: "test"}, - }, + Labels: labels.FromStrings("__name__", "metric", "job", "test"), Samples: []cppbridge.Sample{ {Timestamp: 0, Value: 1}, }, }, { - Labels: labels.Labels{ - {Name: "__name__", Value: "metric"}, - {Name: "job", Value: "test2"}, - }, + Labels: labels.FromStrings("__name__", "metric", "job", "test2"), Samples: []cppbridge.Sample{ {Timestamp: 0, Value: 10}, }, @@ -194,10 +176,7 @@ func (s *QuerierTestSuite) TestInstantQueryWithDataStorageLoading() { // Arrange timeSeries := []headtest.TimeSeries{ { - Labels: labels.Labels{ - {Name: "__name__", Value: "metric"}, - {Name: "job", Value: "test"}, - }, + Labels: labels.FromStrings("__name__", "metric", "job", "test"), Samples: []cppbridge.Sample{ {Timestamp: 0, Value: 0}, {Timestamp: 1, Value: 1}, @@ -205,10 +184,7 @@ func (s *QuerierTestSuite) TestInstantQueryWithDataStorageLoading() { }, }, { - Labels: labels.Labels{ - {Name: "__name__", Value: "metric"}, - {Name: "job", Value: "test2"}, - }, + Labels: labels.FromStrings("__name__", "metric", "job", "test2"), Samples: []cppbridge.Sample{ {Timestamp: 0, Value: 10}, {Timestamp: 1, Value: 11}, diff --git a/pp/go/storage/appender/appender.go b/pp/go/storage/appender/appender.go new file mode 100644 index 0000000000..375b9c1976 --- /dev/null +++ b/pp/go/storage/appender/appender.go @@ -0,0 +1,378 @@ +package appender + +import ( + "context" + "errors" + "fmt" + "sync/atomic" + + "github.com/prometheus/prometheus/pp/go/cppbridge" + "github.com/prometheus/prometheus/pp/go/logger" + "github.com/prometheus/prometheus/pp/go/storage/head/task" +) + +const ( + // dsAppendInnerSeries name of task. + dsAppendInnerSeries = "data_storage_append_inner_series" + + // lssInputRelabeling name of task. + lssInputRelabeling = "lss_input_relabeling" + // lssAppendRelabelerSeries name of task. + lssAppendRelabelerSeries = "lss_append_relabeler_series" + + // walWrite name of task. + walWrite = "wal_write" +) + +// errNilState error when incoming state is nil. +var errNilState = errors.New("state is nil") + +// +// Task +// + +// Task the minimum required task [Generic] implementation. +type Task interface { + // Wait for the task to complete on all shards. + Wait() error +} + +// +// Shard +// + +// Shard the minimum required head [Shard] implementation. +type Shard interface { + // AppendInnerSeriesSlice add InnerSeries to [DataStorage]. + AppendInnerSeriesSlice(innerSeriesSlice []*cppbridge.InnerSeries) + + // LSSWithLock calls fn on raws [cppbridge.LabelSetStorage] with write lock. + LSSWithLock(fn func(target, input *cppbridge.LabelSetStorage) error) error + + // LSSWithRLock calls fn on raws [cppbridge.LabelSetStorage] with read lock. + LSSWithRLock(fn func(target, input *cppbridge.LabelSetStorage) error) error + + // LSSResetSnapshot resets the current snapshot. Use only WithLock. + LSSResetSnapshot() + + // Relabeler returns relabeler for shard goroutines. + Relabeler() *cppbridge.PerGoroutineRelabeler + + // ShardID returns the shard ID. + ShardID() uint16 + + // WalWrite append the incoming inner series to wal encoder. + WalWrite(innerSeriesSlice []*cppbridge.InnerSeries) (bool, error) +} + +// +// Head +// + +// Head the minimum required [Head] implementation. +type Head[ + TTask Task, + TShard Shard, +] interface { + // CreateTask create a task for operations on the [Head] shards. + CreateTask(taskName string, shardFn func(shard TShard) error) TTask + + // Enqueue the task to be executed on shards [Head]. + Enqueue(t TTask) + + // Generation returns current generation of [Head]. + Generation() uint64 + + // NumberOfShards returns current number of shards in to [Head]. + NumberOfShards() uint16 +} + +// +// Appender +// + +// Appender adds incoming data to the [Head]. +type Appender[ + TTask Task, + TShard Shard, + THead Head[TTask, TShard], +] struct { + head THead + commitAndFlush func(h THead) error +} + +// New init new [Appender]. +func New[ + TTask Task, + TShard Shard, + THead Head[TTask, TShard], +]( + head THead, + commitAndFlush func(h THead) error, +) Appender[TTask, TShard, THead] { + return Appender[TTask, TShard, THead]{ + head: head, + commitAndFlush: commitAndFlush, + } +} + +// Append incoming data to [Head]. +// +//revive:disable-next-line:flag-parameter this is a flag, but it's more convenient this way +func (a Appender[TTask, TShard, THead]) Append( + ctx context.Context, + incomingData *IncomingData, + state *cppbridge.StateV2, + commitToWal bool, +) ([][]*cppbridge.InnerSeries, cppbridge.RelabelerStats, error) { + if err := a.resolveState(state); err != nil { + return nil, cppbridge.RelabelerStats{}, err + } + + numberOfShards := a.head.NumberOfShards() + shardedInnerSeries := NewShardedInnerSeries(numberOfShards) + shardedRelabeledSeries := NewShardedRelabeledSeries(numberOfShards) + stats, err := a.inputRelabelingStage( + ctx, + state, + NewDestructibleIncomingData(incomingData, int(numberOfShards)), + shardedInnerSeries, + shardedRelabeledSeries, + ) + if err != nil { + return nil, stats, fmt.Errorf("failed input relabeling stage: %w", err) + } + + if !shardedRelabeledSeries.IsEmpty() { + shardedStateUpdates := NewShardedStateUpdates(numberOfShards) + if err = a.appendRelabelerSeriesStage( + ctx, + shardedInnerSeries, + shardedRelabeledSeries, + shardedStateUpdates, + ); err != nil { + return nil, stats, fmt.Errorf("failed append relabeler series stage: %w", err) + } + + if err = a.updateRelabelerStateStage( + ctx, + state, + shardedStateUpdates, + ); err != nil { + return nil, stats, fmt.Errorf("failed update relabeler stage: %w", err) + } + } + + atomicLimitExhausted, err := a.appendInnerSeriesAndWriteToWal(shardedInnerSeries) + if err != nil { + logger.Errorf("failed to write wal: %v", err) + } + + if commitToWal || atomicLimitExhausted > 0 { + if err := a.commitAndFlush(a.head); err != nil { + logger.Errorf("failed to commit wal: %v", err) + } + } + + return shardedInnerSeries.Data(), stats, nil +} + +// inputRelabelingStage first stage - relabeling. +// +//revive:disable-next-line:function-length long but this is first stage. +func (a *Appender[TTask, TShard, THead]) inputRelabelingStage( + ctx context.Context, + state *cppbridge.StateV2, + incomingData *DestructibleIncomingData, + shardedInnerSeries *ShardedInnerSeries, + shardedRelabeledSeries *ShardedRelabeledSeries, +) (cppbridge.RelabelerStats, error) { + stats := make([]cppbridge.RelabelerStats, a.head.NumberOfShards()) + t := a.head.CreateTask( + lssInputRelabeling, + func(shard TShard) error { + var ( + relabeler = shard.Relabeler() + shardID = shard.ShardID() + ok bool + shardedData = incomingData.ShardedData() + innerSeries = shardedInnerSeries.DataBySourceShard(shardID) + ) + + if err := shard.LSSWithRLock(func(target, input *cppbridge.LabelSetStorage) (rErr error) { + stats[shardID], ok, rErr = relabeler.RelabelingFromCache( + ctx, + input, + target, + state, + shardedData, + innerSeries, + ) + + return rErr + }); err != nil { + incomingData.Destroy() + return fmt.Errorf("shard %d: %w", shardID, err) + } + + if ok { + incomingData.Destroy() + return nil + } + + var ( + hasReallocations bool + rstats = cppbridge.RelabelerStats{} + ) + err := shard.LSSWithLock(func(target, input *cppbridge.LabelSetStorage) (rErr error) { + rstats, hasReallocations, rErr = relabeler.Relabeling( + ctx, + input, + target, + state, + shardedData, + innerSeries, + shardedRelabeledSeries.DataByShard(shardID), + ) + + if hasReallocations { + shard.LSSResetSnapshot() + } + + return rErr + }) + + incomingData.Destroy() + if err != nil { + return fmt.Errorf("shard %d: %w", shardID, err) + } + + stats[shardID].Add(rstats) + + return nil + }, + ) + a.head.Enqueue(t) + + resStats := cppbridge.RelabelerStats{} + if err := t.Wait(); err != nil { + return resStats, err + } + + resStats.Add(stats...) + + return resStats, nil +} + +// appendRelabelerSeriesStage second stage - append to lss relabeling ls. +func (a *Appender[TTask, TShard, THead]) appendRelabelerSeriesStage( + ctx context.Context, + shardedInnerSeries *ShardedInnerSeries, + shardedRelabeledSeries *ShardedRelabeledSeries, + shardedStateUpdates *ShardedStateUpdates, +) error { + t := a.head.CreateTask( + lssAppendRelabelerSeries, + func(shard TShard) error { + shardID := shard.ShardID() + + relabeledSeries, ok := shardedRelabeledSeries.DataBySourceShard(shardID) + if !ok { + return nil + } + + return shard.LSSWithLock(func(target, _ *cppbridge.LabelSetStorage) error { + hasReallocations, err := shard.Relabeler().AppendRelabelerSeries( + ctx, + target, + shardedInnerSeries.DataByShard(shardID), + relabeledSeries, + shardedStateUpdates.DataByShard(shardID), + ) + if err != nil { + return fmt.Errorf("shard %d: %w", shardID, err) + } + + if hasReallocations { + shard.LSSResetSnapshot() + } + + return nil + }) + }, + ) + a.head.Enqueue(t) + + return t.Wait() +} + +// updateRelabelerStateStage third stage - update state cache. +func (a *Appender[TTask, TShard, THead]) updateRelabelerStateStage( + ctx context.Context, + state *cppbridge.StateV2, + shardedStateUpdates *ShardedStateUpdates, +) error { + numberOfShards := a.head.NumberOfShards() + for shardID := range numberOfShards { + updates, ok := shardedStateUpdates.DataBySourceShard(shardID) + if !ok { + continue + } + + if err := state.CacheByShard(shardID).Update(ctx, updates); err != nil { + return fmt.Errorf("shard %d: %w", shardID, err) + } + } + + return nil +} + +// appendInnerSeriesAndWriteToWal append [cppbridge.InnerSeries] to [Shard]'s to [DataStorage] and write to [Wal]. +func (a *Appender[TTask, TShard, THead]) appendInnerSeriesAndWriteToWal( + shardedInnerSeries *ShardedInnerSeries, +) (uint32, error) { + tw := task.NewTaskWaiter[TTask](2) //revive:disable-line:add-constant // 2 task for wait + + tAppend := a.head.CreateTask( + dsAppendInnerSeries, + func(shard TShard) error { + shard.AppendInnerSeriesSlice(shardedInnerSeries.DataByShard(shard.ShardID())) + + return nil + }, + ) + a.head.Enqueue(tAppend) + + var atomicLimitExhausted uint32 + tWalWrite := a.head.CreateTask( + walWrite, + func(shard TShard) error { + limitExhausted, errWrite := shard.WalWrite(shardedInnerSeries.DataByShard(shard.ShardID())) + if errWrite != nil { + return fmt.Errorf("shard %d: %w", shard.ShardID(), errWrite) + } + + if limitExhausted { + atomic.AddUint32(&atomicLimitExhausted, 1) + } + + return nil + }, + ) + a.head.Enqueue(tWalWrite) + + tw.Add(tAppend) + tw.Add(tWalWrite) + + return atomicLimitExhausted, tw.Wait() +} + +func (a *Appender[TTask, TShard, THead]) resolveState(state *cppbridge.StateV2) error { + if state == nil { + return errNilState + } + + state.Reconfigure(a.head.Generation(), a.head.NumberOfShards()) + + return nil +} diff --git a/pp/go/storage/appender/appender_test.go b/pp/go/storage/appender/appender_test.go new file mode 100644 index 0000000000..15d69f1287 --- /dev/null +++ b/pp/go/storage/appender/appender_test.go @@ -0,0 +1,68 @@ +package appender_test + +import ( + "testing" + + "github.com/prometheus/prometheus/pp/go/cppbridge" + "github.com/prometheus/prometheus/pp/go/storage/appender" + "github.com/prometheus/prometheus/pp/go/storage/head/head" + "github.com/prometheus/prometheus/pp/go/storage/head/services" + "github.com/prometheus/prometheus/pp/go/storage/head/shard" +) + +func TestXxx(t *testing.T) { + lss := &shard.LSS{} + ds := shard.NewDataStorage() + wl := &testWal{} + sd := shard.NewShard(lss, ds, nil, nil, wl, 0) + id := "test-head-id" + generation := uint64(0) + + h := head.NewHead( + id, + []*shard.Shard{sd}, + shard.NewPerGoroutineShard[*testWal], + nil, + generation, + nil, + ) + + app := appender.New(h, services.CFViaRange) + + _ = app + + t.Log("end") +} + +// testWal test implementation wal. +type testWal struct{} + +// Close test implementation wal. +func (*testWal) Close() error { + return nil +} + +// Commit test implementation wal. +func (*testWal) Commit() error { + return nil +} + +// CurrentSize test implementation wal. +func (*testWal) CurrentSize() int64 { + return 0 +} + +// Flush test implementation wal. +func (*testWal) Flush() error { + return nil +} + +// Sync test implementation wal. +func (*testWal) Sync() error { + return nil +} + +// Write test implementation wal. +func (*testWal) Write(_ []*cppbridge.InnerSeries) (bool, error) { + return false, nil +} diff --git a/pp/go/storage/appender/models.go b/pp/go/storage/appender/models.go new file mode 100644 index 0000000000..dac36dbbad --- /dev/null +++ b/pp/go/storage/appender/models.go @@ -0,0 +1,219 @@ +package appender + +import ( + "sync/atomic" + + "github.com/prometheus/prometheus/pp/go/cppbridge" +) + +// +// ShardedInnerSeries +// + +// ShardedInnerSeries conteiner for InnerSeries for each shard. +type ShardedInnerSeries struct { + // id slice - shard id, data[shard_id] - amount of data = x2 numberOfShards + data [][]*cppbridge.InnerSeries +} + +// NewShardedInnerSeries init new ShardedInnerSeries. +func NewShardedInnerSeries(numberOfShards uint16) *ShardedInnerSeries { + // id slice - shard id + data := make([][]*cppbridge.InnerSeries, numberOfShards) + for i := range data { + // amount of data = x2 numberOfShards + data[i] = cppbridge.NewShardsInnerSeries(numberOfShards) + } + + return &ShardedInnerSeries{ + data: data, + } +} + +// Data return slice of elemets for each shard. +func (sis *ShardedInnerSeries) Data() [][]*cppbridge.InnerSeries { + return sis.data +} + +// DataByShard return slice with the results per shard. +func (sis *ShardedInnerSeries) DataByShard(shardID uint16) []*cppbridge.InnerSeries { + return sis.data[shardID] +} + +// DataBySourceShard return slice with the results per source shard. +func (sis *ShardedInnerSeries) DataBySourceShard(sourceShardID uint16) []*cppbridge.InnerSeries { + data := make([]*cppbridge.InnerSeries, len(sis.data)) + for i, iss := range sis.data { + data[i] = iss[sourceShardID] + } + + return data +} + +// +// ShardedRelabeledSeries +// + +// ShardedRelabeledSeries conteiner for RelabeledSeries for each shard. +type ShardedRelabeledSeries struct { + // id slice - shard id, data[shard_id] id slice - source shard id + // data[shard_id][source_shard_id] - amount of data = numberOfShards + data [][]*cppbridge.RelabeledSeries +} + +// NewShardedRelabeledSeries init new ShardedRelabeledSeries. +func NewShardedRelabeledSeries(numberOfShards uint16) *ShardedRelabeledSeries { + // id slice - shard id + data := make([][]*cppbridge.RelabeledSeries, numberOfShards) + for i := range data { + // data[shard_id] id slice - source shard id + // data[shard_id][source_shard_id] - amount of data = numberOfShards + data[i] = cppbridge.NewShardsRelabeledSeries(numberOfShards) + } + return &ShardedRelabeledSeries{ + data: data, + } +} + +// DataByShard return slice with the results per shard. +func (srs *ShardedRelabeledSeries) DataByShard(shardID uint16) []*cppbridge.RelabeledSeries { + return srs.data[shardID] +} + +// DataBySourceShard return slice with the results per source shard. +func (srs *ShardedRelabeledSeries) DataBySourceShard(sourceShardID uint16) ([]*cppbridge.RelabeledSeries, bool) { + ok := false + data := make([]*cppbridge.RelabeledSeries, len(srs.data)) + for i, rss := range srs.data { + data[i] = rss[sourceShardID] + if data[i].Size() != 0 { + ok = true + } + } + + return data, ok +} + +// IsEmpty return false if there are no elements. +func (srs *ShardedRelabeledSeries) IsEmpty() bool { + for _, rss := range srs.data { + for _, rs := range rss { + if rs.Size() != 0 { + return false + } + } + } + + return true +} + +// +// ShardedStateUpdates +// + +// ShardedStateUpdates conteiner for RelabelerStateUpdate for each shard. +type ShardedStateUpdates struct { + // id slice - shard id, data[shard_id] id slice - source shard id + // data[shard_id][source_shard_id] - amount of data = numberOfShards + data [][]*cppbridge.RelabelerStateUpdate +} + +// NewShardedStateUpdates init new ShardedStateUpdates. +func NewShardedStateUpdates(numberOfShards uint16) *ShardedStateUpdates { + // id slice - shard id + data := make([][]*cppbridge.RelabelerStateUpdate, numberOfShards) + for i := range data { + // data[shard_id] id slice - source shard id + // data[shard_id][source_shard_id] - amount of data = numberOfShards + data[i] = cppbridge.NewShardsRelabelerStateUpdate(numberOfShards) + } + return &ShardedStateUpdates{ + data: data, + } +} + +// DataByShard return slice with the results per shard. +func (sru *ShardedStateUpdates) DataByShard(shardID uint16) []*cppbridge.RelabelerStateUpdate { + return sru.data[shardID] +} + +// DataBySourceShard return slice with the results per source shard. +func (sru *ShardedStateUpdates) DataBySourceShard(sourceShardID uint16) ([]*cppbridge.RelabelerStateUpdate, bool) { + ok := false + data := make([]*cppbridge.RelabelerStateUpdate, len(sru.data)) + for i, rsu := range sru.data { + data[i] = rsu[sourceShardID] + if !data[i].IsEmpty() { + ok = true + } + } + + return data, ok +} + +// +// MetricData +// + +// MetricData is an universal interface for blob protobuf data or batch [model.TimeSeries]. +type MetricData interface { + // Destroy incoming data. + Destroy() +} + +// +// IncomingData +// + +// IncomingData incoming [cppbridge.ShardedData] for shard distribution. +type IncomingData struct { + Hashdex cppbridge.ShardedData + Data MetricData +} + +// Destroy IncomingData. +func (i *IncomingData) Destroy() { + i.Hashdex = nil + if i.Data != nil { + i.Data.Destroy() + } +} + +// ShardedData return hashdex. +func (i *IncomingData) ShardedData() cppbridge.ShardedData { + return i.Hashdex +} + +// +// DestructibleIncomingData +// + +// DestructibleIncomingData wrapeer over [IncomingData] with detroy-counter. +type DestructibleIncomingData struct { + data *IncomingData + destructCount atomic.Int64 +} + +// NewDestructibleIncomingData init new [DestructibleIncomingData]. +func NewDestructibleIncomingData(data *IncomingData, destructCount int) *DestructibleIncomingData { + d := &DestructibleIncomingData{ + data: data, + } + d.destructCount.Store(int64(destructCount)) + + return d +} + +// ShardedData return hashdex. +func (d *DestructibleIncomingData) ShardedData() cppbridge.ShardedData { + return d.data.ShardedData() +} + +// Destroy decrement count or destroy IncomingData. +func (d *DestructibleIncomingData) Destroy() { + if d.destructCount.Add(-1) != 0 { + return + } + + d.data.Destroy() +} diff --git a/pp/go/storage/block/block.go b/pp/go/storage/block/block.go new file mode 100644 index 0000000000..3b1bb834cf --- /dev/null +++ b/pp/go/storage/block/block.go @@ -0,0 +1,177 @@ +package block + +import ( + "fmt" + "io" + "math" + "unsafe" + + "github.com/prometheus/prometheus/pp/go/cppbridge" + "github.com/prometheus/prometheus/tsdb/chunkenc" +) + +// Chunk represents a recoded chunk. +type Chunk struct { + rc *cppbridge.RecodedChunk +} + +// MinT returns the minimum timestamp of the chunk. +func (c *Chunk) MinT() int64 { + return c.rc.MinT +} + +// MaxT returns the maximum timestamp of the chunk. +func (c *Chunk) MaxT() int64 { + return c.rc.MaxT +} + +// SeriesID returns the series ID of the chunk. +func (c *Chunk) SeriesID() uint32 { + return c.rc.SeriesId +} + +// Encoding returns is the identifier of the encoding of the chunk. +func (*Chunk) Encoding() chunkenc.Encoding { + return chunkenc.EncXOR +} + +// SampleCount returns the number of samples in the chunk. +func (c *Chunk) SampleCount() uint8 { + return c.rc.SamplesCount +} + +// Bytes returns the bytes blob of the chunk data. +func (c *Chunk) Bytes() []byte { + return c.rc.ChunkData +} + +// ChunkIterator represents a chunk iterator, it is used to iterate over the chunks. +type ChunkIterator struct { + r *cppbridge.ChunkRecoder + rc *cppbridge.RecodedChunk +} + +// NewChunkIterator init new [ChunkIterator]. +func NewChunkIterator( + lss *cppbridge.LabelSetStorage, + lsIdBatchSize uint32, + ds *cppbridge.HeadDataStorage, + minT, maxT int64, +) ChunkIterator { + return ChunkIterator{ + r: cppbridge.NewChunkRecoder(lss, lsIdBatchSize, ds, cppbridge.TimeInterval{MinT: minT, MaxT: maxT}), + } +} + +// Next advances the iterator by one, if possible. +func (i *ChunkIterator) Next() bool { + if i.rc != nil && !i.rc.HasMoreData { + return false + } + + rc := i.r.RecodeNextChunk() + i.rc = &rc + return rc.SeriesId != math.MaxUint32 +} + +// NextBatch advances the iterator by one batch, if if there is more data. +func (i *ChunkIterator) NextBatch() bool { + i.rc.HasMoreData = i.r.NextBatch() + return i.rc.HasMoreData +} + +// At returns the current chunk. +func (i *ChunkIterator) At() Chunk { + return Chunk{rc: i.rc} +} + +// +// IndexWriter +// + +// IndexWriter represents a index writer, it is used to write the index. +type IndexWriter struct { + cppIndexWriter *cppbridge.IndexWriter + isPrefixWritten bool +} + +// NewIndexWriter init new [IndexWriter]. +func NewIndexWriter(lss *cppbridge.LabelSetStorage) IndexWriter { + return IndexWriter{cppIndexWriter: cppbridge.NewIndexWriter(lss)} +} + +// WriteRestTo writes the rest of the index to the writer. +func (iw *IndexWriter) WriteRestTo(w io.Writer) (n int64, err error) { + bytesWritten, err := w.Write(iw.cppIndexWriter.WriteLabelIndices()) + n += int64(bytesWritten) + if err != nil { + return n, fmt.Errorf("failed to write label indicies: %w", err) + } + + for { + data, hasMoreData := iw.cppIndexWriter.WriteNextPostingsBatch(1 << 20) + bytesWritten, err = w.Write(data) + if err != nil { + return n, fmt.Errorf("failed to write postings: %w", err) + } + n += int64(bytesWritten) + if !hasMoreData { + break + } + } + + bytesWritten, err = w.Write(iw.cppIndexWriter.WriteLabelIndicesTable()) + if err != nil { + return n, fmt.Errorf("failed to write label indicies table: %w", err) + } + n += int64(bytesWritten) + + bytesWritten, err = w.Write(iw.cppIndexWriter.WritePostingsTableOffsets()) + if err != nil { + return n, fmt.Errorf("failed to write posting table offsets: %w", err) + } + n += int64(bytesWritten) + + bytesWritten, err = w.Write(iw.cppIndexWriter.WriteTableOfContents()) + if err != nil { + return n, fmt.Errorf("failed to write table of content: %w", err) + } + n += int64(bytesWritten) + + return n, nil +} + +// WriteSeriesTo writes series(id and chunks) to [io.Writer]. +func (iw *IndexWriter) WriteSeriesTo(id uint32, chunks []ChunkMetadata, w io.Writer) (n int64, err error) { + if !iw.isPrefixWritten { + var bytesWritten int + bytesWritten, err = w.Write(iw.cppIndexWriter.WriteHeader()) + n += int64(bytesWritten) + if err != nil { + return n, fmt.Errorf("failed to write header: %w", err) + } + + bytesWritten, err = w.Write(iw.cppIndexWriter.WriteSymbols()) + n += int64(bytesWritten) + if err != nil { + return n, fmt.Errorf("failed to write symbols: %w", err) + } + iw.isPrefixWritten = true + } + + bytesWritten, err := w.Write(iw.cppIndexWriter.WriteSeries( + id, + *(*[]cppbridge.ChunkMetadata)(unsafe.Pointer(&chunks)), // #nosec G103 // it's meant to be that way + )) + n += int64(bytesWritten) + if err != nil { + return n, fmt.Errorf("failed to write series: %w", err) + } + + return n, nil +} + +// isEmpty returns true if [IndexWriter] contains no samples, an empty block. +func (iw *IndexWriter) isEmpty() bool { + return !iw.isPrefixWritten +} diff --git a/pp/go/storage/block/block_writer.go b/pp/go/storage/block/block_writer.go new file mode 100644 index 0000000000..da8e90f81e --- /dev/null +++ b/pp/go/storage/block/block_writer.go @@ -0,0 +1,357 @@ +package block + +import ( + "crypto/rand" + "encoding/json" + "fmt" + "math" + "os" + "path/filepath" + + "github.com/oklog/ulid" + + "github.com/prometheus/prometheus/tsdb" + "github.com/prometheus/prometheus/tsdb/fileutil" + + "github.com/prometheus/prometheus/pp/go/logger" + "github.com/prometheus/prometheus/pp/go/util" +) + +const ( + tmpForCreationBlockDirSuffix = ".tmp-for-creation" + + indexFilename = "index" + metaFilename = "meta.json" + + metaVersion1 = 1 +) + +// WrittenBlock represents a written block. +type WrittenBlock struct { + Dir string + Meta tsdb.BlockMeta +} + +// ChunkDir returns the chunk directory. +func (block *WrittenBlock) ChunkDir() string { + return filepath.Join(block.Dir, "chunks") +} + +// IndexFilename returns the index filename. +func (block *WrittenBlock) IndexFilename() string { + return filepath.Join(block.Dir, indexFilename) +} + +// MetaFilename returns the meta filename. +func (block *WrittenBlock) MetaFilename() string { + return filepath.Join(block.Dir, metaFilename) +} + +type blockWriter struct { + WrittenBlock + + chunkWriter *ChunkWriter + indexFileWriter *FileWriter + indexWriter IndexWriter + + chunkRecoder chunkRecoder +} + +func newBlockWriter( + dir string, + maxBlockChunkSegmentSize int64, + indexWriter IndexWriter, + chunkIterator ChunkIterator, +) (writer blockWriter, err error) { + uid := ulid.MustNew(ulid.Now(), rand.Reader) + writer.Dir = filepath.Join(dir, uid.String()) + tmpForCreationBlockDirSuffix + + if err = createTmpDir(writer.Dir); err != nil { + return writer, err + } + + if err = writer.createWriters(maxBlockChunkSegmentSize); err != nil { + return writer, err + } + + writer.Meta = tsdb.BlockMeta{ + ULID: uid, + MinTime: math.MaxInt64, + MaxTime: math.MinInt64, + Version: metaVersion1, + Compaction: tsdb.BlockMetaCompaction{ + Level: 1, + Sources: []ulid.ULID{uid}, + }, + } + + writer.indexWriter = indexWriter + writer.chunkRecoder = newChunkRecoder(chunkIterator) + + return writer, err +} + +// isEmpty returns true if [IndexWriter] contains no samples, an empty block. +func (writer *blockWriter) isEmpty() bool { + return writer.indexWriter.isEmpty() +} + +func (writer *blockWriter) createWriters(maxBlockChunkSegmentSize int64) error { + chunkWriter, err := NewChunkWriter(writer.ChunkDir(), maxBlockChunkSegmentSize) + if err != nil { + return fmt.Errorf("failed to create chunk writer: %w", err) + } + + indexFileWriter, err := NewFileWriter(writer.IndexFilename()) + if err != nil { + _ = chunkWriter.Close() + return fmt.Errorf("failed to create index file writer: %w", err) + } + + writer.chunkWriter = chunkWriter + writer.indexFileWriter = indexFileWriter + return nil +} + +func (writer *blockWriter) close() error { + return util.CloseAll(writer.chunkWriter, writer.indexFileWriter) +} + +func (writer *blockWriter) recodeAndWriteChunksBatch() error { + return writer.chunkRecoder.recode(writer.chunkWriter, &writer.Meta, writer.writeSeries) +} + +func (writer *blockWriter) writeRestOfRecodedChunks() error { + return writer.writeSeries(writer.chunkRecoder.previousSeriesID, writer.chunkRecoder.chunksMetadata) +} + +func (writer *blockWriter) writeSeries(seriesID uint32, chunksMetadata []ChunkMetadata) error { + if len(chunksMetadata) > 0 { + if _, err := writer.indexWriter.WriteSeriesTo(seriesID, chunksMetadata, writer.indexFileWriter); err != nil { + return fmt.Errorf("failed to write series %d: %w", seriesID, err) + } + } + + return nil +} + +func (writer *blockWriter) writeIndex() error { + if _, err := writer.indexWriter.WriteRestTo(writer.indexFileWriter); err != nil { + return fmt.Errorf("failed to write index: %w", err) + } + + writer.Meta.MaxTime++ + if _, err := writeBlockMetaFile(writer.MetaFilename(), &writer.Meta); err != nil { + return fmt.Errorf("failed to write block meta file: %w", err) + } + + return nil +} + +func (writer *blockWriter) moveTmpDirToDir() error { + if err := syncDir(writer.Dir); err != nil { + return fmt.Errorf("failed to sync temporary block dir: %w", err) + } + + dir := writer.Dir[:len(writer.Dir)-len(tmpForCreationBlockDirSuffix)] + + if err := fileutil.Replace(writer.Dir, dir); err != nil { + return fmt.Errorf("failed to move temporary block dir {%s} to {%s}: %w", writer.Dir, dir, err) + } + + writer.Dir = dir + return nil +} + +type blockWriters []blockWriter + +// append appends a writer to the block writers. + +//nolint:gocritic // hugeParam // we accumulate the writers +func (bw *blockWriters) append(writer blockWriter) { + *bw = append(*bw, writer) +} + +// close closes the block writers. +func (bw *blockWriters) close() { + for i := range *bw { + _ = (*bw)[i].close() + } +} + +// recodeAndWriteChunksBatch recodes and writes the chunks batch. +func (bw *blockWriters) recodeAndWriteChunksBatch() error { + for i := range *bw { + if err := (*bw)[i].recodeAndWriteChunksBatch(); err != nil { + return err + } + } + + return nil +} + +// writeRestOfRecodedChunks writes the rest of the recoded chunks. +func (bw *blockWriters) writeRestOfRecodedChunks() error { + for i := range *bw { + if err := (*bw)[i].writeRestOfRecodedChunks(); err != nil { + return err + } + } + + return nil +} + +// writeIndexAndMoveTmpDirToDir writes the index and moves the temporary directory to the directory. +func (bw *blockWriters) writeIndexAndMoveTmpDirToDir() ([]WrittenBlock, error) { + writtenBlocks := make([]WrittenBlock, 0, len(*bw)) + for i := range *bw { + if (*bw)[i].isEmpty() { + _ = (*bw)[i].close() + if err := os.RemoveAll((*bw)[i].Dir); err != nil { + logger.Warnf("failed remove empty block: %s", (*bw)[i].Dir) + } + + continue + } + + if err := (*bw)[i].writeIndex(); err != nil { + return nil, err + } + + if err := (*bw)[i].moveTmpDirToDir(); err != nil { + return nil, err + } + + writtenBlocks = append(writtenBlocks, (*bw)[i].WrittenBlock) + } + + return writtenBlocks, nil +} + +type chunkRecoder struct { + chunkIterator ChunkIterator + chunksMetadata []ChunkMetadata + previousSeriesID uint32 +} + +func (recoder *chunkRecoder) recode( + chunkWriter *ChunkWriter, + blockMeta *tsdb.BlockMeta, + writeSeries func(seriesID uint32, chunksMetadata []ChunkMetadata) error, +) (err error) { + for recoder.chunkIterator.Next() { + chunk := recoder.chunkIterator.At() + + var chunkMetadata ChunkMetadata + if chunkMetadata, err = chunkWriter.Write(chunk); err != nil { + return fmt.Errorf("failed to write chunk: %w", err) + } + + adjustBlockMetaTimeRange(blockMeta, chunk.MinT(), chunk.MaxT()) + blockMeta.Stats.NumChunks++ + blockMeta.Stats.NumSamples += uint64(chunk.SampleCount()) + seriesID := chunk.SeriesID() + + if recoder.previousSeriesID == seriesID { + recoder.chunksMetadata = append(recoder.chunksMetadata, chunkMetadata) + } else { + if err = writeSeries(recoder.previousSeriesID, recoder.chunksMetadata); err != nil { + return err + } + blockMeta.Stats.NumSeries++ + recoder.chunksMetadata = append(recoder.chunksMetadata[:0], chunkMetadata) + recoder.previousSeriesID = seriesID + } + } + + recoder.chunkIterator.NextBatch() + return nil +} + +func newChunkRecoder(chunkIterator ChunkIterator) chunkRecoder { + return chunkRecoder{ + chunkIterator: chunkIterator, + previousSeriesID: math.MaxUint32, + } +} + +func adjustBlockMetaTimeRange(blockMeta *tsdb.BlockMeta, mint, maxt int64) { + if mint < blockMeta.MinTime { + blockMeta.MinTime = mint + } + + if maxt > blockMeta.MaxTime { + blockMeta.MaxTime = maxt + } +} + +func writeBlockMetaFile(fileName string, blockMeta *tsdb.BlockMeta) (int64, error) { + tmp := fileName + ".tmp" + defer func() { + if err := os.RemoveAll(tmp); err != nil { + logger.Errorf("failed to remove directory: %v", err) + } + }() + + metaFile, err := os.Create(tmp) // #nosec G304 // it's meant to be that way + if err != nil { + return 0, fmt.Errorf("failed to create block meta file: %w", err) + } + defer func() { + if metaFile != nil { + if err = metaFile.Close(); err != nil { + logger.Errorf("failed to close metadata file: %v", err) + } + } + }() + + jsonBlockMeta, err := json.MarshalIndent(blockMeta, "", "\t") + if err != nil { + return 0, fmt.Errorf("failed to marshal meta json: %w", err) + } + + n, err := metaFile.Write(jsonBlockMeta) + if err != nil { + return 0, fmt.Errorf("failed to write meta json: %w", err) + } + + if err = metaFile.Sync(); err != nil { + return 0, fmt.Errorf("failed to sync meta file: %w", err) + } + + if err = metaFile.Close(); err != nil { + return 0, fmt.Errorf("faield to close meta file: %w", err) + } + metaFile = nil + + return int64(n), fileutil.Replace(tmp, fileName) +} + +func createTmpDir(dir string) error { + if err := os.RemoveAll(dir); err != nil { + return err + } + + return os.MkdirAll( //nolint:gosec // need this permissions + dir, + 0o777, //revive:disable-line:add-constant // file permissions simple readable as octa-number + ) +} + +func syncDir(dir string) error { + df, err := fileutil.OpenDir(dir) + if err != nil { + return err + } + defer func() { + if df != nil { + _ = df.Close() + } + }() + + if err = df.Sync(); err != nil { + return err + } + + return df.Close() +} diff --git a/pp/go/storage/block/chunk_writer.go b/pp/go/storage/block/chunk_writer.go new file mode 100644 index 0000000000..7d8648f9e9 --- /dev/null +++ b/pp/go/storage/block/chunk_writer.go @@ -0,0 +1,273 @@ +package block + +import ( + "bufio" + "encoding/binary" + "errors" + "fmt" + "hash" + "hash/crc32" + "io" + "os" + "path/filepath" + + "github.com/prometheus/prometheus/tsdb/chunks" + "github.com/prometheus/prometheus/tsdb/fileutil" +) + +const ( + chunksFormatV1 = 1 +) + +// ChunkMetadata meta information for the chunk. +type ChunkMetadata struct { + MinT int64 + MaxT int64 + Ref uint64 +} + +// ChunkWriter a writer for encoding and writing chunks. +type ChunkWriter struct { + dirFile *os.File + files []*os.File + wbuf *bufio.Writer + n int64 + crc32 hash.Hash + segmentSize int64 + buf [binary.MaxVarintLen32]byte +} + +// NewChunkWriter init new [ChunkWriter]. +func NewChunkWriter(dir string, segmentSize int64) (*ChunkWriter, error) { + if segmentSize < 0 { + segmentSize = DefaultChunkSegmentSize + } + + if err := os.MkdirAll( //nolint:gosec // need this permissions + dir, + 0o777, //revive:disable-line:add-constant // file permissions simple readable as octa-number + ); err != nil { + return nil, fmt.Errorf("failed to create all dirs: %w", err) + } + + dirFile, err := fileutil.OpenDir(dir) + if err != nil { + return nil, fmt.Errorf("failed to open dir: %w", err) + } + + return &ChunkWriter{ + dirFile: dirFile, + crc32: crc32.New(crc32.MakeTable(crc32.Castagnoli)), + segmentSize: segmentSize, + }, nil +} + +// Close writes all pending data to the current tail file and closes chunk's files. +func (w *ChunkWriter) Close() (err error) { + if err = w.finalizeTail(); err != nil { + return fmt.Errorf("failed to finalize tail on close: %w", err) + } + + return w.dirFile.Close() +} + +// Write encoding and write to buffer chunk. +func (w *ChunkWriter) Write(chunk Chunk) (meta ChunkMetadata, err error) { + // calculate chunk size + chunkSize := int64(chunks.MaxChunkLengthFieldSize) + chunkSize += chunks.ChunkEncodingSize + chunkSize += int64(len(chunk.Bytes())) + chunkSize += crc32.Size + + // check segment boundaries and cut if needed + if w.n == 0 || w.n+chunkSize > w.segmentSize { + if err = w.cut(); err != nil { + return meta, fmt.Errorf("failed to cut file: %w", err) + } + } + + // write chunk + return w.writeChunk(chunk) +} + +func (w *ChunkWriter) cut() error { + // Sync current tail to disk and close. + if err := w.finalizeTail(); err != nil { + return err + } + + f, n, err := cutSegmentFile(w.dirFile, w.seq(), chunks.MagicChunks, chunksFormatV1, w.segmentSize) + if err != nil { + return err + } + w.n = int64(n) + + w.files = append(w.files, f) + if w.wbuf != nil { + w.wbuf.Reset(f) + } else { + w.wbuf = bufio.NewWriterSize(f, 8*1024*1024) + } + + return nil +} + +// finalizeTail writes all pending data to the current tail file, +// truncates its size, and closes it. +func (w *ChunkWriter) finalizeTail() error { + tf := w.tail() + if tf == nil { + return nil + } + + if err := w.wbuf.Flush(); err != nil { + return err + } + + if err := tf.Sync(); err != nil { + return err + } + // As the file was pre-allocated, we truncate any superfluous zero bytes. + off, err := tf.Seek(0, io.SeekCurrent) + if err != nil { + return err + } + if err := tf.Truncate(off); err != nil { + return err + } + + return tf.Close() +} + +func (w *ChunkWriter) seq() int { + return len(w.files) - 1 +} + +func (w *ChunkWriter) tail() *os.File { + if len(w.files) == 0 { + return nil + } + return w.files[len(w.files)-1] +} + +func (w *ChunkWriter) writeChunk(chunk Chunk) (meta ChunkMetadata, err error) { + meta.Ref = uint64(chunks.NewBlockChunkRef(uint64(w.seq()), uint64(w.n))) // #nosec G115 // no overflow + + n := binary.PutUvarint(w.buf[:], uint64(len(chunk.Bytes()))) + if err = w.writeToBuf(w.buf[:n]); err != nil { + return meta, err + } + + w.buf[0] = byte(chunk.Encoding()) + if err = w.writeToBuf(w.buf[:1]); err != nil { + return meta, err + } + + if err = w.writeToBuf(chunk.Bytes()); err != nil { + return meta, err + } + + w.crc32.Reset() + + buf := append(w.buf[:0], byte(chunk.Encoding())) + if _, err = w.crc32.Write(buf[:1]); err != nil { + return meta, err + } + + if _, err = w.crc32.Write(chunk.Bytes()); err != nil { + return meta, err + } + + if err = w.writeToBuf(w.crc32.Sum(w.buf[:0])); err != nil { + return meta, err + } + + meta.MinT = chunk.MinT() + meta.MaxT = chunk.MaxT() + + return meta, nil +} + +func (w *ChunkWriter) writeToBuf(b []byte) error { + n, err := w.wbuf.Write(b) + w.n += int64(n) + return err +} + +//revive:disable-next-line:function-length // long but readable. +//revive:disable-next-line:cyclomatic // but readable +func cutSegmentFile( + dirFile *os.File, + currentSeq int, + magicNumber uint32, + chunksFormat byte, + allocSize int64, +) (newFile *os.File, headerSize int, returnErr error) { + p, err := nextSequenceFile(dirFile.Name(), currentSeq) + if err != nil { + return nil, 0, fmt.Errorf("next sequence file: %w", err) + } + ptmp := p + ".tmp" + f, err := os.Create(ptmp) // #nosec G304 // it's meant to be that way + if err != nil { + return nil, 0, fmt.Errorf("open temp file: %w", err) + } + defer func() { + if returnErr != nil { + if f != nil { + returnErr = errors.Join(returnErr, f.Close()) + } + // Calling RemoveAll on a non-existent file does not return error. + returnErr = errors.Join(returnErr, os.RemoveAll(ptmp)) + } + }() + if allocSize > 0 { + if err = fileutil.Preallocate(f, allocSize, true); err != nil { + return nil, 0, fmt.Errorf("preallocate: %w", err) + } + } + + if err = dirFile.Sync(); err != nil { + return nil, 0, fmt.Errorf("sync directory: %w", err) + } + + // Write header metadata for new file. + metab := make([]byte, chunks.SegmentHeaderSize) + binary.BigEndian.PutUint32(metab[:chunks.MagicChunksSize], magicNumber) + metab[4] = chunksFormat //revive:disable-line:add-constant // 4 byte for chunksFormat + + n, err := f.Write(metab) + if err != nil { + return nil, 0, fmt.Errorf("write header: %w", err) + } + if err = f.Close(); err != nil { + return nil, 0, fmt.Errorf("close temp file: %w", err) + } + f = nil + + if err = fileutil.Rename(ptmp, p); err != nil { + return nil, 0, fmt.Errorf("replace file: %w", err) + } + + f, err = os.OpenFile( //nolint:gosec // need this permissions + p, + os.O_WRONLY, + 0o666, //revive:disable-line:add-constant // file permissions simple readable as octa-number + ) + if err != nil { + return nil, 0, fmt.Errorf("open final file: %w", err) + } + // Skip header for further writes. + if _, err := f.Seek(int64(n), 0); err != nil { + return nil, 0, fmt.Errorf("seek in final file: %w", err) + } + return f, n, nil +} + +func nextSequenceFile(dir string, currentSeq int) (string, error) { + return segmentFile(dir, currentSeq+1), nil +} + +func segmentFile(baseDir string, index int) string { + return filepath.Join(baseDir, fmt.Sprintf("%0.6d", index)) +} diff --git a/pp/go/storage/block/file_writer.go b/pp/go/storage/block/file_writer.go new file mode 100644 index 0000000000..199c90fc5c --- /dev/null +++ b/pp/go/storage/block/file_writer.go @@ -0,0 +1,62 @@ +package block + +import ( + "bufio" + "fmt" + "os" + "path/filepath" + + "github.com/prometheus/prometheus/tsdb/fileutil" +) + +// FileWriter a buffered file writer. +type FileWriter struct { + file *os.File + writeBuffer *bufio.Writer +} + +// NewFileWriter init new [FileWriter]. +func NewFileWriter(fileName string) (*FileWriter, error) { + dir := filepath.Dir(fileName) + df, err := fileutil.OpenDir(dir) + if err != nil { + return nil, fmt.Errorf("failed to open parent dir {%s}: %w", dir, err) + } + defer func() { _ = df.Close() }() + + if err = os.RemoveAll(fileName); err != nil { + return nil, fmt.Errorf("failed to cleanup {%s}: %w", fileName, err) + } + + indexFile, err := os.OpenFile( //nolint:gosec // need this permissions + fileName, + os.O_CREATE|os.O_WRONLY, + 0o666, //revive:disable-line:add-constant // file permissions simple readable as octa-number + ) + if err != nil { + return nil, fmt.Errorf(" failed to open file {%s}: %w", fileName, err) + } + + return &FileWriter{ + file: indexFile, + writeBuffer: bufio.NewWriterSize(indexFile, 1<<22), + }, nil +} + +// Close flush buffer to file and sync and closes file. +func (w *FileWriter) Close() error { + if err := w.writeBuffer.Flush(); err != nil { + return fmt.Errorf("failed to flush write buffer: %w", err) + } + + if err := w.file.Sync(); err != nil { + return fmt.Errorf("failed to sync index file: %w", err) + } + + return w.file.Close() +} + +// Write writes the contents of p into the buffer. +func (w *FileWriter) Write(p []byte) (n int, err error) { + return w.writeBuffer.Write(p) +} diff --git a/pp/go/storage/block/writer.go b/pp/go/storage/block/writer.go new file mode 100644 index 0000000000..3d2b154af7 --- /dev/null +++ b/pp/go/storage/block/writer.go @@ -0,0 +1,170 @@ +package block + +import ( + "time" + + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/prometheus/pp/go/cppbridge" + "github.com/prometheus/prometheus/pp/go/storage/head/shard" + "github.com/prometheus/prometheus/pp/go/util" +) + +const ( + // DefaultChunkSegmentSize is the default chunks segment size. + DefaultChunkSegmentSize = 512 * 1024 * 1024 + // DefaultBlockDuration is the default block duration. + DefaultBlockDuration = 2 * time.Hour +) + +// LsIdBatchSize is the batch size for label set ID. +var LsIdBatchSize uint32 = 100000 + +// Shard the minimum required head [Shard] implementation. +type Shard interface { + LSS() *shard.LSS + + DataStorage() *shard.DataStorage + + UnloadedDataStorage() *shard.UnloadedDataStorage +} + +// Writer represents a block writer. It is used to write blocks to disk from a shard. +type Writer[TShard Shard] struct { + dataDir string + maxBlockChunkSegmentSize int64 + blockDurationMs int64 + blockWriteDuration *prometheus.GaugeVec +} + +// NewWriter creates a new [Writer]. +func NewWriter[TShard Shard]( + dataDir string, + maxBlockChunkSegmentSize int64, + blockDuration time.Duration, + registerer prometheus.Registerer, +) *Writer[TShard] { + factory := util.NewUnconflictRegisterer(registerer) + return &Writer[TShard]{ + dataDir: dataDir, + maxBlockChunkSegmentSize: maxBlockChunkSegmentSize, + blockDurationMs: blockDuration.Milliseconds(), + blockWriteDuration: factory.NewGaugeVec(prometheus.GaugeOpts{ + Name: "prompp_block_write_duration", + Help: "Block write duration in milliseconds.", + }, []string{"block_id"}), + } +} + +// Write writes blocks to disk from a shard. +func (w *Writer[TShard]) Write(sd TShard) (writtenBlocks []WrittenBlock, err error) { + _ = sd.LSS().WithRLock(func(_, _ *cppbridge.LabelSetStorage) error { + var writers blockWriters + writers, err = w.createWriters(sd) + if err != nil { + return err + } + + defer func() { + writers.close() + }() + + if err = w.recodeAndWriteChunks(sd, writers); err != nil { + return err + } + + writtenBlocks, err = writers.writeIndexAndMoveTmpDirToDir() + + return nil + }) + + return writtenBlocks, err +} + +// createWriters creates writers for the shard. +func (w *Writer[TShard]) createWriters(sd TShard) (blockWriters, error) { + var writers blockWriters + + timeInterval := sd.DataStorage().TimeInterval(false) + + quantStart := (timeInterval.MinT / w.blockDurationMs) * w.blockDurationMs + for ; quantStart <= timeInterval.MaxT; quantStart += w.blockDurationMs { + minT, maxT := quantStart, quantStart+w.blockDurationMs-1 + if minT < timeInterval.MinT { + minT = timeInterval.MinT + } + if maxT > timeInterval.MaxT { + maxT = timeInterval.MaxT + } + + var chunkIterator ChunkIterator + _ = sd.DataStorage().WithRLock(func(*cppbridge.HeadDataStorage) error { + chunkIterator = NewChunkIterator(sd.LSS().Target(), LsIdBatchSize, sd.DataStorage().Raw(), minT, maxT) + return nil + }) + + writer, err := newBlockWriter( + w.dataDir, + w.maxBlockChunkSegmentSize, + NewIndexWriter(sd.LSS().Target()), + chunkIterator, + ) + if err != nil { + writers.close() + return blockWriters{}, err + } + + writers.append(writer) + } + + return writers, nil +} + +// recodeAndWriteChunks recodes and writes chunks for the shard. +func (*Writer[TShard]) recodeAndWriteChunks(sd TShard, writers blockWriters) error { + var loader *cppbridge.UnloadedDataRevertableLoader + _ = sd.DataStorage().WithRLock(func(*cppbridge.HeadDataStorage) error { + loader = sd.DataStorage().CreateRevertableLoader(sd.LSS().Target(), LsIdBatchSize) + return nil + }) + + isFirstBatch := true + + loadData := func() (bool, error) { + if isFirstBatch { + isFirstBatch = false + } else if !loader.NextBatch() { + return false, nil + } + + if sd.UnloadedDataStorage() == nil { + return true, nil + } + + return true, sd.UnloadedDataStorage().ForEachSnapshot(loader.Load) + } + + for { + var hasMoreData bool + var err error + _ = sd.DataStorage().WithLock(func(*cppbridge.HeadDataStorage) error { + hasMoreData, err = loadData() + return nil + }) + + if !hasMoreData { + break + } + + if err != nil { + return err + } + + if err = sd.DataStorage().WithRLock(func(*cppbridge.HeadDataStorage) error { + return writers.recodeAndWriteChunksBatch() + }); err != nil { + return err + } + } + + return writers.writeRestOfRecodedChunks() +} diff --git a/pp/go/storage/block/writer_test.go b/pp/go/storage/block/writer_test.go new file mode 100644 index 0000000000..ce1a070921 --- /dev/null +++ b/pp/go/storage/block/writer_test.go @@ -0,0 +1,302 @@ +package block_test + +import ( + "encoding/json" + "os" + "path/filepath" + "testing" + "time" + + "github.com/jonboulle/clockwork" + "github.com/oklog/ulid" + "github.com/prometheus/client_golang/prometheus" + "github.com/stretchr/testify/suite" + + "github.com/prometheus/prometheus/model/labels" + "github.com/prometheus/prometheus/pp/go/cppbridge" + "github.com/prometheus/prometheus/pp/go/storage" + "github.com/prometheus/prometheus/pp/go/storage/block" + "github.com/prometheus/prometheus/pp/go/storage/catalog" + "github.com/prometheus/prometheus/pp/go/storage/head/services" + "github.com/prometheus/prometheus/pp/go/storage/head/shard" + "github.com/prometheus/prometheus/pp/go/storage/storagetest" + "github.com/prometheus/prometheus/tsdb" +) + +const ( + numberOfShards uint16 = 1 + + maxSegmentSize uint32 = 1024 + + unloadDataStorageInterval time.Duration = time.Second + + blockDuration = 2 * time.Hour +) + +type WriterSuite struct { + suite.Suite + dataDir string + head *storage.Head + blockWriter *block.Writer[*shard.Shard] +} + +func TestWriterSuite(t *testing.T) { + suite.Run(t, new(WriterSuite)) +} + +func (s *WriterSuite) SetupTest() { + s.dataDir = s.createDataDirectory() + s.head = s.mustCreateHead() + s.blockWriter = block.NewWriter[*shard.Shard]( + s.dataDir, + block.DefaultChunkSegmentSize, + blockDuration, + prometheus.DefaultRegisterer, + ) +} + +func (s *WriterSuite) createDataDirectory() string { + dataDir := filepath.Join(s.T().TempDir(), "data") + s.Require().NoError(os.MkdirAll(dataDir, os.ModeDir)) + return dataDir +} + +func (s *WriterSuite) mustCreateCatalog() *catalog.Catalog { + l, err := catalog.NewFileLogV2(filepath.Join(s.dataDir, "catalog.log")) + s.Require().NoError(err) + + c, err := catalog.New( + clockwork.NewFakeClock(), + l, + &catalog.DefaultIDGenerator{}, + catalog.DefaultMaxLogFileSize, + nil, + ) + s.Require().NoError(err) + + return c +} + +func (s *WriterSuite) mustCreateHead() *storage.Head { + h, err := storage.NewBuilder( + s.mustCreateCatalog(), + s.dataDir, + maxSegmentSize, + prometheus.DefaultRegisterer, + unloadDataStorageInterval, + ).Build(0, numberOfShards) + s.Require().NoError(err) + return h +} + +func (s *WriterSuite) mustReadFile(filename string) []byte { + b, err := os.ReadFile(filename) + s.Require().NoError(err) + return b +} + +func (s *WriterSuite) mustReadBlockMeta(filename string) tsdb.BlockMeta { + var meta tsdb.BlockMeta + s.NoError(json.Unmarshal(s.mustReadFile(filename), &meta)) + return meta +} + +func (s *WriterSuite) shard() *shard.Shard { + for sd := range s.head.RangeShards() { + return sd + } + + return nil +} + +func (s *WriterSuite) fillHead() { + ts := time.UnixMilli(1753805651969) + storagetest.MustAppendTimeSeries(&s.Suite, s.head, []storagetest.TimeSeries{ + { + Labels: labels.FromStrings("__name__", "value1"), + Samples: []cppbridge.Sample{ + {Timestamp: ts.UnixMilli(), Value: 0}, + {Timestamp: ts.Add(time.Minute).UnixMilli(), Value: 1}, + {Timestamp: ts.Add(time.Hour * 2).UnixMilli(), Value: 2}, + }, + }, + { + Labels: labels.FromStrings("__name__", "value2"), + Samples: []cppbridge.Sample{ + {Timestamp: ts.UnixMilli(), Value: 0}, + {Timestamp: ts.Add(time.Minute).UnixMilli(), Value: 1}, + {Timestamp: ts.Add(time.Hour * 2).UnixMilli(), Value: 2}, + }, + }, + }) +} + +func (s *WriterSuite) assertWrittenBlocks(blocks []block.WrittenBlock, err error) { + s.Require().NoError(err) + + s.Require().Len(blocks, 2) + + meta1 := s.mustReadBlockMeta(blocks[0].MetaFilename()) + s.Equal(tsdb.BlockMeta{ + MinTime: 1753805651969, + MaxTime: 1753805711970, + Stats: tsdb.BlockStats{ + NumSamples: 4, + NumSeries: 2, + NumChunks: 2, + }, + ULID: meta1.ULID, + Compaction: tsdb.BlockMetaCompaction{ + Level: 1, + Sources: []ulid.ULID{meta1.ULID}, + }, + Version: 1, + }, meta1) + + meta2 := s.mustReadBlockMeta(blocks[1].MetaFilename()) + s.Equal(tsdb.BlockMeta{ + MinTime: 1753812851969, + MaxTime: 1753812851970, + Stats: tsdb.BlockStats{ + NumSamples: 2, + NumSeries: 2, + NumChunks: 2, + }, + ULID: meta2.ULID, + Compaction: tsdb.BlockMetaCompaction{ + Level: 1, + Sources: []ulid.ULID{meta2.ULID}, + }, + Version: 1, + }, meta2) + + s.Equal([]byte{ + 0xba, 0xaa, 0xd7, 0x00, 0x02, 0x00, 0x00, 0x00, 0x1c, 0x00, 0x00, 0x00, 0x04, 0x00, 0x08, 0x5f, + 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x5f, 0x5f, 0x06, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x31, 0x06, 0x76, + 0x61, 0x6c, 0x75, 0x65, 0x32, 0x4b, 0x1c, 0x0b, 0x1f, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x0e, 0x01, 0x01, 0x02, 0x01, 0x82, 0x80, 0xb4, 0xef, 0x8a, 0x66, 0xe0, 0xd4, 0x03, 0x08, 0xfc, + 0x04, 0xb8, 0xde, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x0e, 0x01, 0x01, 0x03, 0x01, 0x82, 0x80, 0xb4, 0xef, 0x8a, 0x66, 0xe0, 0xd4, 0x03, 0x24, 0xa0, + 0x8b, 0x2d, 0xe0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x02, + 0x00, 0x00, 0x00, 0x03, 0xfc, 0x41, 0x54, 0x42, 0x00, 0x00, 0x00, 0x0c, 0x00, 0x00, 0x00, 0x02, + 0x00, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, 0x05, 0xbb, 0x33, 0xcb, 0xd4, 0x00, 0x00, 0x00, 0x08, + 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x03, 0xa7, 0x69, 0x2e, 0xd2, 0x00, 0x00, 0x00, 0x08, + 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x05, 0x81, 0xc8, 0xc9, 0x3a, 0x00, 0x00, 0x00, 0x0f, + 0x00, 0x00, 0x00, 0x01, 0x01, 0x08, 0x5f, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x5f, 0x5f, 0x70, 0x10, + 0x86, 0x91, 0x1b, 0x00, 0x00, 0x00, 0x2f, 0x00, 0x00, 0x00, 0x03, 0x02, 0x00, 0x00, 0x88, 0x01, + 0x02, 0x08, 0x5f, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x5f, 0x5f, 0x06, 0x76, 0x61, 0x6c, 0x75, 0x65, + 0x31, 0x9c, 0x01, 0x02, 0x08, 0x5f, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x5f, 0x5f, 0x06, 0x76, 0x61, + 0x6c, 0x75, 0x65, 0x32, 0xac, 0x01, 0xb4, 0x22, 0x64, 0x7c, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x05, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x29, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x70, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xbc, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x88, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xd3, 0x13, 0xb6, 0x16, 0x2c, + }, s.mustReadFile(blocks[0].IndexFilename())) + + s.Equal([]byte{ + 0xba, 0xaa, 0xd7, 0x00, 0x02, 0x00, 0x00, 0x00, 0x1c, 0x00, 0x00, 0x00, 0x04, 0x00, 0x08, 0x5f, + 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x5f, 0x5f, 0x06, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x31, 0x06, 0x76, + 0x61, 0x6c, 0x75, 0x65, 0x32, 0x4b, 0x1c, 0x0b, 0x1f, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x0c, 0x01, 0x01, 0x02, 0x01, 0x82, 0xf4, 0xa2, 0xf6, 0x8a, 0x66, 0x00, 0x08, 0xbe, 0xfa, 0x4f, + 0x7a, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x0c, 0x01, 0x01, 0x03, 0x01, 0x82, 0xf4, 0xa2, 0xf6, 0x8a, 0x66, 0x00, 0x1e, 0x43, 0x53, 0x14, + 0x58, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x10, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x02, 0x00, 0x00, 0x00, 0x02, + 0x00, 0x00, 0x00, 0x03, 0xfc, 0x41, 0x54, 0x42, 0x00, 0x00, 0x00, 0x0c, 0x00, 0x00, 0x00, 0x02, + 0x00, 0x00, 0x00, 0x03, 0x00, 0x00, 0x00, 0x05, 0xbb, 0x33, 0xcb, 0xd4, 0x00, 0x00, 0x00, 0x08, + 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x03, 0xa7, 0x69, 0x2e, 0xd2, 0x00, 0x00, 0x00, 0x08, + 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x05, 0x81, 0xc8, 0xc9, 0x3a, 0x00, 0x00, 0x00, 0x0f, + 0x00, 0x00, 0x00, 0x01, 0x01, 0x08, 0x5f, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x5f, 0x5f, 0x70, 0x10, + 0x86, 0x91, 0x1b, 0x00, 0x00, 0x00, 0x2f, 0x00, 0x00, 0x00, 0x03, 0x02, 0x00, 0x00, 0x88, 0x01, + 0x02, 0x08, 0x5f, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x5f, 0x5f, 0x06, 0x76, 0x61, 0x6c, 0x75, 0x65, + 0x31, 0x9c, 0x01, 0x02, 0x08, 0x5f, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x5f, 0x5f, 0x06, 0x76, 0x61, + 0x6c, 0x75, 0x65, 0x32, 0xac, 0x01, 0xb4, 0x22, 0x64, 0x7c, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x05, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x29, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x70, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xbc, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x88, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xd3, 0x13, 0xb6, 0x16, 0x2c, + }, s.mustReadFile(blocks[1].IndexFilename())) + + s.Equal([]byte{ + 0x85, 0xBD, 0x40, 0xDD, 0x01, 0x00, 0x00, 0x00, 0x16, 0x01, 0x00, 0x02, 0x82, 0x80, 0xB4, 0xEF, + 0x8A, 0x66, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xE0, 0xD4, 0x03, 0xC4, 0x57, 0xFE, + 0xC0, 0xEE, 0xF0, 0x5E, 0x16, 0x01, 0x00, 0x02, 0x82, 0x80, 0xB4, 0xEF, 0x8A, 0x66, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xE0, 0xD4, 0x03, 0xC4, 0x57, 0xFE, 0xC0, 0xEE, 0xF0, 0x5E, + }, s.mustReadFile(filepath.Join(blocks[0].ChunkDir(), "000000"))) + + s.Equal([]byte{ + 0x85, 0xBD, 0x40, 0xDD, 0x01, 0x00, 0x00, 0x00, 0x10, 0x01, 0x00, 0x01, 0x82, 0xF4, 0xA2, 0xF6, + 0x8A, 0x66, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x29, 0x21, 0xAB, 0xF9, 0x10, 0x01, + 0x00, 0x01, 0x82, 0xF4, 0xA2, 0xF6, 0x8A, 0x66, 0x40, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x29, 0x21, 0xAB, 0xF9, + }, s.mustReadFile(filepath.Join(blocks[1].ChunkDir(), "000000"))) +} + +func (s *WriterSuite) TestWrite() { + // Arrange + s.fillHead() + + // Act + block.LsIdBatchSize = 2 + blocks, err := s.blockWriter.Write(s.shard()) + + // Assert + s.assertWrittenBlocks(blocks, err) +} + +func (s *WriterSuite) TestWriteInBatches() { + // Arrange + s.fillHead() + + // Act + block.LsIdBatchSize = 1 + blocks, err := s.blockWriter.Write(s.shard()) + + // Assert + s.assertWrittenBlocks(blocks, err) +} + +func (s *WriterSuite) TestWriteWithDataUnloading() { + // Arrange + s.fillHead() + s.Require().NoError(services.UnloadUnusedSeriesDataWithHead(s.head)) + + // Act + block.LsIdBatchSize = 2 + blocks, err := s.blockWriter.Write(s.shard()) + + // Assert + s.assertWrittenBlocks(blocks, err) +} + +func (s *WriterSuite) TestWriteWithDataUnloadingInBatches() { + // Arrange + s.fillHead() + s.Require().NoError(services.UnloadUnusedSeriesDataWithHead(s.head)) + + // Act + block.LsIdBatchSize = 1 + blocks, err := s.blockWriter.Write(s.shard()) + + // Assert + s.assertWrittenBlocks(blocks, err) +} + +func (s *WriterSuite) TestSkipEmptyBlock() { + // Arrange + storagetest.MustAppendTimeSeries(&s.Suite, s.head, []storagetest.TimeSeries{ + { + Labels: labels.FromStrings("__name__", "value1"), + Samples: []cppbridge.Sample{ + {Timestamp: 0, Value: 0}, + {Timestamp: blockDuration.Milliseconds() * 2, Value: 1}, + }, + }, + }) + + // Act + blocks, err := s.blockWriter.Write(s.shard()) + + // Assert + s.Require().NoError(err) + s.Len(blocks, 2) +} diff --git a/pp/go/storage/builder.go b/pp/go/storage/builder.go new file mode 100644 index 0000000000..e3f8fb7445 --- /dev/null +++ b/pp/go/storage/builder.go @@ -0,0 +1,160 @@ +package storage + +import ( + "errors" + "fmt" + "os" + "path/filepath" + "time" + + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/prometheus/pp/go/cppbridge" + "github.com/prometheus/prometheus/pp/go/logger" + "github.com/prometheus/prometheus/pp/go/storage/catalog" + "github.com/prometheus/prometheus/pp/go/storage/head/head" + "github.com/prometheus/prometheus/pp/go/storage/head/shard" + "github.com/prometheus/prometheus/pp/go/storage/head/shard/wal" + "github.com/prometheus/prometheus/pp/go/storage/head/shard/wal/writer" + "github.com/prometheus/prometheus/pp/go/util" +) + +// +// Builder +// + +// Builder building new [Head] with parameters. +type Builder struct { + catalog *catalog.Catalog + dataDir string + maxSegmentSize uint32 + registerer prometheus.Registerer + unloadDataStorageInterval time.Duration + // stat + events *prometheus.CounterVec +} + +// NewBuilder init new [Builder]. +func NewBuilder( + hcatalog *catalog.Catalog, + dataDir string, + maxSegmentSize uint32, + registerer prometheus.Registerer, + unloadDataStorageInterval time.Duration, +) *Builder { + factory := util.NewUnconflictRegisterer(registerer) + return &Builder{ + catalog: hcatalog, + dataDir: dataDir, + maxSegmentSize: maxSegmentSize, + registerer: registerer, + unloadDataStorageInterval: unloadDataStorageInterval, + events: factory.NewCounterVec( + prometheus.CounterOpts{ + Name: "prompp_head_event_count", + Help: "Number of head events", + }, + []string{"type"}, + ), + } +} + +// Build new [Head] - [head.Head] with [shard.Shard] with [wal.Wal] which is written to disk. +func (b *Builder) Build(generation uint64, numberOfShards uint16) (*Head, error) { + headRecord, err := b.catalog.Create(numberOfShards) + if err != nil { + return nil, err + } + + headDir := filepath.Join(b.dataDir, headRecord.ID()) + //revive:disable-next-line:add-constant // this is already a constant + if err = os.Mkdir(headDir, 0o777); err != nil { //nolint:gosec // need this permissions + return nil, err + } + defer func() { + if err != nil { + err = errors.Join(err, os.RemoveAll(headDir)) + } + }() + + shards := make([]*shard.Shard, numberOfShards) + swn := writer.NewSegmentWriteNotifier(numberOfShards, headRecord.SetLastAppendedSegmentID) + for shardID := range numberOfShards { + s, err := b.createShardOnDisk(headDir, swn, shardID) + if err != nil { + return nil, err + } + + shards[shardID] = s + } + + b.events.With(prometheus.Labels{"type": "created"}).Inc() + logger.Debugf("[Builder] builded head: %s", headRecord.ID()) + return head.NewHead( + headRecord.ID(), + shards, + shard.NewPerGoroutineShard[*Wal], + headRecord.Acquire(), + generation, + b.registerer, + ), nil +} + +// createShardOnDisk create [shard.Shard] with [wal.Wal] which is written to disk. +// +//revive:disable-next-line:function-length // long but readable. +func (b *Builder) createShardOnDisk( + headDir string, + swn *writer.SegmentWriteNotifier, + shardID uint16, +) (*shard.Shard, error) { + headDir = filepath.Clean(headDir) + //revive:disable-next-line:add-constant // file permissions simple readable as octa-number + shardFile, err := util.CreateFileAppender(GetShardWalFilename(headDir, shardID), 0o666) + if err != nil { + return nil, fmt.Errorf("failed to create shard wal file id %d: %w", shardID, err) + } + + defer func() { + if err == nil { + return + } + + _ = shardFile.Close() + }() + + lss := shard.NewLSS() + // logShards is 0 for single encoder + shardWalEncoder := cppbridge.NewHeadWalEncoder(shardID, 0, lss.Target()) + + _, err = writer.WriteHeader(shardFile, wal.FileFormatVersion, shardWalEncoder.Version()) + if err != nil { + return nil, fmt.Errorf("failed to write header: %w", err) + } + + sw, err := writer.NewBuffered(shardID, shardFile, writer.WriteSegment[*cppbridge.HeadEncodedSegment], swn) + if err != nil { + return nil, fmt.Errorf("failed to create buffered writer shard id %d: %w", shardID, err) + } + + var unloadedDataStorage *shard.UnloadedDataStorage + var queriedSeriesStorage *shard.QueriedSeriesStorage + if b.unloadDataStorageInterval != 0 { + unloadedDataStorage = shard.NewUnloadedDataStorage( + shard.NewAppendFileStorage(GetUnloadedDataStorageFilename(headDir, shardID)), + ) + + queriedSeriesStorage = shard.NewQueriedSeriesStorage( + shard.NewFileStorage(GetQueriedSeriesStorageFilename(headDir, shardID, 0)), + shard.NewFileStorage(GetQueriedSeriesStorageFilename(headDir, shardID, 1)), + ) + } + + return shard.NewShard( + lss, + shard.NewDataStorage(), + unloadedDataStorage, + queriedSeriesStorage, + wal.NewWal(shardWalEncoder, sw, b.maxSegmentSize, shardID, b.registerer), + shardID, + ), nil +} diff --git a/pp/go/storage/catalog/catalog.go b/pp/go/storage/catalog/catalog.go new file mode 100644 index 0000000000..2470d29a41 --- /dev/null +++ b/pp/go/storage/catalog/catalog.go @@ -0,0 +1,344 @@ +package catalog + +import ( + "errors" + "fmt" + "io" + "sort" + "sync" + + "github.com/google/uuid" + "github.com/jonboulle/clockwork" + "github.com/prometheus/client_golang/prometheus" + + "github.com/prometheus/prometheus/pp/go/logger" + "github.com/prometheus/prometheus/pp/go/util" +) + +const ( + // DefaultMaxLogFileSize default size of log file. + DefaultMaxLogFileSize = 4 << 20 + + // compactErr format string for compact error. + compactErr = "compact: %w" + // logWriteErr format string for log write error. + logWriteErr = "log write: %w" + // notFoundErr format string for not found id error. + notFoundErr = "not found: %s" +) + +// +// Log +// + +// Log head-log file, contains [Record]s of heads. +type Log interface { + // ReWrite rewrite [FileLog] with [Record]s. + ReWrite(records ...*Record) error + + // Read [Record] from [FileLog]. + Read(record *Record) error + + // Size return current size of [FileHandler]. + Size() int + + // Write [Record] to [FileLog]. + Write(record *Record) error +} + +// +// IDGenerator +// + +// IDGenerator generator UUID. +type IDGenerator interface { + // Generate UUID. + Generate() uuid.UUID +} + +// DefaultIDGenerator default generator UUID. +type DefaultIDGenerator struct{} + +// Generate UUID. +func (DefaultIDGenerator) Generate() uuid.UUID { + return uuid.New() +} + +// +// Catalog +// + +// Catalog of current head records. +type Catalog struct { + mtx sync.Mutex + clock clockwork.Clock + log Log + idGenerator IDGenerator + records map[string]*Record + maxLogFileSize int + corruptedHead prometheus.Counter + activeHeadCreatedAt prometheus.Gauge +} + +// New init new [Catalog]. +func New( + clock clockwork.Clock, + log Log, + idGenerator IDGenerator, + maxLogFileSize int, + registerer prometheus.Registerer, +) (*Catalog, error) { + factory := util.NewUnconflictRegisterer(registerer) + catalog := &Catalog{ + clock: clock, + log: log, + idGenerator: idGenerator, + records: make(map[string]*Record), + maxLogFileSize: maxLogFileSize, + corruptedHead: factory.NewCounter( + prometheus.CounterOpts{ + Name: "prompp_head_catalog_corrupted_head_total", + Help: "Total number of corrupted heads.", + }, + ), + activeHeadCreatedAt: factory.NewGauge( + prometheus.GaugeOpts{ + Name: "prompp_head_catalog_active_head_created_at", + Help: "The time when the active head was created.", + }, + ), + } + + if err := catalog.sync(); err != nil { + return nil, fmt.Errorf("failed to sync catalog: %w", err) + } + + return catalog, nil +} + +// Compact catalog. +func (c *Catalog) Compact() error { + c.mtx.Lock() + defer c.mtx.Unlock() + return c.compactLog() +} + +// Create creates new [Record] and write to [Log]. +func (c *Catalog) Create(numberOfShards uint16) (*Record, error) { + c.mtx.Lock() + defer c.mtx.Unlock() + + if err := c.compactIfNeeded(); err != nil { + return nil, fmt.Errorf(compactErr, err) + } + + id := c.idGenerator.Generate() + now := c.clock.Now().UnixMilli() + r := &Record{ + id: id, + numberOfShards: numberOfShards, + createdAt: now, + updatedAt: now, + deletedAt: 0, + referenceCount: 0, + status: StatusNew, + } + + if err := c.log.Write(r); err != nil { + return r, fmt.Errorf(logWriteErr, err) + } + c.records[id.String()] = r + + return r, nil +} + +// Delete record by ID. +func (c *Catalog) Delete(id string) error { + c.mtx.Lock() + defer c.mtx.Unlock() + + if err := c.compactIfNeeded(); err != nil { + return fmt.Errorf(compactErr, err) + } + + r, ok := c.records[id] + if !ok || r.deletedAt > 0 { + return nil + } + + changed := createRecordCopy(r) + changed.deletedAt = c.clock.Now().UnixMilli() + changed.updatedAt = r.deletedAt + + if err := c.log.Write(changed); err != nil { + return fmt.Errorf(logWriteErr, err) + } + + applyRecordChanges(r, changed) + delete(c.records, r.id.String()) + + return nil +} + +// Get returns [Record] if exist. +func (c *Catalog) Get(id string) (*Record, error) { + c.mtx.Lock() + defer c.mtx.Unlock() + + r, ok := c.records[id] + if !ok { + return nil, fmt.Errorf(notFoundErr, id) + } + + return r, nil +} + +// List returns slice of records with filter and sort. +func (c *Catalog) List(filterFn func(record *Record) bool, sortLess func(lhs, rhs *Record) bool) []*Record { + records := c.listWithFilter(filterFn) + + if sortLess != nil { + sort.Slice(records, func(i, j int) bool { + return sortLess(records[i], records[j]) + }) + } + + return records +} + +// listWithFilter returns slice of filtered records +func (c *Catalog) listWithFilter(filterFn func(record *Record) bool) []*Record { + c.mtx.Lock() + defer c.mtx.Unlock() + + records := make([]*Record, 0, len(c.records)) + for _, record := range c.records { + if filterFn != nil && !filterFn(record) { + continue + } + records = append(records, record) + } + + return records +} + +// OnDiskSize size of [Log] file on disk. +func (c *Catalog) OnDiskSize() int64 { + return int64(c.log.Size()) +} + +// SetCorrupted set corrupted flag for ID and returns [Record] if exist. +func (c *Catalog) SetCorrupted(id string) (_ *Record, err error) { + c.mtx.Lock() + defer c.mtx.Unlock() + + if err = c.compactIfNeeded(); err != nil { + return nil, fmt.Errorf(compactErr, err) + } + + r, ok := c.records[id] + if !ok { + return nil, fmt.Errorf(notFoundErr, id) + } + + if r.corrupted { + return r, nil + } + + changed := createRecordCopy(r) + changed.corrupted = true + changed.updatedAt = c.clock.Now().UnixMilli() + + if err = c.log.Write(changed); err != nil { + return r, fmt.Errorf(logWriteErr, err) + } + + applyRecordChanges(r, changed) + c.records[id] = r + + c.corruptedHead.Inc() + + return r, nil +} + +// SetStatus set status for ID and returns [Record] if exist. +func (c *Catalog) SetStatus(id string, status Status) (_ *Record, err error) { + c.mtx.Lock() + defer c.mtx.Unlock() + + if err = c.compactIfNeeded(); err != nil { + return nil, fmt.Errorf(compactErr, err) + } + + r, ok := c.records[id] + if !ok { + return nil, fmt.Errorf(notFoundErr, id) + } + + if r.status == status { + if status == StatusActive { + c.activeHeadCreatedAt.Set(float64(r.createdAt)) + } + + return r, nil + } + + changed := createRecordCopy(r) + changed.status = status + changed.updatedAt = c.clock.Now().UnixMilli() + + if err = c.log.Write(changed); err != nil { + return r, fmt.Errorf(logWriteErr, err) + } + + applyRecordChanges(r, changed) + c.records[id] = r + + if status == StatusActive { + c.activeHeadCreatedAt.Set(float64(r.createdAt)) + } + + return r, nil +} + +// compactIfNeeded compact [Catalog] if necessary. +func (c *Catalog) compactIfNeeded() error { + if c.log.Size() < c.maxLogFileSize { + return nil + } + + return c.compactLog() +} + +// compactLog delete old(deleted [Record]s). +func (c *Catalog) compactLog() error { + records := make([]*Record, 0, len(c.records)) + for _, record := range c.records { + if record.deletedAt == 0 { + records = append(records, record) + } + } + + sort.Slice(records, func(i, j int) bool { + return records[i].createdAt < records[j].createdAt + }) + + return c.log.ReWrite(records...) +} + +// sync catalog with [Log]. +func (c *Catalog) sync() error { + for { + r := NewEmptyRecord() + if err := c.log.Read(r); err != nil { + if errors.Is(err, io.EOF) { + return nil + } + // this could happen if log file is corrupted + logger.Errorf("catalog is corrupted: %v", err) + + return c.compactLog() + } + c.records[r.id.String()] = r + } +} diff --git a/pp/go/storage/catalog/catalog_test.go b/pp/go/storage/catalog/catalog_test.go new file mode 100644 index 0000000000..5718b7a81f --- /dev/null +++ b/pp/go/storage/catalog/catalog_test.go @@ -0,0 +1,178 @@ +package catalog_test + +import ( + "os" + "sort" + "testing" + "time" + + "github.com/google/uuid" + "github.com/jonboulle/clockwork" + "github.com/prometheus/client_golang/prometheus" + "github.com/stretchr/testify/suite" + + "github.com/prometheus/prometheus/pp/go/storage/catalog" +) + +type CatalogSuite struct { + suite.Suite +} + +func TestCatalogSuite(t *testing.T) { + suite.Run(t, new(CatalogSuite)) +} + +func (s *CatalogSuite) TestHappyPath() { + tmpFile, err := os.CreateTemp("", "log_file") + s.Require().NoError(err) + + logFileName := tmpFile.Name() + s.Require().NoError(tmpFile.Close()) + + l, err := catalog.NewFileLogV2(logFileName) + s.Require().NoError(err) + + clock := clockwork.NewFakeClockAt(time.Now()) + + idGenerator := &testIDGenerator{} + + c, err := catalog.New( + clock, + l, + idGenerator, + catalog.DefaultMaxLogFileSize, + nil, + ) + s.Require().NoError(err) + + now := clock.Now().UnixMilli() + + var nos1 uint16 = 2 + var nos2 uint16 = 4 + + r1, err := c.Create(nos1) + s.Require().NoError(err) + + s.Require().Equal(idGenerator.last(), r1.ID()) + s.Require().Equal(idGenerator.last(), r1.Dir()) + s.Require().Equal(nos1, r1.NumberOfShards()) + s.Require().Equal(now, r1.CreatedAt()) + s.Require().Equal(now, r1.UpdatedAt()) + s.Require().Equal(int64(0), r1.DeletedAt()) + s.Require().Equal(catalog.StatusNew, r1.Status()) + + clock.Advance(time.Second) + now = clock.Now().UnixMilli() + + r2, err := c.Create(nos2) + s.Require().NoError(err) + + s.Require().Equal(idGenerator.last(), r2.ID()) + s.Require().Equal(idGenerator.last(), r2.Dir()) + s.Require().Equal(nos2, r2.NumberOfShards()) + s.Require().Equal(now, r2.CreatedAt()) + s.Require().Equal(now, r2.UpdatedAt()) + s.Require().Equal(int64(0), r2.DeletedAt()) + s.Require().Equal(catalog.StatusNew, r2.Status()) + + _, err = c.SetStatus(r1.ID(), catalog.StatusPersisted) + s.Require().NoError(err) + + c = nil + s.Require().NoError(l.Close()) + + l, err = catalog.NewFileLogV2(logFileName) + s.Require().NoError(err) + c, err = catalog.New( + clock, + l, + catalog.DefaultIDGenerator{}, + catalog.DefaultMaxLogFileSize, + nil, + ) + s.Require().NoError(err) + + records := c.List(nil, nil) + sort.Slice(records, func(i, j int) bool { + return records[i].CreatedAt() < records[j].CreatedAt() + }) +} + +func (s *CatalogSuite) TestCatalogSyncFail() { + tmpFile, err := os.CreateTemp("", "log_file") + s.Require().NoError(err) + + logFileName := tmpFile.Name() + s.Require().NoError(tmpFile.Close()) + + l, err := catalog.NewFileLogV2(logFileName) + s.Require().NoError(err) + + clock := clockwork.NewFakeClockAt(time.Now()) + + idGenerator := &testIDGenerator{} + + c, err := catalog.New( + clock, + l, + idGenerator, + catalog.DefaultMaxLogFileSize, + prometheus.DefaultRegisterer, + ) + s.Require().NoError(err) + + var nos1 uint16 = 2 + var nos2 uint16 = 4 + + r1, err := c.Create(nos1) + s.Require().NoError(err) + + r2, err := c.Create(nos2) + s.Require().NoError(err) + + fileInfo, err := os.Stat(logFileName) + s.Require().NoError(err) + s.Require().NoError(os.Truncate(logFileName, fileInfo.Size()-1)) + + l, err = catalog.NewFileLogV2(logFileName) + s.Require().NoError(err) + + c, err = catalog.New( + clock, + l, + idGenerator, + catalog.DefaultMaxLogFileSize, + nil, + ) + s.Require().NoError(err) + + restoredR1, err := c.Get(r1.ID()) + s.Require().NoError(err) + + _, err = c.Get(r2.ID()) + s.Require().Error(err) + + s.Require().Equal(r1.ID(), restoredR1.ID()) + s.Require().Equal(r1.Dir(), restoredR1.Dir()) + s.Require().Equal(r1.NumberOfShards(), restoredR1.NumberOfShards()) + s.Require().Equal(r1.CreatedAt(), restoredR1.CreatedAt()) + s.Require().Equal(r1.UpdatedAt(), restoredR1.UpdatedAt()) + s.Require().Equal(r1.DeletedAt(), restoredR1.DeletedAt()) + s.Require().Equal(r1.Status(), restoredR1.Status()) +} + +// testIDGenerator generator UUID for test. +type testIDGenerator struct { + lastUUID uuid.UUID +} + +// Generate UUID. Implementation [catalog.IDGenerator]. +func (g *testIDGenerator) Generate() uuid.UUID { + g.lastUUID = uuid.New() + return g.lastUUID +} + +// last returns last UUID as string. +func (g *testIDGenerator) last() string { + return g.lastUUID.String() +} diff --git a/pp/go/storage/catalog/decoder.go b/pp/go/storage/catalog/decoder.go new file mode 100644 index 0000000000..d38b95cdd4 --- /dev/null +++ b/pp/go/storage/catalog/decoder.go @@ -0,0 +1,315 @@ +package catalog + +import ( + "encoding/binary" + "errors" + "fmt" + "hash" + "hash/crc32" + "io" + + "github.com/google/uuid" + + "github.com/prometheus/prometheus/pp/go/util/optional" +) + +const ( + // size of uint32. + sizeOfUint32 = 4 + // size of int64 or uint64. + sizeOf64 = 8 +) + +// +// DecoderV1 +// + +// DecoderV1 decodes [Record], version 1. +// +// Deprecated: For backward compatibility. +type DecoderV1 struct{} + +// DecodeFrom decode [Record] from [io.Reader]. +// +//revive:disable-next-line:cyclomatic this is decode. +func (DecoderV1) DecodeFrom(reader io.Reader, r *Record) (err error) { + var size uint64 + if err = binary.Read(reader, binary.LittleEndian, &size); err != nil { + return fmt.Errorf("read id size: %w", err) + } + + defer func() { + if err != nil && errors.Is(err, io.EOF) { + err = fmt.Errorf("%s: %w", err.Error(), io.ErrUnexpectedEOF) + } + }() + + buf := make([]byte, size) + if _, err = reader.Read(buf); err != nil { + return fmt.Errorf("read id: %w", err) + } + r.id = uuid.MustParse(string(buf)) + + if err = binary.Read(reader, binary.LittleEndian, &size); err != nil { + return fmt.Errorf("read dir size: %w", err) + } + + buf = make([]byte, size) + if _, err = reader.Read(buf); err != nil { + return fmt.Errorf("read dir: %w", err) + } + + if err = binary.Read(reader, binary.LittleEndian, &r.numberOfShards); err != nil { + return fmt.Errorf("read number of shards: %w", err) + } + + if err = binary.Read(reader, binary.LittleEndian, &r.createdAt); err != nil { + return fmt.Errorf("read created at: %w", err) + } + + if err = binary.Read(reader, binary.LittleEndian, &r.updatedAt); err != nil { + return fmt.Errorf("read updated at: %w", err) + } + + if err = binary.Read(reader, binary.LittleEndian, &r.deletedAt); err != nil { + return fmt.Errorf("read deleted at: %w", err) + } + + if err = binary.Read(reader, binary.LittleEndian, &r.status); err != nil { + return fmt.Errorf("read status: %w", err) + } + + return nil +} + +// +// DecoderV2 +// + +// DecoderV2 decodes [Record], version 2. +type DecoderV2 struct{} + +// DecodeFrom decode [Record] from [io.Reader]. +// +//revive:disable-next-line:cyclomatic this is decode. +//revive:disable-next-line:function-length long but this is decode. +func (DecoderV2) DecodeFrom(reader io.Reader, r *Record) (err error) { + var size uint8 + if err = binary.Read(reader, binary.LittleEndian, &size); err != nil { + return fmt.Errorf("read record size: %w", err) + } + + rReader := newReaderWithCounter(reader) + + defer func() { + if err != nil && errors.Is(err, io.EOF) || int(size) != rReader.BytesRead() { + if err == nil { + err = fmt.Errorf("bytes read: %d, bytes expected: %d", rReader.BytesRead(), size) + } + err = fmt.Errorf("%s: %w", err.Error(), io.ErrUnexpectedEOF) + } + }() + + if err = binary.Read(rReader, binary.LittleEndian, &r.id); err != nil { + return fmt.Errorf("read record id: %w", err) + } + + if err = binary.Read(rReader, binary.LittleEndian, &r.numberOfShards); err != nil { + return fmt.Errorf("read number of shards: %w", err) + } + + if err = binary.Read(rReader, binary.LittleEndian, &r.createdAt); err != nil { + return fmt.Errorf("read created at: %w", err) + } + + if err = binary.Read(rReader, binary.LittleEndian, &r.updatedAt); err != nil { + return fmt.Errorf("read updated at: %w", err) + } + + if err = binary.Read(rReader, binary.LittleEndian, &r.deletedAt); err != nil { + return fmt.Errorf("read deleted at: %w", err) + } + + if err = binary.Read(rReader, binary.LittleEndian, &r.corrupted); err != nil { + return fmt.Errorf("read currupted: %w", err) + } + + if err = binary.Read(rReader, binary.LittleEndian, &r.status); err != nil { + return fmt.Errorf("read status: %w", err) + } + + if err = decodeOptionalValue(rReader, binary.LittleEndian, &r.lastAppendedSegmentID); err != nil { + return fmt.Errorf("read last written segment id: %w", err) + } + + return nil +} + +// readerWithCounter reader with a counter of read bytes. +type readerWithCounter struct { + reader io.Reader + n int +} + +// newReaderWithCounter init new [readerWithCounter]. +func newReaderWithCounter(reader io.Reader) *readerWithCounter { + return &readerWithCounter{reader: reader} +} + +// Read reads up to len(p) bytes into p. +func (r *readerWithCounter) Read(p []byte) (n int, err error) { + n, err = r.reader.Read(p) + r.n += n + return n, err +} + +// BytesRead return a counter of read bytes. +func (r *readerWithCounter) BytesRead() int { + return r.n +} + +// decodeOptionalValue decode [optional.Optional[T]] from [io.Reader]. +func decodeOptionalValue[T any]( + reader io.Reader, + byteOrder binary.ByteOrder, + valueRef *optional.Optional[T], +) (err error) { + var nilIndicator uint8 + if err = binary.Read(reader, byteOrder, &nilIndicator); err != nil { + return err + } + if nilIndicator == 0 { + return nil + } + + var value T + if err = binary.Read(reader, byteOrder, &value); err != nil { + return err + } + valueRef.Set(value) + return nil +} + +// +// DecoderV3 +// + +// DecoderV3 decodes [Record], version 3. +type DecoderV3 struct { + offset int + size uint8 + buffer [RecordFrameSizeV3]byte + hasher hash.Hash32 +} + +// NewDecoderV3 init new [DecoderV3]. +func NewDecoderV3() *DecoderV3 { + return &DecoderV3{ + hasher: crc32.NewIEEE(), + } +} + +// DecodeFrom decode [Record] from [io.Reader]. +// +//revive:disable-next-line:cyclomatic this is decode. +//revive:disable-next-line:function-length long but this is decode. +func (d *DecoderV3) DecodeFrom(reader io.Reader, r *Record) (err error) { + d.reset() + + if err = d.readSize(reader); err != nil { + return err + } + + defer func() { + if err != nil && errors.Is(err, io.EOF) { + err = fmt.Errorf("%s: %w", err.Error(), io.ErrUnexpectedEOF) + } + }() + + if err = d.readRecord(reader); err != nil { + return err + } + + if err = d.validateCRC32(); err != nil { + return fmt.Errorf("read crc32: %w", err) + } + + targetOffset := d.offset + 16 //revive:disable-line:add-constant it's size of UUID + r.id = uuid.UUID(d.buffer[d.offset:targetOffset]) + d.offset = targetOffset + + r.numberOfShards = binary.LittleEndian.Uint16(d.buffer[d.offset:]) + d.offset += 2 //revive:disable-line:add-constant it's size of uint16 + + r.createdAt = int64(binary.LittleEndian.Uint64(d.buffer[d.offset:])) // #nosec G115 // no overflow + d.offset += sizeOf64 + + r.updatedAt = int64(binary.LittleEndian.Uint64(d.buffer[d.offset:])) // #nosec G115 // no overflow + d.offset += sizeOf64 + + r.deletedAt = int64(binary.LittleEndian.Uint64(d.buffer[d.offset:])) // #nosec G115 // no overflow + d.offset += sizeOf64 + + r.corrupted = d.buffer[d.offset] > 0 + d.offset++ + + r.status = Status(d.buffer[d.offset]) + d.offset++ + + r.numberOfSegments = binary.LittleEndian.Uint32(d.buffer[d.offset:]) + d.offset += sizeOfUint32 + + r.mint = int64(binary.LittleEndian.Uint64(d.buffer[d.offset:])) // #nosec G115 // no overflow + d.offset += sizeOf64 + + r.maxt = int64(binary.LittleEndian.Uint64(d.buffer[d.offset:])) // #nosec G115 // no overflow + d.offset += sizeOf64 + + return nil +} + +// reset state of decoder. +func (d *DecoderV3) reset() { + d.offset = 0 + d.size = 0 + d.hasher.Reset() +} + +// readSize read size of buffer from [io.Reader]. +func (d *DecoderV3) readSize(reader io.Reader) error { + if _, err := reader.Read(d.buffer[:1]); err != nil { + return fmt.Errorf("read record size: %w", err) + } + d.size = d.buffer[0] + + if int(d.size) != len(d.buffer) { + return fmt.Errorf("invalid size: %d", d.size) + } + + return nil +} + +// readRecord read [Record] from [io.Reader]. +func (d *DecoderV3) readRecord(reader io.Reader) error { + if _, err := reader.Read(d.buffer[:d.size]); err != nil { + return fmt.Errorf("read whole record: %w", err) + } + return nil +} + +// validateCRC32 validate [Record] on CRC32. +func (d *DecoderV3) validateCRC32() (err error) { + expectedCRC32Hash := binary.LittleEndian.Uint32(d.buffer[d.offset:]) + d.offset += sizeOfUint32 + + if _, err = d.hasher.Write(d.buffer[d.offset:]); err != nil { + return fmt.Errorf("write to crc32 hasher: %w", err) + } + + actualCRC32Hash := d.hasher.Sum32() + if expectedCRC32Hash != actualCRC32Hash { + return fmt.Errorf("invalid crc32: expected: %d, actual: %d", expectedCRC32Hash, actualCRC32Hash) + } + + return nil +} diff --git a/pp/go/storage/catalog/decoder_test.go b/pp/go/storage/catalog/decoder_test.go new file mode 100644 index 0000000000..82c994037b --- /dev/null +++ b/pp/go/storage/catalog/decoder_test.go @@ -0,0 +1,80 @@ +package catalog_test + +import ( + "bytes" + "testing" + + "github.com/google/uuid" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" + + "github.com/prometheus/prometheus/pp/go/storage/catalog" +) + +type DecoderSuite struct { + suite.Suite +} + +func TestDecoderSuite(t *testing.T) { + suite.Run(t, new(DecoderSuite)) +} + +func (s *DecoderSuite) TestDecoderV3Decode() { + buffer := bytes.NewBuffer(nil) + record := catalog.NewRecordWithDataV3(uuid.New(), 5, 25, 26, 27, true, catalog.StatusActive, 25, 2, 3) + + encoder := catalog.NewEncoderV3() + s.Require().NoError(encoder.EncodeTo(buffer, record)) + + decoder := catalog.NewDecoderV3() + decodedRecord := &catalog.Record{} + s.Require().NoError(decoder.DecodeFrom(buffer, decodedRecord)) + + s.Require().Equal(record.ID(), decodedRecord.ID()) + s.Require().Equal(record.NumberOfShards(), decodedRecord.NumberOfShards()) + s.Require().Equal(record.CreatedAt(), decodedRecord.CreatedAt()) + s.Require().Equal(record.UpdatedAt(), decodedRecord.UpdatedAt()) + s.Require().Equal(record.DeletedAt(), decodedRecord.DeletedAt()) + s.Require().Equal(record.Corrupted(), decodedRecord.Corrupted()) + s.Require().Equal(record.Status(), decodedRecord.Status()) + s.Require().Equal(record.NumberOfSegments(), decodedRecord.NumberOfSegments()) + s.Require().Equal(record.Maxt(), decodedRecord.Maxt()) + s.Require().Equal(record.Mint(), decodedRecord.Mint()) +} + +func BenchmarkDecodeV3(b *testing.B) { + buffer := bytes.NewBuffer(nil) + record := catalog.NewRecordWithDataV3(uuid.New(), 5, 25, 26, 27, true, catalog.StatusActive, 25, 2, 3) + var encoder catalog.Encoder + decodedRecord := &catalog.Record{} + b.StopTimer() + b.ResetTimer() + + for i := 0; i < b.N; i++ { + buffer.Reset() + encoder = catalog.NewEncoderV3() + require.NoError(b, encoder.EncodeTo(buffer, record)) + decoder := catalog.NewDecoderV3() + b.StartTimer() + require.NoError(b, decoder.DecodeFrom(buffer, decodedRecord)) + b.StopTimer() + } +} + +func BenchmarkDecodeV3_State(b *testing.B) { + buffer := bytes.NewBuffer(nil) + record := catalog.NewRecordWithDataV3(uuid.New(), 5, 25, 26, 27, true, catalog.StatusActive, 25, 2, 3) + encoder := catalog.NewEncoderV3() + decodedRecord := &catalog.Record{} + decoder := catalog.NewDecoderV3() + b.StopTimer() + b.ResetTimer() + + for i := 0; i < b.N; i++ { + buffer.Reset() + require.NoError(b, encoder.EncodeTo(buffer, record)) + b.StartTimer() + require.NoError(b, decoder.DecodeFrom(buffer, decodedRecord)) + b.StopTimer() + } +} diff --git a/pp/go/storage/catalog/encoder.go b/pp/go/storage/catalog/encoder.go new file mode 100644 index 0000000000..0fc4a73dbc --- /dev/null +++ b/pp/go/storage/catalog/encoder.go @@ -0,0 +1,249 @@ +package catalog + +import ( + "bytes" + "encoding/binary" + "fmt" + "hash" + "hash/crc32" + "io" + + "github.com/prometheus/prometheus/pp/go/util/optional" +) + +const ( + // RecordStructMaxSizeV2 max size of [Record] for [EncoderV2]. + RecordStructMaxSizeV2 = 50 + // RecordFrameSizeV3 size of frame [Record] for [EncoderV3]. + RecordFrameSizeV3 = 68 +) + +// +// EncoderV1 +// + +// EncoderV1 encodes [Record], version 1. +// +// Deprecated. +type EncoderV1 struct{} + +// EncodeTo encode [Record] to [io.Writer]. +func (EncoderV1) EncodeTo(writer io.Writer, r *Record) (err error) { + if err = encodeString(writer, r.id.String()); err != nil { + return fmt.Errorf("v1: encode id: %w", err) + } + + if err = encodeString(writer, r.id.String()); err != nil { + return fmt.Errorf("v1: encode dir: %w", err) + } + + if err = binary.Write(writer, binary.LittleEndian, &r.numberOfShards); err != nil { + return fmt.Errorf("v1: write number of shards: %w", err) + } + + if err = binary.Write(writer, binary.LittleEndian, &r.createdAt); err != nil { + return fmt.Errorf("v1: write created at: %w", err) + } + + if err = binary.Write(writer, binary.LittleEndian, &r.updatedAt); err != nil { + return fmt.Errorf("v1: write updated at: %w", err) + } + + if err = binary.Write(writer, binary.LittleEndian, &r.deletedAt); err != nil { + return fmt.Errorf("v1: write deleted at: %w", err) + } + + if err = binary.Write(writer, binary.LittleEndian, &r.status); err != nil { + return fmt.Errorf("v1: write status: %w", err) + } + + return nil +} + +// encodeString encode string to [io.Writer]. +func encodeString(writer io.Writer, value string) (err error) { + if err = binary.Write(writer, binary.LittleEndian, uint64(len(value))); err != nil { + return fmt.Errorf("write string length: %w", err) + } + + if _, err = writer.Write([]byte(value)); err != nil { + return fmt.Errorf("write string: %w", err) + } + + return nil +} + +// +// EncoderV2 +// + +// EncoderV2 encodes [Record], version 2. +type EncoderV2 struct { + buffer *bytes.Buffer +} + +// NewEncoderV2 init new [EncoderV2]. +func NewEncoderV2() *EncoderV2 { + return &EncoderV2{ + buffer: bytes.NewBuffer(make([]byte, 0, RecordStructMaxSizeV2)), + } +} + +// EncodeTo encode [Record] to [io.Writer]. +// +//revive:disable-next-line:cyclomatic this is encode. +//revive:disable-next-line:function-length long but this is encode. +func (e *EncoderV2) EncodeTo(writer io.Writer, r *Record) (err error) { + e.buffer.Reset() + + if err = binary.Write(e.buffer, binary.LittleEndian, uint8(0)); err != nil { + return fmt.Errorf("v2: encode size filler: %w", err) + } + + if err = binary.Write(e.buffer, binary.LittleEndian, r.id); err != nil { + return fmt.Errorf("v2: encode id: %w", err) + } + + if err = binary.Write(e.buffer, binary.LittleEndian, &r.numberOfShards); err != nil { + return fmt.Errorf("v2: write number of shards: %w", err) + } + + if err = binary.Write(e.buffer, binary.LittleEndian, &r.createdAt); err != nil { + return fmt.Errorf("v2: write created at: %w", err) + } + + if err = binary.Write(e.buffer, binary.LittleEndian, &r.updatedAt); err != nil { + return fmt.Errorf("v2: write updated at: %w", err) + } + + if err = binary.Write(e.buffer, binary.LittleEndian, &r.deletedAt); err != nil { + return fmt.Errorf("v2: write deleted at: %w", err) + } + + if err = binary.Write(e.buffer, binary.LittleEndian, &r.corrupted); err != nil { + return fmt.Errorf("v2: write corrupted: %w", err) + } + + if err = binary.Write(e.buffer, binary.LittleEndian, &r.status); err != nil { + return fmt.Errorf("v2: write status: %w", err) + } + + if err = encodeOptionalValue(e.buffer, binary.LittleEndian, r.lastAppendedSegmentID); err != nil { + return fmt.Errorf("v2: write last written segment id: %w", err) + } + + e.buffer.Bytes()[0] = uint8(len(e.buffer.Bytes()) - 1) // #nosec G115 // no overflow + + if _, err = e.buffer.WriteTo(writer); err != nil { + return fmt.Errorf("v2: write record: %w", err) + } + + return nil +} + +// encodeOptionalValue encode [optional.Optional[T]] to [io.Writer]. +func encodeOptionalValue[T any](writer io.Writer, byteOrder binary.ByteOrder, value optional.Optional[T]) (err error) { + var nilIndicator uint8 + if value.IsNil() { + return binary.Write(writer, byteOrder, nilIndicator) + } + + nilIndicator = 1 + if err = binary.Write(writer, byteOrder, nilIndicator); err != nil { + return err + } + + return binary.Write(writer, byteOrder, value.Value()) +} + +// +// EncoderV3 +// + +// EncoderV3 encodes [Record], version 3. +type EncoderV3 struct { + buffer *bytes.Buffer + crc32Hasher hash.Hash32 +} + +// NewEncoderV3 init new [EncoderV3]. +func NewEncoderV3() *EncoderV3 { + return &EncoderV3{ + buffer: bytes.NewBuffer(make([]byte, 0, RecordFrameSizeV3+1)), // +1 is for size byte + crc32Hasher: crc32.NewIEEE(), + } +} + +// EncodeTo encode [Record] to [io.Writer]. +// +//revive:disable-next-line:cyclomatic this is encode. +//revive:disable-next-line:function-length long but this is encode. +func (e *EncoderV3) EncodeTo(writer io.Writer, r *Record) (err error) { + e.buffer.Reset() + + if err = binary.Write(e.buffer, binary.LittleEndian, uint8(0)); err != nil { + return fmt.Errorf("v3: encode size filler: %w", err) + } + + if err = binary.Write(e.buffer, binary.LittleEndian, uint32(0)); err != nil { + return fmt.Errorf("v3: encode crc32 filler: %w", err) + } + + if err = binary.Write(e.buffer, binary.LittleEndian, r.id); err != nil { + return fmt.Errorf("v3: encode id: %w", err) + } + + if err = binary.Write(e.buffer, binary.LittleEndian, &r.numberOfShards); err != nil { + return fmt.Errorf("v3: write number of shards: %w", err) + } + + if err = binary.Write(e.buffer, binary.LittleEndian, &r.createdAt); err != nil { + return fmt.Errorf("v3: write created at: %w", err) + } + + if err = binary.Write(e.buffer, binary.LittleEndian, &r.updatedAt); err != nil { + return fmt.Errorf("v3: write updated at: %w", err) + } + + if err = binary.Write(e.buffer, binary.LittleEndian, &r.deletedAt); err != nil { + return fmt.Errorf("v3: write deleted at: %w", err) + } + + if err = binary.Write(e.buffer, binary.LittleEndian, &r.corrupted); err != nil { + return fmt.Errorf("v3: write corrupted: %w", err) + } + + if err = binary.Write(e.buffer, binary.LittleEndian, &r.status); err != nil { + return fmt.Errorf("v3: write status: %w", err) + } + + if err = binary.Write(e.buffer, binary.LittleEndian, &r.numberOfSegments); err != nil { + return fmt.Errorf("v3: write number of segments: %w", err) + } + + if err = binary.Write(e.buffer, binary.LittleEndian, &r.mint); err != nil { + return fmt.Errorf("v3: write min timestamp: %w", err) + } + + if err = binary.Write(e.buffer, binary.LittleEndian, &r.maxt); err != nil { + return fmt.Errorf("v3: write max timestamp: %w", err) + } + + e.buffer.Bytes()[0] = uint8(len(e.buffer.Bytes()) - 1) // #nosec G115 // no overflow + + e.crc32Hasher.Reset() + _, err = e.crc32Hasher.Write(e.buffer.Bytes()[5:]) + if err != nil { + return fmt.Errorf("v3: write hash: %w", err) + } + + var binaryCRC32 [4]byte + binary.LittleEndian.PutUint32(binaryCRC32[:], e.crc32Hasher.Sum32()) + copy(e.buffer.Bytes()[1:5], binaryCRC32[:]) + + if _, err = e.buffer.WriteTo(writer); err != nil { + return fmt.Errorf("v3: write record: %w", err) + } + + return nil +} diff --git a/pp/go/storage/catalog/gc.go b/pp/go/storage/catalog/gc.go new file mode 100644 index 0000000000..a947d237c1 --- /dev/null +++ b/pp/go/storage/catalog/gc.go @@ -0,0 +1,173 @@ +package catalog + +import ( + "context" + "errors" + "os" + "path/filepath" + "time" + + "github.com/jonboulle/clockwork" + "github.com/prometheus/prometheus/pp/go/logger" +) + +// +// HeadsCatalog +// + +// HeadsCatalog of current head records. +type HeadsCatalog interface { + // Delete record by ID. + Delete(id string) error + + // List returns slice of records with filter and sort. + List(filterFn func(record *Record) bool, sortLess func(lhs, rhs *Record) bool) []*Record +} + +// +// Notifiable +// + +// Notifiable notifies the recipient that it is ready to work. +type Notifiable interface { + // ReadyChan notifies the recipient that it is ready to work. + ReadyChan() <-chan struct{} +} + +// +// RemovedHeadNotifier +// + +// RemovedHeadNotifier notifies that the [Head] has been removed. +type RemovedHeadNotifier interface { + // Chan returns channel with notifications. + Chan() <-chan struct{} +} + +// +// GC +// + +// GC garbage collector for old [Head]. +type GC struct { + dataDir string + catalog HeadsCatalog + clock clockwork.Clock + readyNotifiable Notifiable + removedHeadNotifier RemovedHeadNotifier + maxRetentionPeriod time.Duration + stop chan struct{} + stopped chan struct{} +} + +// NewGC init new [GC]. +func NewGC( + dataDir string, + catalog HeadsCatalog, + clock clockwork.Clock, + readyNotifiable Notifiable, + removedHeadNotifier RemovedHeadNotifier, + maxRetentionPeriod time.Duration, +) *GC { + return &GC{ + dataDir: dataDir, + catalog: catalog, + clock: clock, + readyNotifiable: readyNotifiable, + removedHeadNotifier: removedHeadNotifier, + maxRetentionPeriod: maxRetentionPeriod, + stop: make(chan struct{}), + stopped: make(chan struct{}), + } +} + +// Iterate over the [Catalog] list and remove old [Head]s. +func (gc *GC) Iterate() { + logger.Debugf("catalog gc iteration: head started") + defer logger.Debugf("catalog gc iteration: head ended") + + records := gc.catalog.List( + gc.possibleRemoval, + func(lhs, rhs *Record) bool { + return lhs.CreatedAt() < rhs.CreatedAt() + }, + ) + + for _, record := range records { + if record.DeletedAt() != 0 { + continue + } + + logger.Debugf("catalog gc iteration: head: %s", record.ID()) + if record.ReferenceCount() > 0 { + return + } + + if record.Corrupted() { + logger.Debugf("catalog gc iteration: head: %s: %s", record.ID(), "corrupted") + continue + } + + if err := os.RemoveAll(filepath.Join(gc.dataDir, record.Dir())); err != nil { + logger.Errorf("failed to remote head dir: %w", err) + return + } + + if err := gc.catalog.Delete(record.ID()); err != nil { + logger.Errorf("failed to delete head record: %w", err) + return + } + + logger.Debugf("catalog gc iteration: head: %s: %s", record.ID(), "removed") + } +} + +// Run main loop [GC]. +func (gc *GC) Run(ctx context.Context) error { + defer close(gc.stopped) + + for { + select { + case <-ctx.Done(): + return ctx.Err() + case <-gc.readyNotifiable.ReadyChan(): + case <-gc.stop: + return errors.New("stopped") + } + + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(time.Minute): + gc.Iterate() + case <-gc.removedHeadNotifier.Chan(): + gc.Iterate() + case <-gc.stop: + return errors.New("stopped") + } + } +} + +// Stop the garbage collection loop. +func (gc *GC) Stop() { + close(gc.stop) + <-gc.stopped +} + +// possibleRemoval a filter to remove unwanted wals. +func (gc *GC) possibleRemoval(record *Record) bool { + if record.DeletedAt() != 0 { + return false + } + + // the head is outdated and data on it is no longer required + if gc.clock.Since(time.UnixMilli(record.CreatedAt())) >= gc.maxRetentionPeriod { + return true + } + + if record.Status() != StatusPersisted { + return false + } + + return true +} diff --git a/pp/go/storage/catalog/log.go b/pp/go/storage/catalog/log.go new file mode 100644 index 0000000000..549da10fbb --- /dev/null +++ b/pp/go/storage/catalog/log.go @@ -0,0 +1,296 @@ +package catalog + +import ( + "encoding/binary" + "errors" + "fmt" + "io" + "os" + "strings" + + "github.com/prometheus/prometheus/pp/go/logger" +) + +const ( + // LogFileVersionV1 version 1 of log-file. + LogFileVersionV1 uint64 = 1 + // LogFileVersionV2 version 2 of log-file. + LogFileVersionV2 uint64 = 2 + // LogFileVersionV3 version 3 of log-file. + LogFileVersionV3 uint64 = 3 + + // logFilePerm log-file permissions. + logFilePerm = 0o600 +) + +// +// Encoder +// + +// Encoder encodes [Record]. +type Encoder interface { + // EncodeTo encode [Record] to [io.Writer]. + EncodeTo(writer io.Writer, r *Record) error +} + +// +// Decoder +// + +// Decoder decodes [Record]. +type Decoder interface { + // DecodeFrom decode [Record] from [io.Reader]. + DecodeFrom(reader io.Reader, r *Record) error +} + +// +// FileLog +// + +// FileLog head-log file, contains [Record]s of heads. +type FileLog struct { + version uint64 + file *FileHandler + filePath string + encoder Encoder + decoder Decoder +} + +// NewFileLogV1 init new [FileLog] with [EncoderV1], [DecoderV1], version 1. +// +// Deprecated. +func NewFileLogV1(fileName string) (*FileLog, error) { + file, err := NewFileHandler(fileName) + if err != nil { + return nil, err + } + + fl := &FileLog{ + version: LogFileVersionV1, + file: file, + encoder: EncoderV1{}, + decoder: DecoderV1{}, + } + + defer func() { + if err != nil { + _ = fl.Close() + } + }() + + if file.Size() == 0 { + if err = binary.Write(file, binary.LittleEndian, fl.version); err != nil { + return nil, errors.Join(fmt.Errorf("failed to write log file version: %w", err), fl.Close()) + } + } else { + var version uint64 + if err = binary.Read(file, binary.LittleEndian, &version); err != nil { + return nil, errors.Join(fmt.Errorf("failed to read log file version: %w", err), fl.Close()) + } + if version != fl.version { + return nil, errors.Join(fmt.Errorf("invalid log file version: %d", version), fl.Close()) + } + } + + return fl, nil +} + +// NewFileLogV2 init new [FileLog] with [EncoderV2], [DecoderV2], version 2. +func NewFileLogV2(filePath string) (*FileLog, error) { + return NewFileLog(filePath, LogFileVersionV2) +} + +// NewFileLogV3 init new [FileLog] with [EncoderV3], [DecoderV3], version 3. +func NewFileLogV3(filePath string) (*FileLog, error) { + return NewFileLog(filePath, LogFileVersionV3) +} + +// NewFileLog init new [FileLog] with migrate to target version encoder and decoder. +func NewFileLog(filePath string, targetVersion uint64) (*FileLog, error) { + sourceFilePath := filePath + fl, err := openFileLog(filePath, sourceFilePath, targetVersion) + if err == nil { + return fl, nil + } + + if !errors.Is(err, ErrUnreadableLogFile) { + return nil, err + } + + logger.Errorf("unreadable log file: filepath: %s, error: %v", sourceFilePath, err) + + sourceFilePath = fmt.Sprintf("%s.compacted", filePath) + fl, err = openFileLog(filePath, sourceFilePath, targetVersion) + if err == nil { + return fl, nil + } + + if !errors.Is(err, ErrUnreadableLogFile) { + return nil, err + } + + logger.Errorf("unreadable log file: filepath: %s, error: %v", sourceFilePath, err) + + return newFileLogByVersion(filePath, targetVersion) +} + +// openFileLog open [FileLog] with migrate to version. +func openFileLog(filePath, sourceFilePath string, version uint64) (*FileLog, error) { + file, encoder, decoder, err := migrate(filePath, sourceFilePath, version) + if err != nil { + return nil, err + } + + return &FileLog{ + version: version, + file: file, + filePath: filePath, + encoder: encoder, + decoder: decoder, + }, nil +} + +// newFileLogByVersion init new [FileLog] by version. +func newFileLogByVersion(filePath string, version uint64) (*FileLog, error) { + encoder, decoder, err := codecsByVersion(version) + if err != nil { + return nil, fmt.Errorf("create encoder/decoder: %w", err) + } + + file, err := createFileHandlerByVersion(filePath, version) + if err != nil { + return nil, fmt.Errorf("create file handler: %w", err) + } + + return &FileLog{ + version: version, + file: file, + filePath: filePath, + encoder: encoder, + decoder: decoder, + }, nil +} + +// Close closes the [FileHandler], rendering it unusable for I/O. +func (fl *FileLog) Close() error { + return fl.file.Close() +} + +// ReWrite rewrite [FileLog] with [Record]s. +func (fl *FileLog) ReWrite(records ...*Record) error { + oldFile := fl.file + swapFilePath := fmt.Sprintf("%s.compacted", strings.TrimSuffix(fl.filePath, ".compacted")) + newFile, err := writeSwapAndSwitchAtFilePath(fl.filePath, swapFilePath, fl.version, fl.encoder, records...) + if err != nil { + return fmt.Errorf("write log file: %w", err) + } + + fl.file = newFile + if err = oldFile.Close(); err != nil { + logger.Errorf("failed to close old file: %v", err) + } + + return nil +} + +// Read [Record] from [FileLog]. +func (fl *FileLog) Read(r *Record) error { + return fl.decoder.DecodeFrom(fl.file, r) +} + +// Size return current size of [FileHandler]. +func (fl *FileLog) Size() int { + return fl.file.Size() +} + +// Write [Record] to [FileLog]. +func (fl *FileLog) Write(r *Record) error { + return fl.encoder.EncodeTo(fl.file, r) +} + +func writeSwapAndSwitchAtFilePath( + targetFilePath, swapFilePath string, + version uint64, + encoder Encoder, + records ...*Record, +) (*FileHandler, error) { + swapFile, err := createSwapFile(swapFilePath, version, encoder, records...) + if err != nil { + return nil, fmt.Errorf("create swap file: %w", err) + } + + defer func() { + if err != nil { + err = errors.Join(err, swapFile.Close(), os.RemoveAll(swapFilePath)) + } + }() + + if err = os.Rename(swapFilePath, targetFilePath); err != nil { + return nil, fmt.Errorf("rename swap file: %w", err) + } + + return swapFile, nil +} + +// creates swap file, writes records & sets read offset at first record. +func createSwapFile(fileName string, version uint64, encoder Encoder, records ...*Record) (*FileHandler, error) { + swapFile, err := NewFileHandlerWithOpts(fileName, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, logFilePerm) + if err != nil { + return nil, fmt.Errorf("new file handler: %w", err) + } + + defer func() { + if err != nil { + err = errors.Join(err, swapFile.Close(), os.RemoveAll(fileName)) + } + }() + + offset, err := writeLogFileVersion(swapFile, version) + if err != nil { + return nil, fmt.Errorf("write log file version: %w", err) + } + + for _, record := range records { + if err = encoder.EncodeTo(swapFile, record); err != nil { + return nil, fmt.Errorf("encode record: %w", err) + } + } + + if err = swapFile.Sync(); err != nil { + return nil, fmt.Errorf("sync swap file: %w", err) + } + + swapFile.SetReadOffset(int64(offset)) + + return swapFile, nil +} + +// readLogFileVersion reads log file version respecting on disk version size. +func readLogFileVersion(reader io.Reader) (version uint64, err error) { + var v [8]byte + _, err = reader.Read(v[:1]) + if err != nil { + return 0, err + } + + version = binary.LittleEndian.Uint64(v[:8]) + if version <= LogFileVersionV2 { + // skip next 7 bytes + _, err = reader.Read(v[1:8]) + return version, err + } + + return version, nil +} + +// writeLogFileVersion writes log file version respecting on disk version size. +func writeLogFileVersion(writer io.Writer, version uint64) (int, error) { + var v [8]byte + binary.LittleEndian.PutUint64(v[:8], version) + numberOfBytesToWrite := len(v) + if version >= LogFileVersionV3 { + numberOfBytesToWrite = 1 + } + bytesWritten, err := writer.Write(v[:numberOfBytesToWrite]) + return bytesWritten, err +} diff --git a/pp/go/storage/catalog/log_file_handler.go b/pp/go/storage/catalog/log_file_handler.go new file mode 100644 index 0000000000..d111fb5130 --- /dev/null +++ b/pp/go/storage/catalog/log_file_handler.go @@ -0,0 +1,102 @@ +package catalog + +import ( + "errors" + "fmt" + "io" + "os" +) + +// +// FileHandler +// + +// FileHandler handler for work with [os.File]. +type FileHandler struct { + file *os.File + size int + readOffset int64 + writeOffset int64 +} + +// NewFileHandler init new [FileHandler]. +func NewFileHandler(filePath string) (*FileHandler, error) { + return NewFileHandlerWithOpts(filePath, os.O_CREATE|os.O_RDWR, logFilePerm) +} + +// NewFileHandlerWithOpts init new [FileHandler] with opts. +func NewFileHandlerWithOpts(filePath string, flag int, perm os.FileMode) (*FileHandler, error) { + file, err := os.OpenFile(filePath, flag, perm) //#nosec G304 // it's meant to be that way + if err != nil { + return nil, fmt.Errorf("open file: %w", err) + } + defer func() { + if err != nil { + err = errors.Join(err, file.Close()) + } + }() + + fileInfo, err := file.Stat() + if err != nil { + return nil, fmt.Errorf("read file info: %w", err) + } + + return &FileHandler{ + file: file, + size: int(fileInfo.Size()), + writeOffset: fileInfo.Size(), + }, nil +} + +// Close closes the [os.File], rendering it unusable for I/O. +func (fh *FileHandler) Close() error { + return fh.file.Close() +} + +// FileName returns the current name of the file. +func (fh *FileHandler) FileName() string { + return fh.file.Name() +} + +// Read reads len(b) bytes from the [os.File]. +func (fh *FileHandler) Read(p []byte) (n int, err error) { + n, err = fh.file.ReadAt(p, fh.readOffset) + if err != nil { + if !errors.Is(err, io.EOF) { + return 0, fmt.Errorf("read file: %w", err) + } + } + fh.readOffset += int64(n) + return n, err +} + +// SetReadOffset set offset for read file. +func (fh *FileHandler) SetReadOffset(offset int64) { + fh.readOffset = offset +} + +// Size returns current size of file. +func (fh *FileHandler) Size() int { + return fh.size +} + +// Sync commits the current contents of the file to stable storage. +func (fh *FileHandler) Sync() error { + return fh.file.Sync() +} + +// Write writes len(b) bytes to the [os.File]. +func (fh *FileHandler) Write(p []byte) (n int, err error) { + n, err = fh.file.WriteAt(p, fh.writeOffset) + if err != nil { + return 0, fmt.Errorf("write file: %w", err) + } + + if err = fh.file.Sync(); err != nil { + return 0, fmt.Errorf("sync file: %w", err) + } + + fh.size += n + fh.writeOffset += int64(n) + return n, nil +} diff --git a/pp/go/storage/catalog/log_migrate.go b/pp/go/storage/catalog/log_migrate.go new file mode 100644 index 0000000000..b9e8a21763 --- /dev/null +++ b/pp/go/storage/catalog/log_migrate.go @@ -0,0 +1,286 @@ +package catalog + +import ( + "errors" + "fmt" + "io" + "os" + + "github.com/prometheus/prometheus/pp/go/logger" + "github.com/prometheus/prometheus/pp/go/util/optional" +) + +var ( + // ErrUnsupportedVersion unsupported version error. + ErrUnsupportedVersion = errors.New("unsupported version") + // ErrUnreadableLogFile unreadable log file error. + ErrUnreadableLogFile = errors.New("unreadable log file") +) + +// migrate source file to target file on target version. +// +//revive:disable-next-line:function-result-limit there is no point in packing it into a structure. +func migrate( + targetFilePath, sourceFilePath string, + targetVersion uint64, +) (_ *FileHandler, _ Encoder, _ Decoder, _ error) { + sourceFile, sourceVersion, err := loadFile(sourceFilePath) + if err != nil { + return nil, nil, nil, err + } + + sourceEncoder, sourceDecoder, err := codecsByVersion(sourceVersion) + if err != nil { + return nil, nil, nil, errors.Join(ErrUnreadableLogFile, sourceFile.Close()) + } + + if sourceVersion != targetVersion { + return migrateTo(sourceFile, sourceDecoder, targetFilePath, sourceFilePath, targetVersion, sourceVersion) + } + + if sourceFilePath == targetFilePath { + return sourceFile, sourceEncoder, sourceDecoder, nil + } + + err = os.Rename(sourceFilePath, targetFilePath) + if err != nil { + return nil, nil, nil, errors.Join(err, sourceFile.Close()) + } + + return sourceFile, sourceEncoder, sourceDecoder, nil +} + +// migrateTo source file to target file on target version. +// +//revive:disable-next-line:function-result-limit there is no point in packing it into a structure. +func migrateTo( + sourceFile *FileHandler, + sourceDecoder Decoder, + targetFilePath, sourceFilePath string, + targetVersion, sourceVersion uint64, +) (_ *FileHandler, _ Encoder, _ Decoder, _ error) { + targetEncoder, targetDecoder, err := codecsByVersion(targetVersion) + if err != nil { + return nil, nil, nil, errors.Join(err, sourceFile.Close()) + } + + records := make([]*Record, 0, 10) //revive:disable-line:add-constant it's average value of records + for { + record := &Record{} + if err = sourceDecoder.DecodeFrom(sourceFile, record); err != nil { + if errors.Is(err, io.EOF) { + break + } + logger.Errorf("failed to decode record: %v", err) + break + } + records = append(records, record) + } + + migration := getMigration(sourceVersion, targetVersion) + migratedRecords := make([]*Record, 0, len(records)) + for _, record := range records { + migratedRecords = append(migratedRecords, migration.Migrate(record)) + } + + swapFilePath := fmt.Sprintf("%s.swap", sourceFilePath) + targetFile, err := writeSwapAndSwitchAtFilePath( + targetFilePath, + swapFilePath, + targetVersion, + targetEncoder, + migratedRecords..., + ) + if err != nil { + return nil, nil, nil, errors.Join(err, sourceFile.Close()) + } + + if err = sourceFile.Close(); err != nil { + logger.Errorf("failed to close file: %v", err) + } + + return targetFile, targetEncoder, targetDecoder, nil +} + +// loadFile load [FileHandler] from file. +func loadFile(filePath string) (_ *FileHandler, _ uint64, err error) { + fileInfo, err := os.Stat(filePath) + if err != nil { + if os.IsNotExist(err) { + err = ErrUnreadableLogFile + } + return nil, 0, err + } + + if fileInfo.Size() == 0 { + return nil, 0, ErrUnreadableLogFile + } + + fh, err := NewFileHandlerWithOpts(filePath, os.O_RDWR, logFilePerm) + if err != nil { + return nil, 0, err + } + + version, err := readLogFileVersion(fh) + if err != nil { + return nil, 0, errors.Join(fmt.Errorf("read log file version: %w", err), fh.Close()) + } + + return fh, version, nil +} + +// createFileHandlerByVersion create [FileHandler] by version. +func createFileHandlerByVersion(filePath string, version uint64) (*FileHandler, error) { + fh, err := NewFileHandlerWithOpts(filePath, os.O_CREATE|os.O_RDWR|os.O_TRUNC, logFilePerm) + if err != nil { + return nil, err + } + + offset, err := writeLogFileVersion(fh, version) + if err != nil { + return nil, errors.Join(err, fh.Close()) + } + + fh.SetReadOffset(int64(offset)) + + return fh, nil +} + +// codecsByVersion select codec by version. +func codecsByVersion(version uint64) (e Encoder, d Decoder, err error) { + switch version { + case LogFileVersionV1: + return EncoderV1{}, DecoderV1{}, nil + case LogFileVersionV2: + return NewEncoderV2(), DecoderV2{}, nil + case LogFileVersionV3: + return NewEncoderV3(), NewDecoderV3(), nil + default: + return nil, nil, ErrUnsupportedVersion + } +} + +// +// Migration +// + +// Migration migrates record interface. +type Migration interface { + Migrate(record *Record) *Record +} + +// MigrationFunc is Migration interface function wrapper. +type MigrationFunc func(record *Record) *Record + +// Migrate reacord version. +func (fn MigrationFunc) Migrate(record *Record) *Record { + return fn(record) +} + +// +// MigrationV2 +// + +// MigrationV2 migrates record from v1 to v2 and vice versa. +type MigrationV2 struct{} + +// Up migrates from v1 to v2. +func (MigrationV2) Up(record *Record) *Record { + if record.status == StatusCorrupted { + record.corrupted = true + record.status = StatusRotated + } + return record +} + +// Down migrates from v2 to v1. +func (MigrationV2) Down(record *Record) *Record { + if record.status == StatusRotated && record.corrupted { + record.status = StatusCorrupted + } + return record +} + +// +// MigrationV3 +// + +// MigrationV3 migrates record from v2 to v3 and vice versa. +type MigrationV3 struct{} + +// Up migrates from v2 to v3. +func (MigrationV3) Up(record *Record) *Record { + record.numberOfSegments = 0 + if !record.lastAppendedSegmentID.IsNil() { + record.numberOfSegments = record.lastAppendedSegmentID.Value() + 1 + } + return record +} + +// Down migrates from v3 to v2. +func (MigrationV3) Down(record *Record) *Record { + if record.numberOfSegments > 0 { + record.lastAppendedSegmentID.Set(record.numberOfSegments - 1) + } else { + record.lastAppendedSegmentID = optional.WithRawValue[uint32](nil) + } + return record +} + +// +// ChainedMigration +// + +// ChainedMigration combines migrations to provide multiple migrations. +type ChainedMigration struct { + migrations []Migration +} + +// NewChainedMigration constructor. +func NewChainedMigration(migrations ...Migration) *ChainedMigration { + return &ChainedMigration{migrations: migrations} +} + +// Migrate is an Migration interface implementation. +func (c *ChainedMigration) Migrate(record *Record) *Record { + for _, migration := range c.migrations { + record = migration.Migrate(record) + } + return record +} + +// getMigration create [Migration] from version to version. +func getMigration(from, to uint64) Migration { + up := true + if from > to { + up = false + from, to = to, from + } + + var migrations []Migration + for i := from + 1; i <= to; i++ { + migrations = append(migrations, migrationByVersion(i, up)) + } + + return NewChainedMigration(migrations...) +} + +// migrationByVersion create [Migration] by version. +// +//revive:disable-next-line:flag-parameter this is a flag, but it's more convenient this way +func migrationByVersion(version uint64, up bool) Migration { + switch version { + case LogFileVersionV2: + if up { + return MigrationFunc(MigrationV2{}.Up) + } + return MigrationFunc(MigrationV2{}.Down) + case LogFileVersionV3: + if up { + return MigrationFunc(MigrationV3{}.Up) + } + return MigrationFunc(MigrationV3{}.Down) + default: + panic(fmt.Sprintf("invalid version: %d", version)) + } +} diff --git a/pp/go/storage/catalog/log_test.go b/pp/go/storage/catalog/log_test.go new file mode 100644 index 0000000000..2a23f6ef61 --- /dev/null +++ b/pp/go/storage/catalog/log_test.go @@ -0,0 +1,102 @@ +package catalog_test + +import ( + "encoding/hex" + "os" + "path/filepath" + "testing" + + "github.com/google/uuid" + "github.com/stretchr/testify/suite" + + "github.com/prometheus/prometheus/pp/go/storage/catalog" +) + +type FileLogTestSuite struct { + suite.Suite + records []*catalog.Record +} + +func TestFileLog(t *testing.T) { + suite.Run(t, new(FileLogTestSuite)) +} + +func (s *FileLogTestSuite) SetupSuite() { + id1 := uuid.MustParse("fe52d991-fe22-41d9-9642-7e4d66d81a0c") + var lastWrittenSegmentIDForID1 uint32 = 5 + id2 := uuid.MustParse("2d89cb33-9daa-4aea-9855-f844add5e3e4") + id3 := uuid.MustParse("ec0c2898-9c42-449c-9a58-74bea665481c") + + s.records = []*catalog.Record{ + catalog.NewRecordWithData(id1, 1, 4, 5, 0, false, 0, catalog.StatusNew, &lastWrittenSegmentIDForID1), + catalog.NewRecordWithData(id2, 2, 34, 420, 0, false, 0, catalog.StatusCorrupted, nil), + catalog.NewRecordWithData(id3, 3, 25, 256, 0, false, 0, catalog.StatusPersisted, nil), + } +} + +func (s *FileLogTestSuite) TestMigrateV1ToV2() { + logFileV1Name := filepath.Join(s.T().TempDir(), "v1.log") + logFileV1, err := catalog.NewFileLogV1(logFileV1Name) + s.Require().NoError(err) + + for _, record := range s.records { + s.Require().NoError(logFileV1.Write(record)) + } + s.Require().NoError(logFileV1.Close()) + + fileContentIsEqual(s, logFileV1Name, "testdata/headv1.log") + + logFile, err := catalog.NewFileLogV2(logFileV1Name) + s.Require().NoError(err) + s.Require().NoError(logFile.Close()) + + fileContentIsEqual(s, logFileV1Name, "testdata/headv2.log") +} + +func (s *FileLogTestSuite) TestMigrateV2ToV3() { + tmpDir := filepath.Join(s.T().TempDir(), "logtest") + s.Require().NoError(os.CopyFS(tmpDir, os.DirFS("testdata"))) + + logFilePath := filepath.Join(tmpDir, "headv2.log") + logFile, err := catalog.NewFileLogV3(logFilePath) + s.Require().NoError(err) + s.Require().NoError(logFile.Close()) + + fileContentIsEqual(s, logFilePath, "testdata/headv3.log") +} + +func (s *FileLogTestSuite) TestMigrateV3ToV2() { + tmpDir := filepath.Join(s.T().TempDir(), "logtest") + s.Require().NoError(os.CopyFS(tmpDir, os.DirFS("testdata"))) + + logFilePath := filepath.Join(tmpDir, "headv3.log") + logFile, err := catalog.NewFileLogV2(logFilePath) + s.Require().NoError(err) + s.Require().NoError(logFile.Close()) + + fileContentIsEqual(s, logFilePath, "testdata/headv2.log") +} + +func (s *FileLogTestSuite) TestMigrateV1ToV3() { + tmpDir := filepath.Join(s.T().TempDir(), "logtest") + s.Require().NoError(os.CopyFS(tmpDir, os.DirFS("testdata"))) + + logFilePath := filepath.Join(tmpDir, "headv1.log") + logFile, err := catalog.NewFileLogV3(logFilePath) + s.Require().NoError(err) + s.Require().NoError(logFile.Close()) + + fileContentIsEqual(s, logFilePath, "testdata/headv3.log") +} + +func fileContentIsEqual(s *FileLogTestSuite, filePath1, filePath2 string) { + data1, err := os.ReadFile(filePath1) + s.Require().NoError(err) + s.T().Log(hex.EncodeToString(data1)) + + data2, err := os.ReadFile(filePath2) + s.Require().NoError(err) + s.T().Log(hex.EncodeToString(data1)) + + s.Require().Equal(data1, data2) +} diff --git a/pp/go/storage/catalog/record.go b/pp/go/storage/catalog/record.go new file mode 100644 index 0000000000..df55a4c3e5 --- /dev/null +++ b/pp/go/storage/catalog/record.go @@ -0,0 +1,215 @@ +package catalog + +import ( + "sync" + "sync/atomic" + + "github.com/google/uuid" + "github.com/prometheus/prometheus/pp/go/util/optional" +) + +// +// Status +// + +// Status of [Head] for record in catalog. +type Status uint8 + +const ( + // StatusNew status for new [Head]. + StatusNew Status = iota + // StatusRotated status for rotated [Head]. + StatusRotated + // StatusCorrupted status for corrupted [Head]. Deprecated. + StatusCorrupted + // StatusPersisted status for persisted [Head]. + StatusPersisted + // StatusActive status for active [Head]. + StatusActive +) + +// +// Record +// + +// Record information about the [Head] in the catalog. +type Record struct { + id uuid.UUID // uuid + numberOfShards uint16 // number of shards + createdAt int64 // time of record creation + updatedAt int64 + deletedAt int64 + corrupted bool + lastAppendedSegmentID optional.Optional[uint32] + referenceCount int64 + status Status // status + numberOfSegments uint32 + mint int64 + maxt int64 +} + +// NewEmptyRecord init new empty [Record]. +func NewEmptyRecord() *Record { + return &Record{} +} + +// NewRecordWithData init new [Record] with parameters. +func NewRecordWithData( + id uuid.UUID, + numberOfShards uint16, + createdAt int64, + updatedAt int64, + deletedAt int64, + corrupted bool, + referenceCount int64, + status Status, + lastAppendedSegmentID *uint32, +) *Record { + return &Record{ + id: id, + numberOfShards: numberOfShards, + createdAt: createdAt, + updatedAt: updatedAt, + deletedAt: deletedAt, + corrupted: corrupted, + referenceCount: referenceCount, + status: status, + lastAppendedSegmentID: optional.WithRawValue(lastAppendedSegmentID), + } +} + +// NewRecordWithDataV3 init new [Record] version 3 with parameters. +func NewRecordWithDataV3( + id uuid.UUID, + numberOfShards uint16, + createdAt int64, + updatedAt int64, + deletedAt int64, + corrupted bool, + status Status, + numberOfSegments uint32, + mint int64, + maxt int64, +) *Record { + return &Record{ + id: id, + numberOfShards: numberOfShards, + createdAt: createdAt, + updatedAt: updatedAt, + deletedAt: deletedAt, + corrupted: corrupted, + status: status, + numberOfSegments: numberOfSegments, + mint: mint, + maxt: maxt, + } +} + +// Acquire increase reference count to [Head]. Returns func decrease reference count. +func (r *Record) Acquire() func() { + atomic.AddInt64(&r.referenceCount, 1) + var onceRelease sync.Once + return func() { + onceRelease.Do(func() { + atomic.AddInt64(&r.referenceCount, -1) + }) + } +} + +// Corrupted returns true if [Head] is corrupted. +func (r *Record) Corrupted() bool { + return r.corrupted +} + +// CreatedAt returns the timestamp when the [Record]([Head]) was created. +func (r *Record) CreatedAt() int64 { + return r.createdAt +} + +// DeletedAt returns the timestamp when the [Record]([Head]) was deleted. +func (r *Record) DeletedAt() int64 { + return r.deletedAt +} + +// Dir returns dir of [Head]. +func (r *Record) Dir() string { + return r.id.String() +} + +// ID returns id of [Head]. +func (r *Record) ID() string { + return r.id.String() +} + +// LastAppendedSegmentID returns last appended segment id if exist, else nil. +func (r *Record) LastAppendedSegmentID() *uint32 { + return r.lastAppendedSegmentID.RawValue() +} + +// Maxt returns max timestamp in [Head]. +func (r *Record) Maxt() int64 { + return r.maxt +} + +// Mint returns min timestamp in [Head]. +func (r *Record) Mint() int64 { + return r.mint +} + +// NumberOfSegments returns number of segments in [Head]. +func (r *Record) NumberOfSegments() uint32 { + return r.numberOfSegments +} + +// NumberOfShards returns number of shards of [Head]. +func (r *Record) NumberOfShards() uint16 { + return r.numberOfShards +} + +// ReferenceCount returns current of reference count. +func (r *Record) ReferenceCount() int64 { + return atomic.LoadInt64(&r.referenceCount) +} + +// SetLastAppendedSegmentID set last appended segment id. +func (r *Record) SetLastAppendedSegmentID(segmentID uint32) { + r.lastAppendedSegmentID.Set(segmentID) +} + +// SetNumberOfSegments number of segments in [Head]. +func (r *Record) SetNumberOfSegments(numberOfSegments uint32) { + r.numberOfSegments = numberOfSegments +} + +// Status returns current status of [Head]. +func (r *Record) Status() Status { + return r.status +} + +// UpdatedAt returns the timestamp when the [Record]([Head]) was updated. +func (r *Record) UpdatedAt() int64 { + return r.updatedAt +} + +// createRecordCopy create a copy of the [Record]. +func createRecordCopy(r *Record) *Record { + c := *r + return &c +} + +// applyRecordChanges apply changes to current [Record]. +func applyRecordChanges(r, changed *Record) { + r.createdAt = changed.createdAt + r.updatedAt = changed.updatedAt + r.deletedAt = changed.deletedAt + r.corrupted = changed.corrupted + r.status = changed.status + r.numberOfShards = changed.numberOfShards + r.mint = changed.mint + r.maxt = changed.maxt +} + +// LessByUpdateAt less [Record] by UpdateAt. +func LessByUpdateAt(lhs, rhs *Record) bool { + return lhs.UpdatedAt() < rhs.UpdatedAt() +} diff --git a/pp/go/storage/catalog/record_test.go b/pp/go/storage/catalog/record_test.go new file mode 100644 index 0000000000..75ffa9edd7 --- /dev/null +++ b/pp/go/storage/catalog/record_test.go @@ -0,0 +1,28 @@ +package catalog_test + +import ( + "testing" + + "github.com/stretchr/testify/suite" + + "github.com/prometheus/prometheus/pp/go/storage/catalog" +) + +type RecordSuite struct { + suite.Suite +} + +func TestRecordSuite(t *testing.T) { + suite.Run(t, new(RecordSuite)) +} + +func (s *RecordSuite) TestReferenceCounterIncDecValue() { + r := catalog.NewEmptyRecord() + s.Require().Equal(int64(0), r.ReferenceCount()) + release := r.Acquire() + s.Require().Equal(int64(1), r.ReferenceCount()) + release() + s.Require().Equal(int64(0), r.ReferenceCount()) + release() + s.Require().Equal(int64(0), r.ReferenceCount()) +} diff --git a/pp/go/storage/catalog/testdata/headv1.log b/pp/go/storage/catalog/testdata/headv1.log new file mode 100644 index 0000000000..ba6115a7b1 Binary files /dev/null and b/pp/go/storage/catalog/testdata/headv1.log differ diff --git a/pp/go/storage/catalog/testdata/headv2.log b/pp/go/storage/catalog/testdata/headv2.log new file mode 100644 index 0000000000..54d0f38eb5 Binary files /dev/null and b/pp/go/storage/catalog/testdata/headv2.log differ diff --git a/pp/go/storage/catalog/testdata/headv3.log b/pp/go/storage/catalog/testdata/headv3.log new file mode 100644 index 0000000000..11ea720a22 Binary files /dev/null and b/pp/go/storage/catalog/testdata/headv3.log differ diff --git a/pp/go/storage/head.md b/pp/go/storage/head.md new file mode 100644 index 0000000000..1eb3777e95 --- /dev/null +++ b/pp/go/storage/head.md @@ -0,0 +1,64 @@ +# Head + +## Creation + +Creates shards(**LSS**, **DataStorage**, **Wal**), run goroutines of the head, stored in the **Manager**. + +## Active + +Head is used to append and read current data, stored in the **Manager**: + +- **Appender** - add current data: + - *Append*: + - **LSS** - write; + - **DataStorage** - write; + - **Wal** via task: + - *Commit* - encode(LSS read) segment and add to segment writer(buffer); + - *Flush* - write to storage from buffer if exist; +- **Querier** - provides querying access over time series data: + - **LSS** - read; + - **DataStorage** - read; +- **Manager**: + - *MergeOutOfOrderChunks*: + - **DataStorage**: + - *MergeOutOfOrderChunks* - write; + - *CommitToWal* by timer: + - **Wal** via task: + - *Commit* - encode(LSS read) segment and add to segment writer(buffer); + - *Flush* - write to storage from buffer if exist; + - *Rotate* by timer: + - **DataStorage** via task: + - *MergeOutOfOrderChunks* - write; + - **Wal** via range: + - *Commit* - encode(LSS read) segment and add to segment writer(buffer); + - *Flush* - write to storage from buffer if exist; + - *Shutdown*: + - **ActiveHeadContainer** - container for active Head with weighted locker: + - *Close* - wait all active task is finished and close semaphore with lock(on append returns error); + - **Wal** via range: + - *Commit* - encode(LSS read) segment and add to segment writer(buffer); + - *Flush* - write to storage from buffer if exist; + - **Head**: + - *Close* - wait all active task is finished and close query semaphore with lock(on select returns empty series set), stop goroutine, **Wal** close. + +## Rotated + +The head that has completed its work, but has not yet been converted into blocks, is read-only, and new data is not being added, stored in the **Keeper**: + +- **Querier** - provides querying access over time series data: + - **LSS** - read; + - **DataStorage** - read; +- **Keeper**: + - *Write*: + - **Wal** via range: + - *Flush* - write to storage from buffer if exist; + - *Close* - if flush operations were successful; + - **BlockWriter** - converts the head into prom blocks and writes them to a storage: + - *WriteBlock*: + - **LSS** - read; + - **DataStorage** - read; + - *Shutdown*: + - **Wal** via range: + - *Flush* - write to storage from buffer if exist; + - **Head**: + - *Close* - wait all active task is finished and close query semaphore with lock(on select returns empty series set), stop goroutine, **Wal** close. diff --git a/pp/go/storage/head/container/weighted.go b/pp/go/storage/head/container/weighted.go new file mode 100644 index 0000000000..2f57397c54 --- /dev/null +++ b/pp/go/storage/head/container/weighted.go @@ -0,0 +1,74 @@ +package container + +import ( + "context" + "fmt" + "sync/atomic" + "unsafe" + + "github.com/prometheus/prometheus/pp/go/util/locker" +) + +// Head the minimum required Head implementation for a container. +type Head[T any] interface { + // Concurrency return current head workers concurrency. + Concurrency() int64 + + // for use as a pointer + *T +} + +// Weighted container for [Head] with weighted locker. +type Weighted[T any, THead Head[T]] struct { + wlocker *locker.Weighted + head *T +} + +// NewWeighted init new [Weighted]. +func NewWeighted[T any, THead Head[T]](head THead) *Weighted[T, THead] { + return &Weighted[T, THead]{ + wlocker: locker.NewWeighted(2 * head.Concurrency()), // x2 for back pressure + head: head, + } +} + +// Close closes wlocker semaphore for the inability work with [Head]. +func (c *Weighted[T, THead]) Close() error { + return c.wlocker.Close() +} + +// Get the active head [Head] without lock and return. +func (c *Weighted[T, THead]) Get() THead { + return (*T)(atomic.LoadPointer( + (*unsafe.Pointer)(unsafe.Pointer(&c.head))), // #nosec G103 // it's meant to be that way + ) +} + +// Replace the active head [Head] with a new head under the exlusive priority lock. +func (c *Weighted[T, THead]) Replace(ctx context.Context, newHead THead) error { + unlock, err := c.wlocker.LockWithPriority(ctx) + if err != nil { + return fmt.Errorf("weighted lock with priority: %w", err) + } + + atomic.StorePointer( + (*unsafe.Pointer)(unsafe.Pointer(&c.head)), // #nosec G103 // it's meant to be that way + unsafe.Pointer(newHead), // #nosec G103 // it's meant to be that way + ) + c.wlocker = locker.NewWeighted(2 * newHead.Concurrency()) // x2 for back pressure + + unlock() + + return nil +} + +// With calls fn(h Head) under the non-exlusive lock. +func (c *Weighted[T, THead]) With(ctx context.Context, fn func(h THead) error) error { + runlock, err := c.wlocker.RLock(ctx) + if err != nil { + return fmt.Errorf("weighted rlock: %w", err) + } + defer runlock() + + return fn(c.head) +} diff --git a/pp/go/storage/head/container/weighted_test.go b/pp/go/storage/head/container/weighted_test.go new file mode 100644 index 0000000000..7ced3aa0fe --- /dev/null +++ b/pp/go/storage/head/container/weighted_test.go @@ -0,0 +1,137 @@ +package container_test + +import ( + "context" + "fmt" + "testing" + "unsafe" + + "github.com/stretchr/testify/suite" + + "github.com/prometheus/prometheus/pp/go/storage/head/container" + "github.com/prometheus/prometheus/pp/go/util/locker" +) + +type WeightedSuite struct { + suite.Suite +} + +func TestWeightedSuite(t *testing.T) { + suite.Run(t, new(WeightedSuite)) +} + +func (s *WeightedSuite) TestGet() { + expectedHead := &testHead{c: 2} + c := container.NewWeighted(expectedHead) + + actualHead := c.Get() + + s.Equal(expectedHead, actualHead) +} + +func (s *WeightedSuite) TestReplace() { + baseCtx := context.Background() + expectedHead := &testHead{c: 2} + newHead := &testHead{c: 3} + c := container.NewWeighted(expectedHead) + + err := c.Replace(baseCtx, newHead) + s.Require().NoError(err) + + actualHead := c.Get() + + s.NotEqual(expectedHead, actualHead) + s.NotEqual(unsafe.Pointer(expectedHead), unsafe.Pointer(actualHead)) + s.Equal(newHead, actualHead) + s.Equal(unsafe.Pointer(newHead), unsafe.Pointer(actualHead)) +} + +func (s *WeightedSuite) TestReplaceError() { + expectedHead := &testHead{c: 2} + newHead := &testHead{c: 3} + c := container.NewWeighted(expectedHead) + ctx, cancel := context.WithCancel(context.Background()) + cancel() + + err := c.Replace(ctx, newHead) + s.Error(err) +} + +func (s *WeightedSuite) TestWith() { + baseCtx := context.Background() + expectedHead := &testHead{c: 2} + c := container.NewWeighted(expectedHead) + + err := c.With(baseCtx, func(h *testHead) error { + if expectedHead.c != h.c { + return fmt.Errorf("expectedHead(%d) not equal actual(%d)", expectedHead.c, h.c) + } + + return nil + }) + + s.NoError(err) +} + +func (s *WeightedSuite) TestWithError() { + baseCtx := context.Background() + expectedHead := &testHead{c: 1} + c := container.NewWeighted(expectedHead) + step1 := make(chan struct{}) + step2 := make(chan struct{}) + ctx, cancel := context.WithCancel(baseCtx) + + go c.With(baseCtx, func(_ *testHead) error { + close(step1) + cancel() + <-step2 + return nil + }) + + <-step1 + err := c.With(ctx, func(_ *testHead) error { + return nil + }) + close(step2) + + s.Error(err) +} + +func (s *WeightedSuite) TestClose() { + baseCtx := context.Background() + expectedHead := &testHead{c: 2} + c := container.NewWeighted(expectedHead) + + err := c.Close() + s.Require().NoError(err) + + actualHead := c.Get() + s.Require().NotNil(actualHead) + s.Equal(expectedHead.c, actualHead.c) + + err = c.Replace(baseCtx, &testHead{c: 3}) + s.Require().ErrorIs(err, locker.ErrSemaphoreClosed) + + err = c.With(baseCtx, func(h *testHead) error { + if expectedHead.c != h.c { + return fmt.Errorf("expectedHead(%d) not equal actual(%d)", expectedHead.c, h.c) + } + + return nil + }) + s.Require().ErrorIs(err, locker.ErrSemaphoreClosed) +} + +// +// testHead +// + +// testHead implementation [container.Head]. +type testHead struct { + c int64 +} + +// Concurrency implementation [container.Head]. +func (h *testHead) Concurrency() int64 { + return h.c +} diff --git a/pp/go/storage/head/head/head.go b/pp/go/storage/head/head/head.go new file mode 100644 index 0000000000..a10e84e08f --- /dev/null +++ b/pp/go/storage/head/head/head.go @@ -0,0 +1,321 @@ +package head + +import ( + "context" + "errors" + "fmt" + "runtime" + "sync" + "sync/atomic" + + "github.com/prometheus/client_golang/prometheus" + + "github.com/prometheus/prometheus/pp/go/logger" + "github.com/prometheus/prometheus/pp/go/storage/head/task" + "github.com/prometheus/prometheus/pp/go/util" + "github.com/prometheus/prometheus/pp/go/util/locker" +) + +//go:generate -command moq go run github.com/matryer/moq --rm --skip-ensure --pkg head_test --out +//go:generate moq head_moq_test.go . Shard + +// ExtraWorkers number of extra workers for operation on shards. +var ExtraWorkers = 0 + +// defaultNumberOfWorkers default number of workers. +const defaultNumberOfWorkers = 2 + +// +// Shard +// + +// Shard the minimum required head Shard implementation. +type Shard interface { + // ShardID returns the shard ID. + ShardID() uint16 + + // Close closes the wal segmentWriter. + Close() error +} + +// +// Head +// + +// Head stores and manages shards, handles reads and writes of time series data within a time window. +type Head[TShard Shard, TGorutineShard Shard] struct { + id string + generation uint64 + + gshardCtor func(s TShard, numberOfShards uint16) TGorutineShard + releaseHeadFn func() + + shards []TShard + taskChs []chan *task.Generic[TGorutineShard] + querySemaphore *locker.Weighted + + stopc chan struct{} + wg sync.WaitGroup + closeOnce sync.Once + + readOnly uint32 + + // for clearing [Head] metrics + memoryInUse *prometheus.GaugeVec + // for tasks metrics + tasksCreated *prometheus.CounterVec + tasksDone *prometheus.CounterVec + tasksLive *prometheus.CounterVec + tasksExecute *prometheus.CounterVec +} + +// NewHead init new [Head]. +// +//revive:disable-next-line:function-length long but readable. +func NewHead[TShard Shard, TGoroutineShard Shard]( + id string, + shards []TShard, + gshardCtor func(TShard, uint16) TGoroutineShard, + releaseHeadFn func(), + generation uint64, + registerer prometheus.Registerer, +) *Head[TShard, TGoroutineShard] { + numberOfShards := len(shards) + taskChs := make([]chan *task.Generic[TGoroutineShard], numberOfShards) + concurrency := calculateHeadConcurrency(numberOfShards) // current head workers concurrency + for shardID := range numberOfShards { + taskChs[shardID] = make(chan *task.Generic[TGoroutineShard], 4*concurrency) // x4 for back pressure + } + + factory := util.NewUnconflictRegisterer(registerer) + h := &Head[TShard, TGoroutineShard]{ + id: id, + generation: generation, + gshardCtor: gshardCtor, + releaseHeadFn: releaseHeadFn, + shards: shards, + taskChs: taskChs, + querySemaphore: locker.NewWeighted(2 * concurrency), // x2 for back pressure + stopc: make(chan struct{}), + wg: sync.WaitGroup{}, + closeOnce: sync.Once{}, + + // for clearing [Head] metrics + memoryInUse: factory.NewGaugeVec(prometheus.GaugeOpts{ + Name: "prompp_head_cgo_memory_bytes", + Help: "Current value memory in use in bytes.", + }, []string{"head_id", "allocator", "shard_id"}), + // for tasks metrics + tasksCreated: factory.NewCounterVec(prometheus.CounterOpts{ + Name: "prompp_head_task_created_count", + Help: "Number of created tasks.", + }, []string{"type_task"}), + tasksDone: factory.NewCounterVec(prometheus.CounterOpts{ + Name: "prompp_head_task_done_count", + Help: "Number of done tasks.", + }, []string{"type_task"}), + tasksLive: factory.NewCounterVec(prometheus.CounterOpts{ + Name: "prompp_head_task_live_duration_microseconds_sum", + Help: "The duration of the live task in microseconds.", + }, []string{"type_task"}), + tasksExecute: factory.NewCounterVec(prometheus.CounterOpts{ + Name: "prompp_head_task_execute_duration_microseconds_sum", + Help: "The duration of the task execution in microseconds.", + }, []string{"type_task"}), + } + + h.run() + + runtime.SetFinalizer(h, func(h *Head[TShard, TGoroutineShard]) { + h.memoryInUse.DeletePartialMatch(prometheus.Labels{"head_id": h.id}) + logger.Debugf("[Head] %s destroyed", h.String()) + }) + + logger.Debugf("[Head] %s created", h.String()) + + return h +} + +// AcquireQuery acquires the [Head] semaphore with a weight of 1, +// blocking until resources are available or ctx is done. +// On success, returns nil. On failure, returns ctx.Err() and leaves the semaphore unchanged. +func (h *Head[TShard, TGorutineShard]) AcquireQuery(ctx context.Context) (release func(), err error) { + return h.querySemaphore.RLock(ctx) +} + +// Close closes wals, query semaphore for the inability to get query and clear metrics. +func (h *Head[TShard, TGorutineShard]) Close() (err error) { + h.closeOnce.Do(func() { + if err = h.querySemaphore.Close(); err != nil { + return + } + + close(h.stopc) + h.wg.Wait() + + for _, s := range h.shards { + err = errors.Join(err, s.Close()) + } + + if h.releaseHeadFn != nil { + h.releaseHeadFn() + } + + logger.Debugf("[Head] %s is closed", h.String()) + }) + + return err +} + +// Concurrency return current head workers concurrency. +func (h *Head[TShard, TGorutineShard]) Concurrency() int64 { + return calculateHeadConcurrency(len(h.shards)) +} + +// CreateTask create a task for operations on the [Head] shards. +func (h *Head[TShard, TGorutineShard]) CreateTask( + taskName string, + shardFn func(shard TGorutineShard) error, +) *task.Generic[TGorutineShard] { + ls := prometheus.Labels{"type_task": taskName} + + return task.NewGeneric( + shardFn, + h.tasksCreated.With(ls), + h.tasksDone.With(ls), + h.tasksLive.With(ls), + h.tasksExecute.With(ls), + ) +} + +// Enqueue the task to be executed on shards [Head]. +func (h *Head[TShard, TGorutineShard]) Enqueue(t *task.Generic[TGorutineShard]) { + t.SetShardsNumber(h.NumberOfShards()) + + for _, taskCh := range h.taskChs { + taskCh <- t + } +} + +// EnqueueOnShard the task to be executed on head on specific shard. +func (h *Head[TShard, TGorutineShard]) EnqueueOnShard(t *task.Generic[TGorutineShard], shardID uint16) { + t.SetShardsNumber(1) + + h.taskChs[shardID] <- t +} + +// Generation returns current generation of [Head]. +func (h *Head[TShard, TGorutineShard]) Generation() uint64 { + return h.generation +} + +// ID returns id [Head]. +func (h *Head[TShard, TGorutineShard]) ID() string { + return h.id +} + +// IsReadOnly returns true if the [Head] has switched to read-only. +func (h *Head[TShard, TGorutineShard]) IsReadOnly() bool { + return atomic.LoadUint32(&h.readOnly) > 0 +} + +// NumberOfShards returns current number of shards in to [Head]. +func (h *Head[TShard, TGorutineShard]) NumberOfShards() uint16 { + return uint16(len(h.shards)) // #nosec G115 // no overflow +} + +// RangeQueueSize returns an iterator over the [Head] task channels, to collect metrics. +func (h *Head[TShard, TGorutineShard]) RangeQueueSize() func(func(shardID, size int) bool) { + return func(yield func(shardID, size int) bool) { + for shardID, taskCh := range h.taskChs { + if !yield(shardID, len(taskCh)) { + return + } + } + } +} + +// RangeShards returns an iterator over the [Head] [Shard]s, through which the shard can be directly accessed. +func (h *Head[TShard, TGorutineShard]) RangeShards() func(func(TShard) bool) { + return func(yield func(s TShard) bool) { + for _, shard := range h.shards { + if !yield(shard) { + return + } + } + } +} + +// SetReadOnly sets the read-only flag for the [Head]. +func (h *Head[TShard, TGorutineShard]) SetReadOnly() { + atomic.StoreUint32(&h.readOnly, 1) +} + +// String serialize as string. +func (h *Head[TShard, TGorutineShard]) String() string { + return fmt.Sprintf("{id: %s, generation: %d}", h.id, h.generation) +} + +// run loop for each shard. +func (h *Head[TShard, TGorutineShard]) run() { + workers := defaultNumberOfWorkers + ExtraWorkers + numberOfShards := len(h.shards) + h.wg.Add(workers * numberOfShards) + for shardID := range numberOfShards { + for range workers { + go func(sid int) { + defer h.wg.Done() + h.shardLoop(h.taskChs[sid], h.stopc, h.shards[sid]) + }(shardID) + } + } +} + +// shardLoop run shard loop for operation. +func (h *Head[TShard, TGorutineShard]) shardLoop( + taskCH chan *task.Generic[TGorutineShard], + stopc chan struct{}, + s TShard, +) { + pgs := h.gshardCtor(s, h.NumberOfShards()) + + for { + select { + case <-stopc: + return + + case t := <-taskCH: + t.ExecuteOnShard(pgs) + } + } +} + +// calculateHeadConcurrency calculate current head workers concurrency. +func calculateHeadConcurrency(numberOfShards int) int64 { + return int64(defaultNumberOfWorkers+ExtraWorkers) * int64(numberOfShards) +} + +// +// CopyAddedSeries +// + +// CopyAddedSeries copy the label sets from the source lss to the destination lss that were added source lss. +func CopyAddedSeries[TShard Shard, TGorutineShard Shard]( + shardCopier func(source, destination TShard), +) func(source, destination *Head[TShard, TGorutineShard]) { + return func(source, destination *Head[TShard, TGorutineShard]) { + if source.NumberOfShards() != destination.NumberOfShards() { + logger.Warnf( + "source[%d] and destination[%d] number of shards must be the same", + source.NumberOfShards(), + destination.NumberOfShards(), + ) + + return + } + + for shardID := range source.NumberOfShards() { + shardCopier(source.shards[shardID], destination.shards[shardID]) + } + } +} diff --git a/pp/go/storage/head/head/head_moq_test.go b/pp/go/storage/head/head/head_moq_test.go new file mode 100644 index 0000000000..bf5b0e7321 --- /dev/null +++ b/pp/go/storage/head/head/head_moq_test.go @@ -0,0 +1,100 @@ +// Code generated by moq; DO NOT EDIT. +// github.com/matryer/moq + +package head_test + +import ( + "sync" +) + +// ShardMock is a mock implementation of head.Shard. +// +// func TestSomethingThatUsesShard(t *testing.T) { +// +// // make and configure a mocked head.Shard +// mockedShard := &ShardMock{ +// CloseFunc: func() error { +// panic("mock out the Close method") +// }, +// ShardIDFunc: func() uint16 { +// panic("mock out the ShardID method") +// }, +// } +// +// // use mockedShard in code that requires head.Shard +// // and then make assertions. +// +// } +type ShardMock struct { + // CloseFunc mocks the Close method. + CloseFunc func() error + + // ShardIDFunc mocks the ShardID method. + ShardIDFunc func() uint16 + + // calls tracks calls to the methods. + calls struct { + // Close holds details about calls to the Close method. + Close []struct { + } + // ShardID holds details about calls to the ShardID method. + ShardID []struct { + } + } + lockClose sync.RWMutex + lockShardID sync.RWMutex +} + +// Close calls CloseFunc. +func (mock *ShardMock) Close() error { + if mock.CloseFunc == nil { + panic("ShardMock.CloseFunc: method is nil but Shard.Close was just called") + } + callInfo := struct { + }{} + mock.lockClose.Lock() + mock.calls.Close = append(mock.calls.Close, callInfo) + mock.lockClose.Unlock() + return mock.CloseFunc() +} + +// CloseCalls gets all the calls that were made to Close. +// Check the length with: +// +// len(mockedShard.CloseCalls()) +func (mock *ShardMock) CloseCalls() []struct { +} { + var calls []struct { + } + mock.lockClose.RLock() + calls = mock.calls.Close + mock.lockClose.RUnlock() + return calls +} + +// ShardID calls ShardIDFunc. +func (mock *ShardMock) ShardID() uint16 { + if mock.ShardIDFunc == nil { + panic("ShardMock.ShardIDFunc: method is nil but Shard.ShardID was just called") + } + callInfo := struct { + }{} + mock.lockShardID.Lock() + mock.calls.ShardID = append(mock.calls.ShardID, callInfo) + mock.lockShardID.Unlock() + return mock.ShardIDFunc() +} + +// ShardIDCalls gets all the calls that were made to ShardID. +// Check the length with: +// +// len(mockedShard.ShardIDCalls()) +func (mock *ShardMock) ShardIDCalls() []struct { +} { + var calls []struct { + } + mock.lockShardID.RLock() + calls = mock.calls.ShardID + mock.lockShardID.RUnlock() + return calls +} diff --git a/pp/go/storage/head/head/head_test.go b/pp/go/storage/head/head/head_test.go new file mode 100644 index 0000000000..91713b1b01 --- /dev/null +++ b/pp/go/storage/head/head/head_test.go @@ -0,0 +1,348 @@ +package head_test + +import ( + "sync" + "sync/atomic" + "testing" + + "github.com/stretchr/testify/suite" + + "github.com/prometheus/prometheus/pp/go/storage/head/head" +) + +type HeadSuite struct { + suite.Suite + + id string + generation uint64 +} + +func TestHeadSuite(t *testing.T) { + suite.Run(t, new(HeadSuite)) +} + +func (s *HeadSuite) SetupSuite() { + s.id = "test-head-id" + s.generation = uint64(42) +} + +func (s *HeadSuite) TestClose() { + sd := &ShardMock{ + ShardIDFunc: func() uint16 { return 0 }, + CloseFunc: func() error { return nil }, + } + + closeCount := 0 + h := head.NewHead( + s.id, + []*ShardMock{sd}, + newPerGoroutineShardMockfunc, + func() { closeCount++ }, + s.generation, + nil, + ) + + s.T().Log("first close head", h.String()) + err := h.Close() + s.Require().NoError(err) + + s.Len(sd.CloseCalls(), 1) + s.Equal(1, closeCount) + + s.T().Log("second close head", h.String()) + err = h.Close() + s.Require().NoError(err) + + s.Len(sd.CloseCalls(), 1) + s.Equal(1, closeCount) +} + +func (s *HeadSuite) TestConcurrency() { + sd := &ShardMock{ + ShardIDFunc: func() uint16 { return 0 }, + CloseFunc: func() error { return nil }, + } + + h := head.NewHead( + s.id, + []*ShardMock{sd, sd}, + newPerGoroutineShardMockfunc, + nil, + s.generation, + nil, + ) + defer h.Close() + + s.Equal(int64(4), h.Concurrency()) +} + +func (s *HeadSuite) TestEnqueue() { + sd0 := &ShardMock{ + ShardIDFunc: func() uint16 { return 0 }, + CloseFunc: func() error { return nil }, + } + + sd1 := &ShardMock{ + ShardIDFunc: func() uint16 { return 1 }, + CloseFunc: func() error { return nil }, + } + + h := head.NewHead( + s.id, + []*ShardMock{sd0, sd1}, + newPerGoroutineShardMockfunc, + nil, + s.generation, + nil, + ) + defer h.Close() + + shardsExecuted := uint32(0) + t := h.CreateTask("test-task", func(shard *perGoroutineShardMock) error { + atomic.AddUint32(&shardsExecuted, uint32(shard.ShardID()+1)) + return nil + }) + + h.Enqueue(t) + + err := t.Wait() + s.Require().NoError(err) + + s.Equal(uint32(3), shardsExecuted) +} + +func (s *HeadSuite) TestEnqueueOnShard() { + sd0 := &ShardMock{ + ShardIDFunc: func() uint16 { return 0 }, + CloseFunc: func() error { return nil }, + } + + sd1 := &ShardMock{ + ShardIDFunc: func() uint16 { return 1 }, + CloseFunc: func() error { return nil }, + } + + h := head.NewHead( + s.id, + []*ShardMock{sd0, sd1}, + newPerGoroutineShardMockfunc, + nil, + s.generation, + nil, + ) + defer h.Close() + + shardActual := uint16(1<<16 - 1) + expectedShard := uint16(1) + t := h.CreateTask("test-task", func(shard *perGoroutineShardMock) error { + shardActual = shard.ShardID() + return nil + }) + + h.EnqueueOnShard(t, expectedShard) + + err := t.Wait() + s.Require().NoError(err) + + s.Equal(expectedShard, shardActual) + + expectedShard = uint16(0) + t = h.CreateTask("test-task", func(shard *perGoroutineShardMock) error { + shardActual = shard.ShardID() + return nil + }) + + h.EnqueueOnShard(t, expectedShard) + + err = t.Wait() + s.Require().NoError(err) + + s.Equal(expectedShard, shardActual) +} + +func (s *HeadSuite) TestGeneration() { + sd := &ShardMock{ + ShardIDFunc: func() uint16 { return 0 }, + CloseFunc: func() error { return nil }, + } + + h := head.NewHead( + s.id, + []*ShardMock{sd, sd}, + newPerGoroutineShardMockfunc, + nil, + s.generation, + nil, + ) + defer h.Close() + + s.Equal(s.generation, h.Generation()) +} + +func (s *HeadSuite) TestID() { + sd := &ShardMock{ + ShardIDFunc: func() uint16 { return 0 }, + CloseFunc: func() error { return nil }, + } + + h := head.NewHead( + s.id, + []*ShardMock{sd, sd}, + newPerGoroutineShardMockfunc, + nil, + s.generation, + nil, + ) + defer h.Close() + + s.Equal(s.id, h.ID()) +} + +func (s *HeadSuite) TestIsReadOnly() { + sd := &ShardMock{ + ShardIDFunc: func() uint16 { return 0 }, + CloseFunc: func() error { return nil }, + } + + h := head.NewHead( + s.id, + []*ShardMock{sd, sd}, + newPerGoroutineShardMockfunc, + nil, + s.generation, + nil, + ) + defer h.Close() + + s.False(h.IsReadOnly()) + + h.SetReadOnly() + s.True(h.IsReadOnly()) +} + +func (s *HeadSuite) TestNumberOfShards() { + sd := &ShardMock{ + ShardIDFunc: func() uint16 { return 0 }, + CloseFunc: func() error { return nil }, + } + + h := head.NewHead( + s.id, + []*ShardMock{sd, sd, sd}, + newPerGoroutineShardMockfunc, + nil, + s.generation, + nil, + ) + defer h.Close() + + s.Equal(uint16(3), h.NumberOfShards()) +} + +func (s *HeadSuite) TestRangeQueueSize() { + sd0 := &ShardMock{ + ShardIDFunc: func() uint16 { return 0 }, + CloseFunc: func() error { return nil }, + } + + sd1 := &ShardMock{ + ShardIDFunc: func() uint16 { return 1 }, + CloseFunc: func() error { return nil }, + } + + h := head.NewHead( + s.id, + []*ShardMock{sd0, sd1}, + newPerGoroutineShardMockfunc, + nil, + s.generation, + nil, + ) + defer h.Close() + + execute := sync.WaitGroup{} + execute.Add(4) + + done := make(chan struct{}) + t1 := h.CreateTask("test-task", func(_ *perGoroutineShardMock) error { + execute.Done() + <-done + return nil + }) + h.Enqueue(t1) + + t2 := h.CreateTask("test-task", func(_ *perGoroutineShardMock) error { + execute.Done() + <-done + return nil + }) + h.Enqueue(t2) + + execute.Wait() + + t3 := h.CreateTask("test-task", func(_ *perGoroutineShardMock) error { + <-done + return nil + }) + h.Enqueue(t3) + + expectedShardID := 0 + for shardID, size := range h.RangeQueueSize() { + s.Equal(expectedShardID, shardID) + s.Equal(1, size) + expectedShardID++ + } + + close(done) + + err := t1.Wait() + s.Require().NoError(err) + + err = t2.Wait() + s.Require().NoError(err) + + err = t2.Wait() + s.Require().NoError(err) +} + +func (s *HeadSuite) TestRangeShards() { + sd0 := &ShardMock{ + ShardIDFunc: func() uint16 { return 0 }, + CloseFunc: func() error { return nil }, + } + + sd1 := &ShardMock{ + ShardIDFunc: func() uint16 { return 1 }, + CloseFunc: func() error { return nil }, + } + + h := head.NewHead( + s.id, + []*ShardMock{sd0, sd1}, + newPerGoroutineShardMockfunc, + nil, + s.generation, + nil, + ) + defer h.Close() + + expectedShardID := uint16(0) + for shard := range h.RangeShards() { + s.Equal(expectedShardID, shard.ShardID()) + expectedShardID++ + } +} + +// +// perGoroutineShardMock +// + +// perGoroutineShardMock mock for [PerGoroutineShard]. +type perGoroutineShardMock struct { + *ShardMock +} + +// newPerGoroutineShardMockfunc constructor for [PerGoroutineShard]. +func newPerGoroutineShardMockfunc(sd *ShardMock, _ uint16) *perGoroutineShardMock { + return &perGoroutineShardMock{ShardMock: sd} +} diff --git a/pp/go/storage/head/keeper/keeper.go b/pp/go/storage/head/keeper/keeper.go new file mode 100644 index 0000000000..202eb4aff0 --- /dev/null +++ b/pp/go/storage/head/keeper/keeper.go @@ -0,0 +1,222 @@ +package keeper + +import ( + "container/heap" + "errors" + "sync" + "time" + + "github.com/prometheus/prometheus/pp/go/logger" +) + +type addPolicy = uint8 + +const ( + // MinHeadConvertingQueueSize the minimum value of the [Keeper]'s queue. + MinHeadConvertingQueueSize = 2 + + add addPolicy = 0 + addWithReplace addPolicy = 1 +) + +// ErrorNoSlots error when keeper has no slots. +var ErrorNoSlots = errors.New("keeper has no slots") + +// +// RemovedHeadNotifier +// + +// RemovedHeadNotifier sends a notify that the [Head] has been removed. +type RemovedHeadNotifier interface { + // Notify sends a notify that the [Head] has been removed. + Notify() +} + +type sortableHead[THead any] struct { + head THead + createdAt time.Duration +} + +type headSortedSlice[THead any] []sortableHead[THead] + +func (q *headSortedSlice[THead]) Len() int { + return len(*q) +} + +func (q *headSortedSlice[THead]) Less(i, j int) bool { + return (*q)[i].createdAt < (*q)[j].createdAt +} + +func (q *headSortedSlice[THead]) Swap(i, j int) { + (*q)[i], (*q)[j] = (*q)[j], (*q)[i] +} + +func (q *headSortedSlice[THead]) Push(head any) { + *q = append(*q, head.(sortableHead[THead])) +} + +func (q *headSortedSlice[THead]) Pop() any { + n := len(*q) + item := (*q)[n-1] + *q = (*q)[0 : n-1] + return item +} + +// +// Head +// + +// Head the minimum required [Head] implementation for a [Keeper]. +type Head[T any] interface { + // ID returns id [Head]. + ID() string + + // Close closes wals, query semaphore for the inability to get query and clear metrics. + Close() error + + // for use as a pointer + *T +} + +// +// Keeper +// + +// Keeper holds outdated heads until conversion. +type Keeper[T any, THead Head[T]] struct { + heads headSortedSlice[THead] + removedHeadNotifier RemovedHeadNotifier + lock sync.RWMutex +} + +// NewKeeper init new [Keeper]. +func NewKeeper[T any, THead Head[T]]( + keeperCapacity int, + removedHeadNotifier RemovedHeadNotifier, +) *Keeper[T, THead] { + return &Keeper[T, THead]{ + heads: make(headSortedSlice[THead], 0, max(keeperCapacity, MinHeadConvertingQueueSize)), + removedHeadNotifier: removedHeadNotifier, + } +} + +// Add the [Head] to the [Keeper] if there is a free slot. +func (k *Keeper[T, THead]) Add(head THead, createdAt time.Duration) error { + k.lock.Lock() + result := k.addHead(head, createdAt, add) + k.lock.Unlock() + + return result +} + +// AddWithReplace the [Head] to the [Keeper] with replace if the createdAt is earlier. +func (k *Keeper[T, THead]) AddWithReplace(head THead, createdAt time.Duration) error { + k.lock.Lock() + result := k.addHead(head, createdAt, addWithReplace) + k.lock.Unlock() + + return result +} + +// Close closes for the inability work with [Head]. +func (k *Keeper[T, THead]) Close() error { + k.lock.Lock() + if len(k.heads) == 0 { + k.lock.Unlock() + return nil + } + + errs := make([]error, 0, len(k.heads)) + for _, head := range k.heads { + errs = append(errs, head.head.Close()) + } + k.lock.Unlock() + + return errors.Join(errs...) +} + +// HasSlot returns the tru if there is a slot in the [Keeper]. +func (k *Keeper[T, THead]) HasSlot() bool { + k.lock.RLock() + result := cap(k.heads) > len(k.heads) + k.lock.RUnlock() + return result +} + +// Heads returns a slice of the [Head]s stored in the [Keeper]. +func (k *Keeper[T, THead]) Heads() []THead { + k.lock.RLock() + + if len(k.heads) == 0 { + k.lock.RUnlock() + return nil + } + + headsCopy := make([]THead, 0, len(k.heads)) + for _, head := range k.heads { + headsCopy = append(headsCopy, head.head) + } + + k.lock.RUnlock() + + return headsCopy +} + +// Remove removes [Head]s from the [Keeper]. +func (k *Keeper[T, THead]) Remove(headsForRemove []THead) { + if len(headsForRemove) == 0 { + return + } + + headsMap := make(map[string]THead, len(headsForRemove)) + for _, head := range headsForRemove { + headsMap[head.ID()] = nil + } + + k.lock.Lock() + newHeads := make([]sortableHead[THead], 0, cap(k.heads)) + for _, head := range k.heads { + if _, ok := headsMap[head.head.ID()]; ok { + headsMap[head.head.ID()] = head.head + } else { + newHeads = append(newHeads, head) + } + } + k.setHeads(newHeads) + k.lock.Unlock() + + var shouldNotify bool + for _, head := range headsMap { + if head != nil { + _ = head.Close() + logger.Infof("[Keeper]: head %s persisted, closed and removed", head.ID()) + shouldNotify = true + } + } + + if shouldNotify { + k.removedHeadNotifier.Notify() + } +} + +func (k *Keeper[T, THead]) addHead(head THead, createdAt time.Duration, policy addPolicy) error { + if len(k.heads) < cap(k.heads) { + heap.Push(&k.heads, sortableHead[THead]{head: head, createdAt: createdAt}) + return nil + } + + if policy == addWithReplace && k.heads[0].createdAt < createdAt { + _ = k.heads[0].head.Close() + k.heads[0].head = head + k.heads[0].createdAt = createdAt + heap.Fix(&k.heads, 0) + return nil + } + + return ErrorNoSlots +} + +func (k *Keeper[T, THead]) setHeads(heads headSortedSlice[THead]) { + k.heads = heads + heap.Init(&k.heads) +} diff --git a/pp/go/storage/head/keeper/keeper_test.go b/pp/go/storage/head/keeper/keeper_test.go new file mode 100644 index 0000000000..47a800bfcb --- /dev/null +++ b/pp/go/storage/head/keeper/keeper_test.go @@ -0,0 +1,126 @@ +package keeper + +import ( + "testing" + + "github.com/stretchr/testify/suite" +) + +type headForTest struct { + id string +} + +func newHeadForTest(id string) *headForTest { + return &headForTest{id: id} +} + +func (h *headForTest) ID() string { + return h.id +} + +func (*headForTest) Close() error { + return nil +} + +// +// testRemovedHeadNotifier +// + +// testRemovedHeadNotifier implementation [RemovedHeadNotifier]. +type testRemovedHeadNotifier struct { + count int +} + +// Notify implementation [RemovedHeadNotifier]. +func (n *testRemovedHeadNotifier) Notify() { + n.count++ +} + +type sortedSlice = headSortedSlice[*headForTest] + +type KeeperSuite struct { + suite.Suite + keeper *Keeper[headForTest, *headForTest] +} + +func TestKeeperSuite(t *testing.T) { + suite.Run(t, new(KeeperSuite)) +} + +func (s *KeeperSuite) TestAdd() { + // Arrange + removedHeadNotifier := &testRemovedHeadNotifier{} + s.keeper = NewKeeper[headForTest](2, removedHeadNotifier) + + // Act + _ = s.keeper.Add(newHeadForTest("d"), 4) + _ = s.keeper.Add(newHeadForTest("c"), 3) + err := s.keeper.Add(newHeadForTest("b"), 2) + + // Assert + s.Equal(sortedSlice{ + {head: newHeadForTest("c"), createdAt: 3}, + {head: newHeadForTest("d"), createdAt: 4}, + }, s.keeper.heads) + s.Equal(err, ErrorNoSlots) +} + +func (s *KeeperSuite) TestAddWithReplaceNoReplace() { + // Arrange + removedHeadNotifier := &testRemovedHeadNotifier{} + s.keeper = NewKeeper[headForTest](2, removedHeadNotifier) + + // Act + _ = s.keeper.Add(newHeadForTest("d"), 4) + _ = s.keeper.Add(newHeadForTest("c"), 3) + err := s.keeper.AddWithReplace(newHeadForTest("b"), 3) + + // Assert + s.Equal(sortedSlice{ + {head: newHeadForTest("c"), createdAt: 3}, + {head: newHeadForTest("d"), createdAt: 4}, + }, s.keeper.heads) + s.Equal(err, ErrorNoSlots) +} + +func (s *KeeperSuite) TestAddWithReplace() { + // Arrange + removedHeadNotifier := &testRemovedHeadNotifier{} + s.keeper = NewKeeper[headForTest](2, removedHeadNotifier) + + // Act + _ = s.keeper.Add(newHeadForTest("d"), 4) + _ = s.keeper.Add(newHeadForTest("c"), 3) + err := s.keeper.AddWithReplace(newHeadForTest("b"), 4) + + // Assert + s.Equal(sortedSlice{ + {head: newHeadForTest("b"), createdAt: 4}, + {head: newHeadForTest("d"), createdAt: 4}, + }, s.keeper.heads) + s.NoError(err) +} + +func (s *KeeperSuite) TestRemove() { + // Arrange + const Slots = 5 + + removedHeadNotifier := &testRemovedHeadNotifier{} + s.keeper = NewKeeper[headForTest](Slots, removedHeadNotifier) + _ = s.keeper.Add(newHeadForTest("a"), 1) + _ = s.keeper.Add(newHeadForTest("b"), 2) + _ = s.keeper.Add(newHeadForTest("c"), 3) + _ = s.keeper.Add(newHeadForTest("d"), 4) + _ = s.keeper.Add(newHeadForTest("e"), 5) + + // Act + s.keeper.Remove([]*headForTest{newHeadForTest("a"), newHeadForTest("c"), newHeadForTest("e")}) + + // Assert + s.Equal(sortedSlice{ + {head: newHeadForTest("b"), createdAt: 2}, + {head: newHeadForTest("d"), createdAt: 4}, + }, s.keeper.heads) + s.Equal(Slots, cap(s.keeper.heads)) + s.Equal(1, removedHeadNotifier.count) +} diff --git a/pp/go/storage/head/services/commiter.go b/pp/go/storage/head/services/commiter.go new file mode 100644 index 0000000000..eb6609c13d --- /dev/null +++ b/pp/go/storage/head/services/commiter.go @@ -0,0 +1,67 @@ +package services + +import ( + "context" + + "github.com/prometheus/prometheus/pp/go/logger" +) + +// +// Committer +// + +// Committer finalize segment from encoder and add to wal +// and flush wal segment writer, write all buffered data to storage, do via task. +type Committer[ + TTask Task, + TShard, TGoShard Shard, + THead Head[TTask, TShard, TGoShard], +] struct { + activeHead ActiveHeadContainer[TTask, TShard, TGoShard, THead] + m Mediator + isNewHead func(headID string) bool +} + +// NewCommitter init new [Committer]. +func NewCommitter[ + TTask Task, + TShard, TGoShard Shard, + THead Head[TTask, TShard, TGoShard], +]( + activeHead ActiveHeadContainer[TTask, TShard, TGoShard, THead], + m Mediator, + isNewHead func(headID string) bool, +) *Committer[TTask, TShard, TGoShard, THead] { + return &Committer[TTask, TShard, TGoShard, THead]{ + activeHead: activeHead, + m: m, + isNewHead: isNewHead, + } +} + +// Execute starts the [Committer]. +// +//revive:disable-next-line:confusing-naming // other type of Service. +func (s *Committer[TTask, TShard, TGoShard, THead]) Execute(ctx context.Context) error { + logger.Infof("The Committer is running.") + + for range s.m.C() { + if err := s.activeHead.With(ctx, s.commitFlushSync); err != nil { + logger.Errorf("wal commit failed: %v", err) + } + } + + logger.Infof("The Committer stopped.") + + return nil +} + +// commitFlushSync finalize segment from encoder and add to wal +// and flush wal segment writer, write all buffered data to storage and sync, do via range. +func (s *Committer[TTask, TShard, TGoShard, THead]) commitFlushSync(h THead) error { + if s.isNewHead(h.ID()) { + return nil + } + + return CFSViaRange(h) +} diff --git a/pp/go/storage/head/services/commiter_test.go b/pp/go/storage/head/services/commiter_test.go new file mode 100644 index 0000000000..2c84f613f6 --- /dev/null +++ b/pp/go/storage/head/services/commiter_test.go @@ -0,0 +1,182 @@ +package services_test + +import ( + "context" + "testing" + + "github.com/stretchr/testify/suite" + + "github.com/prometheus/prometheus/pp/go/cppbridge" + "github.com/prometheus/prometheus/pp/go/storage" + "github.com/prometheus/prometheus/pp/go/storage/head/container" + "github.com/prometheus/prometheus/pp/go/storage/head/head" + "github.com/prometheus/prometheus/pp/go/storage/head/services" + "github.com/prometheus/prometheus/pp/go/storage/head/services/mock" + "github.com/prometheus/prometheus/pp/go/storage/head/shard" + "github.com/prometheus/prometheus/pp/go/storage/head/shard/wal" +) + +type CommitterSuite struct { + suite.Suite + + baseCtx context.Context +} + +func TestCommitterSuite(t *testing.T) { + suite.Run(t, new(CommitterSuite)) +} + +func (s *CommitterSuite) SetupSuite() { + s.baseCtx = context.Background() +} + +func (s *CommitterSuite) createHead(segmentWriters []*mock.SegmentWriterMock) *storage.Head { + shards := make([]*shard.Shard, shardsCount) + for shardID, segmentWriter := range segmentWriters { + shards[shardID] = s.createShardOnMemory(segmentWriter, maxSegmentSize, uint16(shardID)) + } + + return head.NewHead( + "test-head-id", + shards, + shard.NewPerGoroutineShard[*storage.Wal], + nil, + 0, + nil, + ) +} + +func (*CommitterSuite) createShardOnMemory( + segmentWriter *mock.SegmentWriterMock, + maxSegmentSize uint32, + shardID uint16, +) *shard.Shard { + lss := shard.NewLSS() + // logShards is 0 for single encoder + shardWalEncoder := cppbridge.NewHeadWalEncoder(shardID, 0, lss.Target()) + + return shard.NewShard( + lss, + shard.NewDataStorage(), + nil, + nil, + wal.NewWal(shardWalEncoder, segmentWriter, maxSegmentSize, shardID, nil), + shardID, + ) +} + +func (s *CommitterSuite) TestHappyPath() { + trigger := make(chan struct{}, 1) + start := make(chan struct{}) + mediator := &mock.MediatorMock{ + CFunc: func() <-chan struct{} { + close(start) + return trigger + }, + } + + segmentWriters := make([]*mock.SegmentWriterMock, shardsCount) + for shardID := range shardsCount { + segmentWriters[shardID] = &mock.SegmentWriterMock{ + WriteFunc: func(*cppbridge.HeadEncodedSegment) error { return nil }, + FlushFunc: func() error { return nil }, + SyncFunc: func() error { return nil }, + CloseFunc: func() error { return nil }, + CurrentSizeFunc: func() int64 { return 0 }, + } + } + activeHeadContainer := container.NewWeighted(s.createHead(segmentWriters)) + isNewHead := func(string) bool { return false } + + committer := services.NewCommitter(activeHeadContainer, mediator, isNewHead) + done := make(chan struct{}) + + s.T().Run("execute", func(t *testing.T) { + t.Parallel() + + err := committer.Execute(s.baseCtx) + close(done) + s.NoError(err) + }) + + s.T().Run("tick", func(t *testing.T) { + t.Parallel() + + <-start + trigger <- struct{}{} + trigger <- struct{}{} + close(trigger) + <-done + + s.Require().NoError(activeHeadContainer.Close()) + + for _, segmentWriter := range segmentWriters { + if !s.Len(segmentWriter.WriteCalls(), 2) { + return + } + if !s.Len(segmentWriter.FlushCalls(), 2) { + return + } + if !s.Len(segmentWriter.SyncCalls(), 2) { + return + } + + for _, call := range segmentWriter.WriteCalls() { + s.Equal(uint32(0), call.Segment.Samples()) + } + } + }) +} + +func (s *CommitterSuite) TestSkipNewHead() { + trigger := make(chan struct{}, 1) + start := make(chan struct{}) + mediator := &mock.MediatorMock{ + CFunc: func() <-chan struct{} { + close(start) + return trigger + }, + } + + segmentWriters := make([]*mock.SegmentWriterMock, shardsCount) + for shardID := range shardsCount { + segmentWriters[shardID] = &mock.SegmentWriterMock{ + WriteFunc: func(*cppbridge.HeadEncodedSegment) error { return nil }, + FlushFunc: func() error { return nil }, + SyncFunc: func() error { return nil }, + CloseFunc: func() error { return nil }, + CurrentSizeFunc: func() int64 { return 0 }, + } + } + activeHeadContainer := container.NewWeighted(s.createHead(segmentWriters)) + + isNewHead := func(string) bool { return true } + committer := services.NewCommitter(activeHeadContainer, mediator, isNewHead) + done := make(chan struct{}) + + s.T().Run("execute", func(t *testing.T) { + t.Parallel() + + err := committer.Execute(s.baseCtx) + close(done) + s.Require().NoError(err) + }) + + s.T().Run("tick", func(t *testing.T) { + t.Parallel() + + <-start + trigger <- struct{}{} + trigger <- struct{}{} + close(trigger) + <-done + + s.Require().NoError(activeHeadContainer.Close()) + + for _, segmentWriter := range segmentWriters { + s.Empty(segmentWriter.WriteCalls()) + s.Empty(segmentWriter.FlushCalls()) + s.Empty(segmentWriter.SyncCalls()) + } + }) +} diff --git a/pp/go/storage/head/services/functions.go b/pp/go/storage/head/services/functions.go new file mode 100644 index 0000000000..871aaa2332 --- /dev/null +++ b/pp/go/storage/head/services/functions.go @@ -0,0 +1,111 @@ +package services + +import ( + "errors" + "fmt" +) + +const ( + // dsMergeOutOfOrderChunks name of task. + dsMergeOutOfOrderChunks = "data_storage_merge_out_of_order_chunks" + + // dsUnloadUnusedSeriesData name of task + dsUnloadUnusedSeriesData = "data_storage_unload_unused_series_data" +) + +// +// Commit, Flush, Sync +// + +// CFViaRange finalize segment from encoder and add to wal +// and flush wal segment writer, write all buffered data to storage without sync, do via range. +func CFViaRange[ + TShard Shard, + THead RangeHead[TShard], +](h THead) error { + // we hope that there will be no mistakes, positive expectations + var errs []error + for shard := range h.RangeShards() { + if err := shard.WalCommit(); err != nil { + errs = append(errs, fmt.Errorf("commit shard id %d: %w", shard.ShardID(), err)) + } + + if err := shard.WalFlush(); err != nil { + errs = append(errs, fmt.Errorf("flush shard id %d: %w", shard.ShardID(), err)) + } + } + + return errors.Join(errs...) +} + +// CFSViaRange finalize segment from encoder and add to wal +// and flush wal segment writer, write all buffered data to storage and sync, do via range. +func CFSViaRange[ + TShard Shard, + THead RangeHead[TShard], +](h THead) error { + // we hope that there will be no mistakes, positive expectations + var errs []error + for shard := range h.RangeShards() { + if err := shard.WalCommit(); err != nil { + errs = append(errs, fmt.Errorf("commit shard id %d: %w", shard.ShardID(), err)) + } + + if err := shard.WalFlush(); err != nil { + errs = append(errs, fmt.Errorf("flush shard id %d: %w", shard.ShardID(), err)) + + // if the flush operation fails, skip the Sync + continue + } + + if err := shard.WalSync(); err != nil { + errs = append(errs, fmt.Errorf("sync shard id %d: %w", shard.ShardID(), err)) + } + } + + return errors.Join(errs...) +} + +// +// UnloadUnusedSeriesDataWithHead +// + +// UnloadUnusedSeriesDataWithHead unload unused series data for [Head]. +func UnloadUnusedSeriesDataWithHead[ + TTask Task, + TShard, TGShard Shard, + THead Head[TTask, TShard, TGShard], +](h THead) error { + t := h.CreateTask( + dsUnloadUnusedSeriesData, + func(shard TGShard) error { + return shard.UnloadUnusedSeriesData() + }, + ) + h.Enqueue(t) + + return t.Wait() +} + +// +// MergeOutOfOrderChunksWithHead +// + +// MergeOutOfOrderChunksWithHead merge chunks with out of order data chunks for [Head]. +func MergeOutOfOrderChunksWithHead[ + TTask Task, + TShard, TGShard Shard, + THead Head[TTask, TShard, TGShard], +](h THead) error { + t := h.CreateTask( + dsMergeOutOfOrderChunks, + func(shard TGShard) error { + shard.MergeOutOfOrderChunks() + + return nil + }, + ) + h.Enqueue(t) + + return t.Wait() +} diff --git a/pp/go/storage/head/services/interface.go b/pp/go/storage/head/services/interface.go new file mode 100644 index 0000000000..b16e27a5a8 --- /dev/null +++ b/pp/go/storage/head/services/interface.go @@ -0,0 +1,258 @@ +package services + +import ( + "context" + "time" + + "github.com/prometheus/prometheus/pp/go/cppbridge" + "github.com/prometheus/prometheus/pp/go/storage/block" + "github.com/prometheus/prometheus/pp/go/storage/catalog" +) + +//go:generate -command moq go run github.com/matryer/moq --rm --skip-ensure --pkg mock --out +//go:generate moq mock/persistener.go . HeadBlockWriter WriteNotifier +//go:generate moq mock/mediator.go . Mediator +//go:generate moq mock/head_builder.go . HeadBuilder +//go:generate moq mock/head_informer.go . HeadInformer +//go:generate moq mock/rotator_config.go . RotatorConfig + +// +// ActiveHeadContainer +// + +// ActiveHeadContainer container for active [Head], the minimum required [ActiveHeadContainer] implementation. +type ActiveHeadContainer[ + TTask Task, + TShard, TGoShard Shard, + THead Head[TTask, TShard, TGoShard], +] interface { + // With calls fn(h Head). + With(ctx context.Context, fn func(h THead) error) error +} + +// +// Head +// + +// RangeHead the minimum required [Head] implementation. +type RangeHead[TShard Shard] interface { + // RangeShards returns an iterator over the [Head] [Shard]s, through which the shard can be directly accessed. + RangeShards() func(func(TShard) bool) +} + +// Head the minimum required [Head] implementation. +type Head[ + TTask Task, + TShard, TGoShard Shard, +] interface { + // CreateTask create a task for operations on the [Head] shards. + CreateTask(taskName string, shardFn func(shard TGoShard) error) TTask + + // Enqueue the task to be executed on shards [Head]. + Enqueue(t TTask) + + // Generation returns current generation of [Head]. + Generation() uint64 + + // ID returns id [Head]. + ID() string + + // NumberOfShards returns current number of shards in to [Head]. + NumberOfShards() uint16 + + // RangeQueueSize returns an iterator over the [Head] task channels, to collect metrics. + RangeQueueSize() func(func(shardID, size int) bool) + + // RangeShards returns an iterator over the [Head] [Shard]s, through which the shard can be directly accessed. + RangeShards() func(func(TShard) bool) + + // IsReadOnly returns true if the [Head] has switched to read-only. + IsReadOnly() bool + + // SetReadOnly sets the read-only flag for the [Head]. + SetReadOnly() + + // Close closes wals, query semaphore for the inability to get query and clear metrics. + Close() error +} + +// +// HeadBuilder +// + +// HeadBuilder building new [Head] with parameters, the minimum required [HeadBuilder] implementation. +type HeadBuilder[ + TTask Task, + TShard, TGoShard Shard, + THead Head[TTask, TShard, TGoShard], +] interface { + // Build new [Head]. + Build(generation uint64, numberOfShards uint16) (THead, error) +} + +// +// HeadInformer +// + +// HeadInformer sets status by headID in to catalog and get info. +type HeadInformer interface { + // CreatedAt returns the timestamp when the [Record]([Head]) was created. + CreatedAt(headID string) time.Duration + + // SetActiveStatus sets the [catalog.StatusActive] status by headID. + SetActiveStatus(headID string) error + + // SetRotatedStatus sets the [catalog.StatusRotated] status by headID. + SetRotatedStatus(headID string) error +} + +// +// Keeper +// + +// Keeper holds outdated heads until conversion. +type Keeper[ + TTask Task, + TShard, TGoShard Shard, + THead Head[TTask, TShard, TGoShard], +] interface { + // Add the [Head] to the [Keeper] if there is a free slot. + Add(head THead, createdAt time.Duration) error + + // AddWithReplace the [Head] to the [Keeper] with replace if the createdAt is earlier. + AddWithReplace(head THead, createdAt time.Duration) error + + // HasSlot returns the tru if there is a slot in the [Keeper]. + HasSlot() bool + + // Heads returns a slice of the [Head]s stored in the [Keeper]. + Heads() []THead + + // Remove removes [Head]s from the [Keeper]. + Remove(headsForRemove []THead) +} + +// +// Mediator +// + +// Mediator notifies about events via the channel. +type Mediator interface { + // C returns channel with events. + C() <-chan struct{} +} + +// +// ProxyHead +// + +// ProxyHead it proxies requests to the active [Head] and the keeper of old [Head]s. +type ProxyHead[ + TTask Task, + TShard, TGoShard Shard, + THead Head[TTask, TShard, TGoShard], +] interface { + // Add the [Head] to the [Keeper] if there is a free slot. + Add(head THead, createdAt time.Duration) error + + // AddWithReplace the [Head] to the [Keeper] with replace if the createdAt is earlier. + AddWithReplace(head THead, createdAt time.Duration) error + + // Get the active [Head]. + Get() THead + + // HasSlot returns the tru if there is a slot in the [Keeper]. + HasSlot() bool + + // Heads returns a slice of the [Head]s stored in the [Keeper]. + Heads() []THead + + // Remove removes [Head]s from the [Keeper]. + Remove(headsForRemove []THead) + + // Replace the active head [Head] with a new head. + Replace(ctx context.Context, newHead THead) error + + // With calls fn(h Head) on active [Head]. + With(ctx context.Context, fn func(h THead) error) error +} + +// +// Shard +// + +// Shard the minimum required head [Shard] implementation. +type Shard interface { + // DSAllocatedMemory return size of allocated memory for [DataStorage]. + DSAllocatedMemory() uint64 + + // LSSAllocatedMemory return size of allocated memory for labelset storages. + LSSAllocatedMemory() uint64 + + // MergeOutOfOrderChunks merge chunks with out of order data chunks in [DataStorage]. + MergeOutOfOrderChunks() + + // ShardID returns the shard ID. + ShardID() uint16 + + // WalCommit finalize segment from encoder and write to wal. + WalCommit() error + + // WalCurrentSize returns current [Wal] size. + WalCurrentSize() int64 + + // WalFlush flush all contetnt into wal. + WalFlush() error + + // WalSync commits the current contents of the [Wal]. + WalSync() error + + // TimeInterval get time interval of data storage + TimeInterval(bool) cppbridge.TimeInterval + + // UnloadUnusedSeriesData unload unused series data + UnloadUnusedSeriesData() error +} + +// +// Task +// + +// Task the minimum required task [Generic] implementation. +type Task interface { + // Wait for the task to complete on all shards. + Wait() error +} + +// +// WriteNotifier +// + +// WriteNotifier sends a notify that the writing is completed. +type WriteNotifier interface { + // Notify sends a notify that the writing is completed. + Notify() +} + +// +// Loader +// + +// Loader loads [Head] from [Wal]. +type Loader[ + TTask Task, + TShard, TGoShard Shard, + THead Head[TTask, TShard, TGoShard], +] interface { + // Load [Head] from [Wal] by head ID. + Load(headRecord *catalog.Record, generation uint64) (THead, bool) +} + +// +// HeadBlockWriter +// + +// HeadBlockWriter writes block on disk from [Head]. +type HeadBlockWriter[TShard Shard] interface { + Write(shard TShard) ([]block.WrittenBlock, error) +} diff --git a/pp/go/storage/head/services/merger.go b/pp/go/storage/head/services/merger.go new file mode 100644 index 0000000000..11627a7b3f --- /dev/null +++ b/pp/go/storage/head/services/merger.go @@ -0,0 +1,71 @@ +package services + +import ( + "context" + + "github.com/prometheus/prometheus/pp/go/logger" +) + +// +// Merger +// + +// Merger a service that merge chunks with out of order data chunks for [Head]. +type Merger[ + TTask Task, + TShard, TGoShard Shard, + THead Head[TTask, TShard, TGoShard], +] struct { + activeHead ActiveHeadContainer[TTask, TShard, TGoShard, THead] + m Mediator + isNewHead func(headID string) bool +} + +// NewMerger init new [Merger]. +func NewMerger[ + TTask Task, + TShard, TGoShard Shard, + THead Head[TTask, TShard, TGoShard], +]( + activeHead ActiveHeadContainer[TTask, TShard, TGoShard, THead], + m Mediator, + isNewHead func(headID string) bool, +) *Merger[TTask, TShard, TGoShard, THead] { + return &Merger[TTask, TShard, TGoShard, THead]{ + activeHead: activeHead, + m: m, + isNewHead: isNewHead, + } +} + +// Execute starts the [Merger]. +// +//revive:disable-next-line:confusing-naming // other type of Service. +func (s *Merger[TTask, TShard, TGoShard, THead]) Execute(ctx context.Context) error { + logger.Infof("The Merger is running.") + + for range s.m.C() { + _ = s.activeHead.With(ctx, s.UnloadAndMerge) + } + + logger.Infof("The Merger stopped.") + + return nil +} + +// UnloadAndMerge unload unused series data and merge chunks with out of order data chunks for [Head]. +func (s *Merger[TTask, TShard, TGoShard, THead]) UnloadAndMerge(h THead) error { + if s.isNewHead(h.ID()) { + return nil + } + + if err := UnloadUnusedSeriesDataWithHead(h); err != nil { + logger.Errorf("unload unused series data failed: %v", err) + } + + if err := MergeOutOfOrderChunksWithHead(h); err != nil { + logger.Errorf("data storage merge failed: %v", err) + } + + return nil +} diff --git a/pp/go/storage/head/services/merger_test.go b/pp/go/storage/head/services/merger_test.go new file mode 100644 index 0000000000..f24147ccdf --- /dev/null +++ b/pp/go/storage/head/services/merger_test.go @@ -0,0 +1,255 @@ +package services_test + +import ( + "context" + "testing" + + "github.com/prometheus/prometheus/pp/go/cppbridge" + "github.com/prometheus/prometheus/pp/go/storage" + "github.com/prometheus/prometheus/pp/go/storage/head/container" + "github.com/prometheus/prometheus/pp/go/storage/head/head" + "github.com/prometheus/prometheus/pp/go/storage/head/services" + "github.com/prometheus/prometheus/pp/go/storage/head/services/mock" + "github.com/prometheus/prometheus/pp/go/storage/head/shard" + "github.com/prometheus/prometheus/pp/go/storage/head/shard/wal" + "github.com/stretchr/testify/suite" +) + +type MergerSuite struct { + suite.Suite + + baseCtx context.Context +} + +func TestMergerSuite(t *testing.T) { + suite.Run(t, new(MergerSuite)) +} + +func (s *MergerSuite) SetupSuite() { + s.baseCtx = context.Background() +} + +func (s *MergerSuite) createHead( + unloadedFS []*mock.AppendFileMock, + queriedSeriesFS [][2]*mock.StorageFileMock, +) *storage.Head { + shards := make([]*shard.Shard, shardsCount) + for shardID := range unloadedFS { + shards[shardID] = s.createShardOnMemory( + unloadedFS[shardID], + queriedSeriesFS[shardID], + maxSegmentSize, + uint16(shardID), + ) + } + + return head.NewHead( + "test-head-id", + shards, + shard.NewPerGoroutineShard[*storage.Wal], + nil, + 0, + nil, + ) +} + +func (*MergerSuite) createShardOnMemory( + unloadedFS *mock.AppendFileMock, + queriedSeriesFS [2]*mock.StorageFileMock, + maxSegmentSize uint32, + shardID uint16, +) *shard.Shard { + lss := shard.NewLSS() + shardWalEncoder := cppbridge.NewHeadWalEncoder(shardID, 0, lss.Target()) + + segmentWriter := &mock.SegmentWriterMock{ + WriteFunc: func(*cppbridge.HeadEncodedSegment) error { return nil }, + FlushFunc: func() error { return nil }, + SyncFunc: func() error { return nil }, + CloseFunc: func() error { return nil }, + CurrentSizeFunc: func() int64 { return 0 }, + } + + unloadedDataStorage := shard.NewUnloadedDataStorage(unloadedFS) + queriedSeriesStorage := shard.NewQueriedSeriesStorage( + queriedSeriesFS[0], + queriedSeriesFS[1], + ) + + return shard.NewShard( + lss, + shard.NewDataStorage(), + unloadedDataStorage, + queriedSeriesStorage, + wal.NewWal(shardWalEncoder, segmentWriter, maxSegmentSize, shardID, nil), + shardID, + ) +} + +func (s *MergerSuite) TestHappyPath() { + trigger := make(chan struct{}, 1) + start := make(chan struct{}) + mediator := &mock.MediatorMock{ + CFunc: func() <-chan struct{} { + close(start) + return trigger + }, + } + + unloadedFS := make([]*mock.AppendFileMock, shardsCount) + queriedSeriesFS := make([][2]*mock.StorageFileMock, shardsCount) + for shardID := range shardsCount { + unloadedFS[shardID] = &mock.AppendFileMock{ + CloseFunc: func() error { return nil }, + OpenFunc: func() error { return nil }, + SyncFunc: func() error { return nil }, + WriteFunc: func([]byte) (int, error) { return 0, nil }, + } + + queriedSeriesFS[shardID] = [2]*mock.StorageFileMock{ + { + CloseFunc: func() error { return nil }, + OpenFunc: func(int) error { return nil }, + SeekFunc: func(int64, int) (int64, error) { return 0, nil }, + SyncFunc: func() error { return nil }, + TruncateFunc: func(int64) error { return nil }, + WriteFunc: func([]byte) (int, error) { return 0, nil }, + }, + { + CloseFunc: func() error { return nil }, + OpenFunc: func(int) error { return nil }, + SeekFunc: func(int64, int) (int64, error) { return 0, nil }, + SyncFunc: func() error { return nil }, + TruncateFunc: func(int64) error { return nil }, + WriteFunc: func([]byte) (int, error) { return 0, nil }, + }, + } + } + + activeHeadContainer := container.NewWeighted(s.createHead(unloadedFS, queriedSeriesFS)) + isNewHead := func(string) bool { return false } + + merger := services.NewMerger(activeHeadContainer, mediator, isNewHead) + done := make(chan struct{}) + + s.T().Run("execute", func(t *testing.T) { + t.Parallel() + + err := merger.Execute(s.baseCtx) + close(done) + s.NoError(err) + }) + + s.T().Run("tick", func(t *testing.T) { + t.Parallel() + + <-start + trigger <- struct{}{} + close(trigger) + <-done + + s.Require().NoError(activeHeadContainer.Close()) + + for shardID := range unloadedFS { + s.Len(unloadedFS[shardID].OpenCalls(), 1) + s.Len(unloadedFS[shardID].SyncCalls(), 1) + s.Len(unloadedFS[shardID].WriteCalls(), 2) + + s.Len(queriedSeriesFS[shardID][0].OpenCalls(), 1) + s.Len(queriedSeriesFS[shardID][0].SeekCalls(), 1) + s.Len(queriedSeriesFS[shardID][0].SyncCalls(), 1) + s.Len(queriedSeriesFS[shardID][0].TruncateCalls(), 1) + s.Len(queriedSeriesFS[shardID][0].WriteCalls(), 2) + + s.Empty(queriedSeriesFS[shardID][1].OpenCalls()) + s.Empty(queriedSeriesFS[shardID][1].SeekCalls()) + s.Empty(queriedSeriesFS[shardID][1].SyncCalls()) + s.Empty(queriedSeriesFS[shardID][1].TruncateCalls()) + s.Empty(queriedSeriesFS[shardID][1].WriteCalls()) + } + }) +} + +func (s *MergerSuite) TestSkipNewHead() { + trigger := make(chan struct{}, 1) + start := make(chan struct{}) + mediator := &mock.MediatorMock{ + CFunc: func() <-chan struct{} { + close(start) + return trigger + }, + } + + unloadedFS := make([]*mock.AppendFileMock, shardsCount) + queriedSeriesFS := make([][2]*mock.StorageFileMock, shardsCount) + for shardID := range shardsCount { + unloadedFS[shardID] = &mock.AppendFileMock{ + CloseFunc: func() error { return nil }, + OpenFunc: func() error { return nil }, + SyncFunc: func() error { return nil }, + WriteFunc: func([]byte) (int, error) { return 0, nil }, + } + + queriedSeriesFS[shardID] = [2]*mock.StorageFileMock{ + { + CloseFunc: func() error { return nil }, + OpenFunc: func(int) error { return nil }, + SeekFunc: func(int64, int) (int64, error) { return 0, nil }, + SyncFunc: func() error { return nil }, + TruncateFunc: func(int64) error { return nil }, + WriteFunc: func([]byte) (int, error) { return 0, nil }, + }, + { + CloseFunc: func() error { return nil }, + OpenFunc: func(int) error { return nil }, + SeekFunc: func(int64, int) (int64, error) { return 0, nil }, + SyncFunc: func() error { return nil }, + TruncateFunc: func(int64) error { return nil }, + WriteFunc: func([]byte) (int, error) { return 0, nil }, + }, + } + } + + activeHeadContainer := container.NewWeighted(s.createHead(unloadedFS, queriedSeriesFS)) + isNewHead := func(string) bool { return true } + + merger := services.NewMerger(activeHeadContainer, mediator, isNewHead) + done := make(chan struct{}) + + s.T().Run("execute", func(t *testing.T) { + t.Parallel() + + err := merger.Execute(s.baseCtx) + close(done) + s.NoError(err) + }) + + s.T().Run("tick", func(t *testing.T) { + t.Parallel() + + <-start + trigger <- struct{}{} + close(trigger) + <-done + + s.Require().NoError(activeHeadContainer.Close()) + + for shardID := range unloadedFS { + s.Empty(unloadedFS[shardID].OpenCalls()) + s.Empty(unloadedFS[shardID].SyncCalls()) + s.Empty(unloadedFS[shardID].WriteCalls()) + + s.Empty(queriedSeriesFS[shardID][0].OpenCalls()) + s.Empty(queriedSeriesFS[shardID][0].SeekCalls()) + s.Empty(queriedSeriesFS[shardID][0].SyncCalls()) + s.Empty(queriedSeriesFS[shardID][0].TruncateCalls()) + s.Empty(queriedSeriesFS[shardID][0].WriteCalls()) + + s.Empty(queriedSeriesFS[shardID][1].OpenCalls()) + s.Empty(queriedSeriesFS[shardID][1].SeekCalls()) + s.Empty(queriedSeriesFS[shardID][1].SyncCalls()) + s.Empty(queriedSeriesFS[shardID][1].TruncateCalls()) + s.Empty(queriedSeriesFS[shardID][1].WriteCalls()) + } + }) +} diff --git a/pp/go/storage/head/services/metrics_updater.go b/pp/go/storage/head/services/metrics_updater.go new file mode 100644 index 0000000000..9301b7dfec --- /dev/null +++ b/pp/go/storage/head/services/metrics_updater.go @@ -0,0 +1,149 @@ +package services + +import ( + "context" + "strconv" + + "github.com/prometheus/client_golang/prometheus" + + "github.com/prometheus/prometheus/pp/go/logger" + "github.com/prometheus/prometheus/pp/go/util" +) + +// HeadStatus holds information about number of series from [Head]. +type HeadStatus interface { + // NumSeries returns number of series. + NumSeries() uint64 +} + +// +// MetricsUpdater +// + +// MetricsUpdater a service that updates [Head] metrics. +type MetricsUpdater[ + TTask Task, + TShard, TGoShard Shard, + THead Head[TTask, TShard, TGoShard], + THeadStatus HeadStatus, +] struct { + proxyHead ProxyHead[TTask, TShard, TGoShard, THead] + m Mediator + queryHeadStatus func(ctx context.Context, head THead, limit int) (THeadStatus, error) + + // [Head] metrics for an active head. + memoryInUse *prometheus.GaugeVec + series prometheus.Gauge + walSize *prometheus.GaugeVec + queueSize *prometheus.GaugeVec +} + +// NewMetricsUpdater init new [MetricsUpdater]. +func NewMetricsUpdater[ + TTask Task, + TShard, TGoShard Shard, + THead Head[TTask, TShard, TGoShard], + THeadStatus HeadStatus, +]( + proxyHead ProxyHead[TTask, TShard, TGoShard, THead], + m Mediator, + queryHeadStatus func(ctx context.Context, head THead, limit int) (THeadStatus, error), + r prometheus.Registerer, +) *MetricsUpdater[TTask, TShard, TGoShard, THead, THeadStatus] { + factory := util.NewUnconflictRegisterer(r) + return &MetricsUpdater[TTask, TShard, TGoShard, THead, THeadStatus]{ + proxyHead: proxyHead, + m: m, + queryHeadStatus: queryHeadStatus, + + memoryInUse: factory.NewGaugeVec( + prometheus.GaugeOpts{ + Name: "prompp_head_cgo_memory_bytes", + Help: "Current value memory in use in bytes.", + }, + []string{"head_id", "allocator", "shard_id"}, + ), + series: factory.NewGauge(prometheus.GaugeOpts{ + Name: "prompp_head_series", + Help: "Total number of series in the heads block.", + }), + walSize: factory.NewGaugeVec( + prometheus.GaugeOpts{ + Name: "prompp_head_current_wal_size", + Help: "The size of the wall of the current head.", + }, + []string{"shard_id"}, + ), + queueSize: factory.NewGaugeVec( + prometheus.GaugeOpts{ + Name: "prompp_head_queue_tasks_size", + Help: "The size of the queue of tasks of the current head.", + }, + []string{"shard_id"}, + ), + } +} + +// Execute starts the [MetricsUpdater]. +// +//revive:disable-next-line:confusing-naming // other type of Service. +func (s *MetricsUpdater[TTask, TShard, TGoShard, THead, THeadStatus]) Execute(ctx context.Context) error { + logger.Infof("The MetricsUpdater is running.") + + for range s.m.C() { + s.collect(ctx) + } + + logger.Infof("The MetricsUpdater stopped.") + + return nil +} + +// collect metrics from the head. +func (s *MetricsUpdater[TTask, TShard, TGoShard, THead, THeadStatus]) collect(ctx context.Context) { + ahead := s.proxyHead.Get() + + status, err := s.queryHeadStatus(ctx, ahead, 0) + if err != nil { + // error may be only head is rotated, skip + return + } + + s.series.Set(float64(status.NumSeries())) + + for shardID, size := range ahead.RangeQueueSize() { + s.queueSize.With(prometheus.Labels{"shard_id": strconv.Itoa(shardID)}).Set(float64(size)) + } + + s.collectFromShards(ahead, true) + + for _, head := range s.proxyHead.Heads() { + if head.ID() == ahead.ID() { + continue + } + + s.collectFromShards(head, false) + } +} + +// fromShards collects metrics from the head's shards. +// +//revive:disable-next-line:flag-parameter this is a flag, but it's more convenient this way +func (s *MetricsUpdater[TTask, TShard, TGoShard, THead, THeadStatus]) collectFromShards(head THead, active bool) { + headID := head.ID() + for shard := range head.RangeShards() { + ls := make(prometheus.Labels, 3) //revive:disable-line:add-constant it's labels count + + ls["shard_id"] = strconv.FormatUint(uint64(shard.ShardID()), 10) //revive:disable-line:add-constant it's base 10 + if active { + s.walSize.With(ls).Set(float64(shard.WalCurrentSize())) + } + + ls["head_id"] = headID + ls["allocator"] = "data_storage" + s.memoryInUse.With(ls).Set(float64(shard.DSAllocatedMemory())) + + ls["allocator"] = "main_lss" + s.memoryInUse.With(ls).Set(float64(shard.LSSAllocatedMemory())) + } +} diff --git a/pp/go/storage/head/services/metrics_updater_test.go b/pp/go/storage/head/services/metrics_updater_test.go new file mode 100644 index 0000000000..0ddb682a4c --- /dev/null +++ b/pp/go/storage/head/services/metrics_updater_test.go @@ -0,0 +1,26 @@ +package services + +import ( + "context" + "testing" + + "github.com/stretchr/testify/suite" +) + +type MetricsUpdaterSuite struct { + suite.Suite + + baseCtx context.Context +} + +func TestMetricsUpdaterSuite(t *testing.T) { + suite.Run(t, new(MetricsUpdaterSuite)) +} + +func (s *MetricsUpdaterSuite) SetupSuite() { + s.baseCtx = context.Background() +} + +func (s *MetricsUpdaterSuite) TestHappyPath() { + s.T().Log("MetricsUpdaterSuite TestHappyPath TODO") +} diff --git a/pp/go/storage/head/services/mock/append_file.go b/pp/go/storage/head/services/mock/append_file.go new file mode 100644 index 0000000000..5d7e206372 --- /dev/null +++ b/pp/go/storage/head/services/mock/append_file.go @@ -0,0 +1,256 @@ +// Code generated by moq; DO NOT EDIT. +// github.com/matryer/moq + +package mock + +import ( + "github.com/prometheus/prometheus/pp/go/storage/head/shard" + "sync" +) + +// AppendFileMock is a mock implementation of AppendFile. +// +// func TestSomethingThatUsesAppendFile(t *testing.T) { +// +// // make and configure a mocked AppendFile +// mockedAppendFile := &AppendFileMock{ +// CloseFunc: func() error { +// panic("mock out the Close method") +// }, +// IsEmptyFunc: func() bool { +// panic("mock out the IsEmpty method") +// }, +// OpenFunc: func() error { +// panic("mock out the Open method") +// }, +// ReaderFunc: func() (shard.StorageReader, error) { +// panic("mock out the Reader method") +// }, +// SyncFunc: func() error { +// panic("mock out the Sync method") +// }, +// WriteFunc: func(p []byte) (int, error) { +// panic("mock out the Write method") +// }, +// } +// +// // use mockedAppendFile in code that requires AppendFile +// // and then make assertions. +// +// } +type AppendFileMock struct { + // CloseFunc mocks the Close method. + CloseFunc func() error + + // IsEmptyFunc mocks the IsEmpty method. + IsEmptyFunc func() bool + + // OpenFunc mocks the Open method. + OpenFunc func() error + + // ReaderFunc mocks the Reader method. + ReaderFunc func() (shard.StorageReader, error) + + // SyncFunc mocks the Sync method. + SyncFunc func() error + + // WriteFunc mocks the Write method. + WriteFunc func(p []byte) (int, error) + + // calls tracks calls to the methods. + calls struct { + // Close holds details about calls to the Close method. + Close []struct { + } + // IsEmpty holds details about calls to the IsEmpty method. + IsEmpty []struct { + } + // Open holds details about calls to the Open method. + Open []struct { + } + // Reader holds details about calls to the Reader method. + Reader []struct { + } + // Sync holds details about calls to the Sync method. + Sync []struct { + } + // Write holds details about calls to the Write method. + Write []struct { + // P is the p argument value. + P []byte + } + } + lockClose sync.RWMutex + lockIsEmpty sync.RWMutex + lockOpen sync.RWMutex + lockReader sync.RWMutex + lockSync sync.RWMutex + lockWrite sync.RWMutex +} + +// Close calls CloseFunc. +func (mock *AppendFileMock) Close() error { + if mock.CloseFunc == nil { + panic("AppendFileMock.CloseFunc: method is nil but AppendFile.Close was just called") + } + callInfo := struct { + }{} + mock.lockClose.Lock() + mock.calls.Close = append(mock.calls.Close, callInfo) + mock.lockClose.Unlock() + return mock.CloseFunc() +} + +// CloseCalls gets all the calls that were made to Close. +// Check the length with: +// +// len(mockedAppendFile.CloseCalls()) +func (mock *AppendFileMock) CloseCalls() []struct { +} { + var calls []struct { + } + mock.lockClose.RLock() + calls = mock.calls.Close + mock.lockClose.RUnlock() + return calls +} + +// IsEmpty calls IsEmptyFunc. +func (mock *AppendFileMock) IsEmpty() bool { + if mock.IsEmptyFunc == nil { + panic("AppendFileMock.IsEmptyFunc: method is nil but AppendFile.IsEmpty was just called") + } + callInfo := struct { + }{} + mock.lockIsEmpty.Lock() + mock.calls.IsEmpty = append(mock.calls.IsEmpty, callInfo) + mock.lockIsEmpty.Unlock() + return mock.IsEmptyFunc() +} + +// IsEmptyCalls gets all the calls that were made to IsEmpty. +// Check the length with: +// +// len(mockedAppendFile.IsEmptyCalls()) +func (mock *AppendFileMock) IsEmptyCalls() []struct { +} { + var calls []struct { + } + mock.lockIsEmpty.RLock() + calls = mock.calls.IsEmpty + mock.lockIsEmpty.RUnlock() + return calls +} + +// Open calls OpenFunc. +func (mock *AppendFileMock) Open() error { + if mock.OpenFunc == nil { + panic("AppendFileMock.OpenFunc: method is nil but AppendFile.Open was just called") + } + callInfo := struct { + }{} + mock.lockOpen.Lock() + mock.calls.Open = append(mock.calls.Open, callInfo) + mock.lockOpen.Unlock() + return mock.OpenFunc() +} + +// OpenCalls gets all the calls that were made to Open. +// Check the length with: +// +// len(mockedAppendFile.OpenCalls()) +func (mock *AppendFileMock) OpenCalls() []struct { +} { + var calls []struct { + } + mock.lockOpen.RLock() + calls = mock.calls.Open + mock.lockOpen.RUnlock() + return calls +} + +// Reader calls ReaderFunc. +func (mock *AppendFileMock) Reader() (shard.StorageReader, error) { + if mock.ReaderFunc == nil { + panic("AppendFileMock.ReaderFunc: method is nil but AppendFile.Reader was just called") + } + callInfo := struct { + }{} + mock.lockReader.Lock() + mock.calls.Reader = append(mock.calls.Reader, callInfo) + mock.lockReader.Unlock() + return mock.ReaderFunc() +} + +// ReaderCalls gets all the calls that were made to Reader. +// Check the length with: +// +// len(mockedAppendFile.ReaderCalls()) +func (mock *AppendFileMock) ReaderCalls() []struct { +} { + var calls []struct { + } + mock.lockReader.RLock() + calls = mock.calls.Reader + mock.lockReader.RUnlock() + return calls +} + +// Sync calls SyncFunc. +func (mock *AppendFileMock) Sync() error { + if mock.SyncFunc == nil { + panic("AppendFileMock.SyncFunc: method is nil but AppendFile.Sync was just called") + } + callInfo := struct { + }{} + mock.lockSync.Lock() + mock.calls.Sync = append(mock.calls.Sync, callInfo) + mock.lockSync.Unlock() + return mock.SyncFunc() +} + +// SyncCalls gets all the calls that were made to Sync. +// Check the length with: +// +// len(mockedAppendFile.SyncCalls()) +func (mock *AppendFileMock) SyncCalls() []struct { +} { + var calls []struct { + } + mock.lockSync.RLock() + calls = mock.calls.Sync + mock.lockSync.RUnlock() + return calls +} + +// Write calls WriteFunc. +func (mock *AppendFileMock) Write(p []byte) (int, error) { + if mock.WriteFunc == nil { + panic("AppendFileMock.WriteFunc: method is nil but AppendFile.Write was just called") + } + callInfo := struct { + P []byte + }{ + P: p, + } + mock.lockWrite.Lock() + mock.calls.Write = append(mock.calls.Write, callInfo) + mock.lockWrite.Unlock() + return mock.WriteFunc(p) +} + +// WriteCalls gets all the calls that were made to Write. +// Check the length with: +// +// len(mockedAppendFile.WriteCalls()) +func (mock *AppendFileMock) WriteCalls() []struct { + P []byte +} { + var calls []struct { + P []byte + } + mock.lockWrite.RLock() + calls = mock.calls.Write + mock.lockWrite.RUnlock() + return calls +} diff --git a/pp/go/storage/head/services/mock/file_storage.go b/pp/go/storage/head/services/mock/file_storage.go new file mode 100644 index 0000000000..ed97aae54e --- /dev/null +++ b/pp/go/storage/head/services/mock/file_storage.go @@ -0,0 +1,413 @@ +// Code generated by moq; DO NOT EDIT. +// github.com/matryer/moq + +package mock + +import ( + "sync" +) + +// StorageFileMock is a mock implementation of StorageFile. +// +// func TestSomethingThatUsesStorageFile(t *testing.T) { +// +// // make and configure a mocked StorageFile +// mockedStorageFile := &StorageFileMock{ +// CloseFunc: func() error { +// panic("mock out the Close method") +// }, +// IsEmptyFunc: func() bool { +// panic("mock out the IsEmpty method") +// }, +// OpenFunc: func(flags int) error { +// panic("mock out the Open method") +// }, +// ReadFunc: func(p []byte) (int, error) { +// panic("mock out the Read method") +// }, +// ReadAtFunc: func(p []byte, off int64) (int, error) { +// panic("mock out the ReadAt method") +// }, +// SeekFunc: func(offset int64, whence int) (int64, error) { +// panic("mock out the Seek method") +// }, +// SyncFunc: func() error { +// panic("mock out the Sync method") +// }, +// TruncateFunc: func(size int64) error { +// panic("mock out the Truncate method") +// }, +// WriteFunc: func(p []byte) (int, error) { +// panic("mock out the Write method") +// }, +// } +// +// // use mockedStorageFile in code that requires StorageFile +// // and then make assertions. +// +// } +type StorageFileMock struct { + // CloseFunc mocks the Close method. + CloseFunc func() error + + // IsEmptyFunc mocks the IsEmpty method. + IsEmptyFunc func() bool + + // OpenFunc mocks the Open method. + OpenFunc func(flags int) error + + // ReadFunc mocks the Read method. + ReadFunc func(p []byte) (int, error) + + // ReadAtFunc mocks the ReadAt method. + ReadAtFunc func(p []byte, off int64) (int, error) + + // SeekFunc mocks the Seek method. + SeekFunc func(offset int64, whence int) (int64, error) + + // SyncFunc mocks the Sync method. + SyncFunc func() error + + // TruncateFunc mocks the Truncate method. + TruncateFunc func(size int64) error + + // WriteFunc mocks the Write method. + WriteFunc func(p []byte) (int, error) + + // calls tracks calls to the methods. + calls struct { + // Close holds details about calls to the Close method. + Close []struct { + } + // IsEmpty holds details about calls to the IsEmpty method. + IsEmpty []struct { + } + // Open holds details about calls to the Open method. + Open []struct { + // Flags is the flags argument value. + Flags int + } + // Read holds details about calls to the Read method. + Read []struct { + // P is the p argument value. + P []byte + } + // ReadAt holds details about calls to the ReadAt method. + ReadAt []struct { + // P is the p argument value. + P []byte + // Off is the off argument value. + Off int64 + } + // Seek holds details about calls to the Seek method. + Seek []struct { + // Offset is the offset argument value. + Offset int64 + // Whence is the whence argument value. + Whence int + } + // Sync holds details about calls to the Sync method. + Sync []struct { + } + // Truncate holds details about calls to the Truncate method. + Truncate []struct { + // Size is the size argument value. + Size int64 + } + // Write holds details about calls to the Write method. + Write []struct { + // P is the p argument value. + P []byte + } + } + lockClose sync.RWMutex + lockIsEmpty sync.RWMutex + lockOpen sync.RWMutex + lockRead sync.RWMutex + lockReadAt sync.RWMutex + lockSeek sync.RWMutex + lockSync sync.RWMutex + lockTruncate sync.RWMutex + lockWrite sync.RWMutex +} + +// Close calls CloseFunc. +func (mock *StorageFileMock) Close() error { + if mock.CloseFunc == nil { + panic("StorageFileMock.CloseFunc: method is nil but StorageFile.Close was just called") + } + callInfo := struct { + }{} + mock.lockClose.Lock() + mock.calls.Close = append(mock.calls.Close, callInfo) + mock.lockClose.Unlock() + return mock.CloseFunc() +} + +// CloseCalls gets all the calls that were made to Close. +// Check the length with: +// +// len(mockedStorageFile.CloseCalls()) +func (mock *StorageFileMock) CloseCalls() []struct { +} { + var calls []struct { + } + mock.lockClose.RLock() + calls = mock.calls.Close + mock.lockClose.RUnlock() + return calls +} + +// IsEmpty calls IsEmptyFunc. +func (mock *StorageFileMock) IsEmpty() bool { + if mock.IsEmptyFunc == nil { + panic("StorageFileMock.IsEmptyFunc: method is nil but StorageFile.IsEmpty was just called") + } + callInfo := struct { + }{} + mock.lockIsEmpty.Lock() + mock.calls.IsEmpty = append(mock.calls.IsEmpty, callInfo) + mock.lockIsEmpty.Unlock() + return mock.IsEmptyFunc() +} + +// IsEmptyCalls gets all the calls that were made to IsEmpty. +// Check the length with: +// +// len(mockedStorageFile.IsEmptyCalls()) +func (mock *StorageFileMock) IsEmptyCalls() []struct { +} { + var calls []struct { + } + mock.lockIsEmpty.RLock() + calls = mock.calls.IsEmpty + mock.lockIsEmpty.RUnlock() + return calls +} + +// Open calls OpenFunc. +func (mock *StorageFileMock) Open(flags int) error { + if mock.OpenFunc == nil { + panic("StorageFileMock.OpenFunc: method is nil but StorageFile.Open was just called") + } + callInfo := struct { + Flags int + }{ + Flags: flags, + } + mock.lockOpen.Lock() + mock.calls.Open = append(mock.calls.Open, callInfo) + mock.lockOpen.Unlock() + return mock.OpenFunc(flags) +} + +// OpenCalls gets all the calls that were made to Open. +// Check the length with: +// +// len(mockedStorageFile.OpenCalls()) +func (mock *StorageFileMock) OpenCalls() []struct { + Flags int +} { + var calls []struct { + Flags int + } + mock.lockOpen.RLock() + calls = mock.calls.Open + mock.lockOpen.RUnlock() + return calls +} + +// Read calls ReadFunc. +func (mock *StorageFileMock) Read(p []byte) (int, error) { + if mock.ReadFunc == nil { + panic("StorageFileMock.ReadFunc: method is nil but StorageFile.Read was just called") + } + callInfo := struct { + P []byte + }{ + P: p, + } + mock.lockRead.Lock() + mock.calls.Read = append(mock.calls.Read, callInfo) + mock.lockRead.Unlock() + return mock.ReadFunc(p) +} + +// ReadCalls gets all the calls that were made to Read. +// Check the length with: +// +// len(mockedStorageFile.ReadCalls()) +func (mock *StorageFileMock) ReadCalls() []struct { + P []byte +} { + var calls []struct { + P []byte + } + mock.lockRead.RLock() + calls = mock.calls.Read + mock.lockRead.RUnlock() + return calls +} + +// ReadAt calls ReadAtFunc. +func (mock *StorageFileMock) ReadAt(p []byte, off int64) (int, error) { + if mock.ReadAtFunc == nil { + panic("StorageFileMock.ReadAtFunc: method is nil but StorageFile.ReadAt was just called") + } + callInfo := struct { + P []byte + Off int64 + }{ + P: p, + Off: off, + } + mock.lockReadAt.Lock() + mock.calls.ReadAt = append(mock.calls.ReadAt, callInfo) + mock.lockReadAt.Unlock() + return mock.ReadAtFunc(p, off) +} + +// ReadAtCalls gets all the calls that were made to ReadAt. +// Check the length with: +// +// len(mockedStorageFile.ReadAtCalls()) +func (mock *StorageFileMock) ReadAtCalls() []struct { + P []byte + Off int64 +} { + var calls []struct { + P []byte + Off int64 + } + mock.lockReadAt.RLock() + calls = mock.calls.ReadAt + mock.lockReadAt.RUnlock() + return calls +} + +// Seek calls SeekFunc. +func (mock *StorageFileMock) Seek(offset int64, whence int) (int64, error) { + if mock.SeekFunc == nil { + panic("StorageFileMock.SeekFunc: method is nil but StorageFile.Seek was just called") + } + callInfo := struct { + Offset int64 + Whence int + }{ + Offset: offset, + Whence: whence, + } + mock.lockSeek.Lock() + mock.calls.Seek = append(mock.calls.Seek, callInfo) + mock.lockSeek.Unlock() + return mock.SeekFunc(offset, whence) +} + +// SeekCalls gets all the calls that were made to Seek. +// Check the length with: +// +// len(mockedStorageFile.SeekCalls()) +func (mock *StorageFileMock) SeekCalls() []struct { + Offset int64 + Whence int +} { + var calls []struct { + Offset int64 + Whence int + } + mock.lockSeek.RLock() + calls = mock.calls.Seek + mock.lockSeek.RUnlock() + return calls +} + +// Sync calls SyncFunc. +func (mock *StorageFileMock) Sync() error { + if mock.SyncFunc == nil { + panic("StorageFileMock.SyncFunc: method is nil but StorageFile.Sync was just called") + } + callInfo := struct { + }{} + mock.lockSync.Lock() + mock.calls.Sync = append(mock.calls.Sync, callInfo) + mock.lockSync.Unlock() + return mock.SyncFunc() +} + +// SyncCalls gets all the calls that were made to Sync. +// Check the length with: +// +// len(mockedStorageFile.SyncCalls()) +func (mock *StorageFileMock) SyncCalls() []struct { +} { + var calls []struct { + } + mock.lockSync.RLock() + calls = mock.calls.Sync + mock.lockSync.RUnlock() + return calls +} + +// Truncate calls TruncateFunc. +func (mock *StorageFileMock) Truncate(size int64) error { + if mock.TruncateFunc == nil { + panic("StorageFileMock.TruncateFunc: method is nil but StorageFile.Truncate was just called") + } + callInfo := struct { + Size int64 + }{ + Size: size, + } + mock.lockTruncate.Lock() + mock.calls.Truncate = append(mock.calls.Truncate, callInfo) + mock.lockTruncate.Unlock() + return mock.TruncateFunc(size) +} + +// TruncateCalls gets all the calls that were made to Truncate. +// Check the length with: +// +// len(mockedStorageFile.TruncateCalls()) +func (mock *StorageFileMock) TruncateCalls() []struct { + Size int64 +} { + var calls []struct { + Size int64 + } + mock.lockTruncate.RLock() + calls = mock.calls.Truncate + mock.lockTruncate.RUnlock() + return calls +} + +// Write calls WriteFunc. +func (mock *StorageFileMock) Write(p []byte) (int, error) { + if mock.WriteFunc == nil { + panic("StorageFileMock.WriteFunc: method is nil but StorageFile.Write was just called") + } + callInfo := struct { + P []byte + }{ + P: p, + } + mock.lockWrite.Lock() + mock.calls.Write = append(mock.calls.Write, callInfo) + mock.lockWrite.Unlock() + return mock.WriteFunc(p) +} + +// WriteCalls gets all the calls that were made to Write. +// Check the length with: +// +// len(mockedStorageFile.WriteCalls()) +func (mock *StorageFileMock) WriteCalls() []struct { + P []byte +} { + var calls []struct { + P []byte + } + mock.lockWrite.RLock() + calls = mock.calls.Write + mock.lockWrite.RUnlock() + return calls +} diff --git a/pp/go/storage/head/services/mock/head_builder.go b/pp/go/storage/head/services/mock/head_builder.go new file mode 100644 index 0000000000..ce96e1e3f4 --- /dev/null +++ b/pp/go/storage/head/services/mock/head_builder.go @@ -0,0 +1,77 @@ +// Code generated by moq; DO NOT EDIT. +// github.com/matryer/moq + +package mock + +import ( + "github.com/prometheus/prometheus/pp/go/storage/head/services" + "sync" +) + +// HeadBuilderMock is a mock implementation of services.HeadBuilder. +// +// func TestSomethingThatUsesHeadBuilder(t *testing.T) { +// +// // make and configure a mocked services.HeadBuilder +// mockedHeadBuilder := &HeadBuilderMock{ +// BuildFunc: func(generation uint64, numberOfShards uint16) (THead, error) { +// panic("mock out the Build method") +// }, +// } +// +// // use mockedHeadBuilder in code that requires services.HeadBuilder +// // and then make assertions. +// +// } +type HeadBuilderMock[TTask services.Task, TShard services.Shard, TGoShard services.Shard, THead services.Head[TTask, TShard, TGoShard]] struct { + // BuildFunc mocks the Build method. + BuildFunc func(generation uint64, numberOfShards uint16) (THead, error) + + // calls tracks calls to the methods. + calls struct { + // Build holds details about calls to the Build method. + Build []struct { + // Generation is the generation argument value. + Generation uint64 + // NumberOfShards is the numberOfShards argument value. + NumberOfShards uint16 + } + } + lockBuild sync.RWMutex +} + +// Build calls BuildFunc. +func (mock *HeadBuilderMock[TTask, TShard, TGoShard, THead]) Build(generation uint64, numberOfShards uint16) (THead, error) { + if mock.BuildFunc == nil { + panic("HeadBuilderMock.BuildFunc: method is nil but HeadBuilder.Build was just called") + } + callInfo := struct { + Generation uint64 + NumberOfShards uint16 + }{ + Generation: generation, + NumberOfShards: numberOfShards, + } + mock.lockBuild.Lock() + mock.calls.Build = append(mock.calls.Build, callInfo) + mock.lockBuild.Unlock() + return mock.BuildFunc(generation, numberOfShards) +} + +// BuildCalls gets all the calls that were made to Build. +// Check the length with: +// +// len(mockedHeadBuilder.BuildCalls()) +func (mock *HeadBuilderMock[TTask, TShard, TGoShard, THead]) BuildCalls() []struct { + Generation uint64 + NumberOfShards uint16 +} { + var calls []struct { + Generation uint64 + NumberOfShards uint16 + } + mock.lockBuild.RLock() + calls = mock.calls.Build + mock.lockBuild.RUnlock() + return calls +} diff --git a/pp/go/storage/head/services/mock/head_informer.go b/pp/go/storage/head/services/mock/head_informer.go new file mode 100644 index 0000000000..6f26894910 --- /dev/null +++ b/pp/go/storage/head/services/mock/head_informer.go @@ -0,0 +1,159 @@ +// Code generated by moq; DO NOT EDIT. +// github.com/matryer/moq + +package mock + +import ( + "sync" + "time" +) + +// HeadInformerMock is a mock implementation of services.HeadInformer. +// +// func TestSomethingThatUsesHeadInformer(t *testing.T) { +// +// // make and configure a mocked services.HeadInformer +// mockedHeadInformer := &HeadInformerMock{ +// CreatedAtFunc: func(headID string) time.Duration { +// panic("mock out the CreatedAt method") +// }, +// SetActiveStatusFunc: func(headID string) error { +// panic("mock out the SetActiveStatus method") +// }, +// SetRotatedStatusFunc: func(headID string) error { +// panic("mock out the SetRotatedStatus method") +// }, +// } +// +// // use mockedHeadInformer in code that requires services.HeadInformer +// // and then make assertions. +// +// } +type HeadInformerMock struct { + // CreatedAtFunc mocks the CreatedAt method. + CreatedAtFunc func(headID string) time.Duration + + // SetActiveStatusFunc mocks the SetActiveStatus method. + SetActiveStatusFunc func(headID string) error + + // SetRotatedStatusFunc mocks the SetRotatedStatus method. + SetRotatedStatusFunc func(headID string) error + + // calls tracks calls to the methods. + calls struct { + // CreatedAt holds details about calls to the CreatedAt method. + CreatedAt []struct { + // HeadID is the headID argument value. + HeadID string + } + // SetActiveStatus holds details about calls to the SetActiveStatus method. + SetActiveStatus []struct { + // HeadID is the headID argument value. + HeadID string + } + // SetRotatedStatus holds details about calls to the SetRotatedStatus method. + SetRotatedStatus []struct { + // HeadID is the headID argument value. + HeadID string + } + } + lockCreatedAt sync.RWMutex + lockSetActiveStatus sync.RWMutex + lockSetRotatedStatus sync.RWMutex +} + +// CreatedAt calls CreatedAtFunc. +func (mock *HeadInformerMock) CreatedAt(headID string) time.Duration { + if mock.CreatedAtFunc == nil { + panic("HeadInformerMock.CreatedAtFunc: method is nil but HeadInformer.CreatedAt was just called") + } + callInfo := struct { + HeadID string + }{ + HeadID: headID, + } + mock.lockCreatedAt.Lock() + mock.calls.CreatedAt = append(mock.calls.CreatedAt, callInfo) + mock.lockCreatedAt.Unlock() + return mock.CreatedAtFunc(headID) +} + +// CreatedAtCalls gets all the calls that were made to CreatedAt. +// Check the length with: +// +// len(mockedHeadInformer.CreatedAtCalls()) +func (mock *HeadInformerMock) CreatedAtCalls() []struct { + HeadID string +} { + var calls []struct { + HeadID string + } + mock.lockCreatedAt.RLock() + calls = mock.calls.CreatedAt + mock.lockCreatedAt.RUnlock() + return calls +} + +// SetActiveStatus calls SetActiveStatusFunc. +func (mock *HeadInformerMock) SetActiveStatus(headID string) error { + if mock.SetActiveStatusFunc == nil { + panic("HeadInformerMock.SetActiveStatusFunc: method is nil but HeadInformer.SetActiveStatus was just called") + } + callInfo := struct { + HeadID string + }{ + HeadID: headID, + } + mock.lockSetActiveStatus.Lock() + mock.calls.SetActiveStatus = append(mock.calls.SetActiveStatus, callInfo) + mock.lockSetActiveStatus.Unlock() + return mock.SetActiveStatusFunc(headID) +} + +// SetActiveStatusCalls gets all the calls that were made to SetActiveStatus. +// Check the length with: +// +// len(mockedHeadInformer.SetActiveStatusCalls()) +func (mock *HeadInformerMock) SetActiveStatusCalls() []struct { + HeadID string +} { + var calls []struct { + HeadID string + } + mock.lockSetActiveStatus.RLock() + calls = mock.calls.SetActiveStatus + mock.lockSetActiveStatus.RUnlock() + return calls +} + +// SetRotatedStatus calls SetRotatedStatusFunc. +func (mock *HeadInformerMock) SetRotatedStatus(headID string) error { + if mock.SetRotatedStatusFunc == nil { + panic("HeadInformerMock.SetRotatedStatusFunc: method is nil but HeadInformer.SetRotatedStatus was just called") + } + callInfo := struct { + HeadID string + }{ + HeadID: headID, + } + mock.lockSetRotatedStatus.Lock() + mock.calls.SetRotatedStatus = append(mock.calls.SetRotatedStatus, callInfo) + mock.lockSetRotatedStatus.Unlock() + return mock.SetRotatedStatusFunc(headID) +} + +// SetRotatedStatusCalls gets all the calls that were made to SetRotatedStatus. +// Check the length with: +// +// len(mockedHeadInformer.SetRotatedStatusCalls()) +func (mock *HeadInformerMock) SetRotatedStatusCalls() []struct { + HeadID string +} { + var calls []struct { + HeadID string + } + mock.lockSetRotatedStatus.RLock() + calls = mock.calls.SetRotatedStatus + mock.lockSetRotatedStatus.RUnlock() + return calls +} diff --git a/pp/go/storage/head/services/mock/mediator.go b/pp/go/storage/head/services/mock/mediator.go new file mode 100644 index 0000000000..ed0357c916 --- /dev/null +++ b/pp/go/storage/head/services/mock/mediator.go @@ -0,0 +1,63 @@ +// Code generated by moq; DO NOT EDIT. +// github.com/matryer/moq + +package mock + +import ( + "sync" +) + +// MediatorMock is a mock implementation of services.Mediator. +// +// func TestSomethingThatUsesMediator(t *testing.T) { +// +// // make and configure a mocked services.Mediator +// mockedMediator := &MediatorMock{ +// CFunc: func() <-chan struct{} { +// panic("mock out the C method") +// }, +// } +// +// // use mockedMediator in code that requires services.Mediator +// // and then make assertions. +// +// } +type MediatorMock struct { + // CFunc mocks the C method. + CFunc func() <-chan struct{} + + // calls tracks calls to the methods. + calls struct { + // C holds details about calls to the C method. + C []struct { + } + } + lockC sync.RWMutex +} + +// C calls CFunc. +func (mock *MediatorMock) C() <-chan struct{} { + if mock.CFunc == nil { + panic("MediatorMock.CFunc: method is nil but Mediator.C was just called") + } + callInfo := struct { + }{} + mock.lockC.Lock() + mock.calls.C = append(mock.calls.C, callInfo) + mock.lockC.Unlock() + return mock.CFunc() +} + +// CCalls gets all the calls that were made to C. +// Check the length with: +// +// len(mockedMediator.CCalls()) +func (mock *MediatorMock) CCalls() []struct { +} { + var calls []struct { + } + mock.lockC.RLock() + calls = mock.calls.C + mock.lockC.RUnlock() + return calls +} diff --git a/pp/go/storage/head/services/mock/mock.go b/pp/go/storage/head/services/mock/mock.go new file mode 100644 index 0000000000..de6d023ef2 --- /dev/null +++ b/pp/go/storage/head/services/mock/mock.go @@ -0,0 +1,26 @@ +package mock + +import ( + "github.com/prometheus/prometheus/pp/go/cppbridge" + "github.com/prometheus/prometheus/pp/go/storage/head/keeper" + "github.com/prometheus/prometheus/pp/go/storage/head/shard" + "github.com/prometheus/prometheus/pp/go/storage/head/shard/wal" +) + +//go:generate -command moq go run github.com/matryer/moq --rm --skip-ensure --pkg mock --out +//go:generate moq segment_writer.go . SegmentWriter +//go:generate moq file_storage.go . StorageFile +//go:generate moq append_file.go . AppendFile +//go:generate moq removed_head_notifier.go . RemovedHeadNotifier + +// SegmentWriter alias for [wal.SegmentWriter] with [cppbridge.HeadEncodedSegment]. +type SegmentWriter = wal.SegmentWriter[*cppbridge.HeadEncodedSegment] + +// StorageFile alias for [shard.StorageFile]. +type StorageFile = shard.StorageFile + +// AppendFile alias for [shard.AppendFile]. +type AppendFile = shard.AppendFile + +// RemovedHeadNotifier alias for [keeper.RemovedHeadNotifier]. +type RemovedHeadNotifier = keeper.RemovedHeadNotifier diff --git a/pp/go/storage/head/services/mock/persistener.go b/pp/go/storage/head/services/mock/persistener.go new file mode 100644 index 0000000000..cf53f58912 --- /dev/null +++ b/pp/go/storage/head/services/mock/persistener.go @@ -0,0 +1,127 @@ +// Code generated by moq; DO NOT EDIT. +// github.com/matryer/moq + +package mock + +import ( + "github.com/prometheus/prometheus/pp/go/storage/block" + "github.com/prometheus/prometheus/pp/go/storage/head/services" + "sync" +) + +// HeadBlockWriterMock is a mock implementation of services.HeadBlockWriter. +// +// func TestSomethingThatUsesHeadBlockWriter(t *testing.T) { +// +// // make and configure a mocked services.HeadBlockWriter +// mockedHeadBlockWriter := &HeadBlockWriterMock{ +// WriteFunc: func(shard TShard) ([]block.WrittenBlock, error) { +// panic("mock out the Write method") +// }, +// } +// +// // use mockedHeadBlockWriter in code that requires services.HeadBlockWriter +// // and then make assertions. +// +// } +type HeadBlockWriterMock[TShard services.Shard] struct { + // WriteFunc mocks the Write method. + WriteFunc func(shard TShard) ([]block.WrittenBlock, error) + + // calls tracks calls to the methods. + calls struct { + // Write holds details about calls to the Write method. + Write []struct { + // Shard is the shard argument value. + Shard TShard + } + } + lockWrite sync.RWMutex +} + +// Write calls WriteFunc. +func (mock *HeadBlockWriterMock[TShard]) Write(shard TShard) ([]block.WrittenBlock, error) { + if mock.WriteFunc == nil { + panic("HeadBlockWriterMock.WriteFunc: method is nil but HeadBlockWriter.Write was just called") + } + callInfo := struct { + Shard TShard + }{ + Shard: shard, + } + mock.lockWrite.Lock() + mock.calls.Write = append(mock.calls.Write, callInfo) + mock.lockWrite.Unlock() + return mock.WriteFunc(shard) +} + +// WriteCalls gets all the calls that were made to Write. +// Check the length with: +// +// len(mockedHeadBlockWriter.WriteCalls()) +func (mock *HeadBlockWriterMock[TShard]) WriteCalls() []struct { + Shard TShard +} { + var calls []struct { + Shard TShard + } + mock.lockWrite.RLock() + calls = mock.calls.Write + mock.lockWrite.RUnlock() + return calls +} + +// WriteNotifierMock is a mock implementation of services.WriteNotifier. +// +// func TestSomethingThatUsesWriteNotifier(t *testing.T) { +// +// // make and configure a mocked services.WriteNotifier +// mockedWriteNotifier := &WriteNotifierMock{ +// NotifyFunc: func() { +// panic("mock out the Notify method") +// }, +// } +// +// // use mockedWriteNotifier in code that requires services.WriteNotifier +// // and then make assertions. +// +// } +type WriteNotifierMock struct { + // NotifyFunc mocks the Notify method. + NotifyFunc func() + + // calls tracks calls to the methods. + calls struct { + // Notify holds details about calls to the Notify method. + Notify []struct { + } + } + lockNotify sync.RWMutex +} + +// Notify calls NotifyFunc. +func (mock *WriteNotifierMock) Notify() { + if mock.NotifyFunc == nil { + panic("WriteNotifierMock.NotifyFunc: method is nil but WriteNotifier.Notify was just called") + } + callInfo := struct { + }{} + mock.lockNotify.Lock() + mock.calls.Notify = append(mock.calls.Notify, callInfo) + mock.lockNotify.Unlock() + mock.NotifyFunc() +} + +// NotifyCalls gets all the calls that were made to Notify. +// Check the length with: +// +// len(mockedWriteNotifier.NotifyCalls()) +func (mock *WriteNotifierMock) NotifyCalls() []struct { +} { + var calls []struct { + } + mock.lockNotify.RLock() + calls = mock.calls.Notify + mock.lockNotify.RUnlock() + return calls +} diff --git a/pp/go/storage/head/services/mock/removed_head_notifier.go b/pp/go/storage/head/services/mock/removed_head_notifier.go new file mode 100644 index 0000000000..b503d98f86 --- /dev/null +++ b/pp/go/storage/head/services/mock/removed_head_notifier.go @@ -0,0 +1,63 @@ +// Code generated by moq; DO NOT EDIT. +// github.com/matryer/moq + +package mock + +import ( + "sync" +) + +// RemovedHeadNotifierMock is a mock implementation of RemovedHeadNotifier. +// +// func TestSomethingThatUsesRemovedHeadNotifier(t *testing.T) { +// +// // make and configure a mocked RemovedHeadNotifier +// mockedRemovedHeadNotifier := &RemovedHeadNotifierMock{ +// NotifyFunc: func() { +// panic("mock out the Notify method") +// }, +// } +// +// // use mockedRemovedHeadNotifier in code that requires RemovedHeadNotifier +// // and then make assertions. +// +// } +type RemovedHeadNotifierMock struct { + // NotifyFunc mocks the Notify method. + NotifyFunc func() + + // calls tracks calls to the methods. + calls struct { + // Notify holds details about calls to the Notify method. + Notify []struct { + } + } + lockNotify sync.RWMutex +} + +// Notify calls NotifyFunc. +func (mock *RemovedHeadNotifierMock) Notify() { + if mock.NotifyFunc == nil { + panic("RemovedHeadNotifierMock.NotifyFunc: method is nil but RemovedHeadNotifier.Notify was just called") + } + callInfo := struct { + }{} + mock.lockNotify.Lock() + mock.calls.Notify = append(mock.calls.Notify, callInfo) + mock.lockNotify.Unlock() + mock.NotifyFunc() +} + +// NotifyCalls gets all the calls that were made to Notify. +// Check the length with: +// +// len(mockedRemovedHeadNotifier.NotifyCalls()) +func (mock *RemovedHeadNotifierMock) NotifyCalls() []struct { +} { + var calls []struct { + } + mock.lockNotify.RLock() + calls = mock.calls.Notify + mock.lockNotify.RUnlock() + return calls +} diff --git a/pp/go/storage/head/services/mock/rotator_config.go b/pp/go/storage/head/services/mock/rotator_config.go new file mode 100644 index 0000000000..cb8b81c26d --- /dev/null +++ b/pp/go/storage/head/services/mock/rotator_config.go @@ -0,0 +1,63 @@ +// Code generated by moq; DO NOT EDIT. +// github.com/matryer/moq + +package mock + +import ( + "sync" +) + +// RotatorConfigMock is a mock implementation of services.RotatorConfig. +// +// func TestSomethingThatUsesRotatorConfig(t *testing.T) { +// +// // make and configure a mocked services.RotatorConfig +// mockedRotatorConfig := &RotatorConfigMock{ +// NumberOfShardsFunc: func() uint16 { +// panic("mock out the NumberOfShards method") +// }, +// } +// +// // use mockedRotatorConfig in code that requires services.RotatorConfig +// // and then make assertions. +// +// } +type RotatorConfigMock struct { + // NumberOfShardsFunc mocks the NumberOfShards method. + NumberOfShardsFunc func() uint16 + + // calls tracks calls to the methods. + calls struct { + // NumberOfShards holds details about calls to the NumberOfShards method. + NumberOfShards []struct { + } + } + lockNumberOfShards sync.RWMutex +} + +// NumberOfShards calls NumberOfShardsFunc. +func (mock *RotatorConfigMock) NumberOfShards() uint16 { + if mock.NumberOfShardsFunc == nil { + panic("RotatorConfigMock.NumberOfShardsFunc: method is nil but RotatorConfig.NumberOfShards was just called") + } + callInfo := struct { + }{} + mock.lockNumberOfShards.Lock() + mock.calls.NumberOfShards = append(mock.calls.NumberOfShards, callInfo) + mock.lockNumberOfShards.Unlock() + return mock.NumberOfShardsFunc() +} + +// NumberOfShardsCalls gets all the calls that were made to NumberOfShards. +// Check the length with: +// +// len(mockedRotatorConfig.NumberOfShardsCalls()) +func (mock *RotatorConfigMock) NumberOfShardsCalls() []struct { +} { + var calls []struct { + } + mock.lockNumberOfShards.RLock() + calls = mock.calls.NumberOfShards + mock.lockNumberOfShards.RUnlock() + return calls +} diff --git a/pp/go/storage/head/services/mock/segment_writer.go b/pp/go/storage/head/services/mock/segment_writer.go new file mode 100644 index 0000000000..7fa95a1871 --- /dev/null +++ b/pp/go/storage/head/services/mock/segment_writer.go @@ -0,0 +1,219 @@ +// Code generated by moq; DO NOT EDIT. +// github.com/matryer/moq + +package mock + +import ( + "github.com/prometheus/prometheus/pp/go/cppbridge" + "sync" +) + +// SegmentWriterMock is a mock implementation of SegmentWriter. +// +// func TestSomethingThatUsesSegmentWriter(t *testing.T) { +// +// // make and configure a mocked SegmentWriter +// mockedSegmentWriter := &SegmentWriterMock{ +// CloseFunc: func() error { +// panic("mock out the Close method") +// }, +// CurrentSizeFunc: func() int64 { +// panic("mock out the CurrentSize method") +// }, +// FlushFunc: func() error { +// panic("mock out the Flush method") +// }, +// SyncFunc: func() error { +// panic("mock out the Sync method") +// }, +// WriteFunc: func(segment *cppbridge.HeadEncodedSegment) error { +// panic("mock out the Write method") +// }, +// } +// +// // use mockedSegmentWriter in code that requires SegmentWriter +// // and then make assertions. +// +// } +type SegmentWriterMock struct { + // CloseFunc mocks the Close method. + CloseFunc func() error + + // CurrentSizeFunc mocks the CurrentSize method. + CurrentSizeFunc func() int64 + + // FlushFunc mocks the Flush method. + FlushFunc func() error + + // SyncFunc mocks the Sync method. + SyncFunc func() error + + // WriteFunc mocks the Write method. + WriteFunc func(segment *cppbridge.HeadEncodedSegment) error + + // calls tracks calls to the methods. + calls struct { + // Close holds details about calls to the Close method. + Close []struct { + } + // CurrentSize holds details about calls to the CurrentSize method. + CurrentSize []struct { + } + // Flush holds details about calls to the Flush method. + Flush []struct { + } + // Sync holds details about calls to the Sync method. + Sync []struct { + } + // Write holds details about calls to the Write method. + Write []struct { + // Segment is the segment argument value. + Segment *cppbridge.HeadEncodedSegment + } + } + lockClose sync.RWMutex + lockCurrentSize sync.RWMutex + lockFlush sync.RWMutex + lockSync sync.RWMutex + lockWrite sync.RWMutex +} + +// Close calls CloseFunc. +func (mock *SegmentWriterMock) Close() error { + if mock.CloseFunc == nil { + panic("SegmentWriterMock.CloseFunc: method is nil but SegmentWriter.Close was just called") + } + callInfo := struct { + }{} + mock.lockClose.Lock() + mock.calls.Close = append(mock.calls.Close, callInfo) + mock.lockClose.Unlock() + return mock.CloseFunc() +} + +// CloseCalls gets all the calls that were made to Close. +// Check the length with: +// +// len(mockedSegmentWriter.CloseCalls()) +func (mock *SegmentWriterMock) CloseCalls() []struct { +} { + var calls []struct { + } + mock.lockClose.RLock() + calls = mock.calls.Close + mock.lockClose.RUnlock() + return calls +} + +// CurrentSize calls CurrentSizeFunc. +func (mock *SegmentWriterMock) CurrentSize() int64 { + if mock.CurrentSizeFunc == nil { + panic("SegmentWriterMock.CurrentSizeFunc: method is nil but SegmentWriter.CurrentSize was just called") + } + callInfo := struct { + }{} + mock.lockCurrentSize.Lock() + mock.calls.CurrentSize = append(mock.calls.CurrentSize, callInfo) + mock.lockCurrentSize.Unlock() + return mock.CurrentSizeFunc() +} + +// CurrentSizeCalls gets all the calls that were made to CurrentSize. +// Check the length with: +// +// len(mockedSegmentWriter.CurrentSizeCalls()) +func (mock *SegmentWriterMock) CurrentSizeCalls() []struct { +} { + var calls []struct { + } + mock.lockCurrentSize.RLock() + calls = mock.calls.CurrentSize + mock.lockCurrentSize.RUnlock() + return calls +} + +// Flush calls FlushFunc. +func (mock *SegmentWriterMock) Flush() error { + if mock.FlushFunc == nil { + panic("SegmentWriterMock.FlushFunc: method is nil but SegmentWriter.Flush was just called") + } + callInfo := struct { + }{} + mock.lockFlush.Lock() + mock.calls.Flush = append(mock.calls.Flush, callInfo) + mock.lockFlush.Unlock() + return mock.FlushFunc() +} + +// FlushCalls gets all the calls that were made to Flush. +// Check the length with: +// +// len(mockedSegmentWriter.FlushCalls()) +func (mock *SegmentWriterMock) FlushCalls() []struct { +} { + var calls []struct { + } + mock.lockFlush.RLock() + calls = mock.calls.Flush + mock.lockFlush.RUnlock() + return calls +} + +// Sync calls SyncFunc. +func (mock *SegmentWriterMock) Sync() error { + if mock.SyncFunc == nil { + panic("SegmentWriterMock.SyncFunc: method is nil but SegmentWriter.Sync was just called") + } + callInfo := struct { + }{} + mock.lockSync.Lock() + mock.calls.Sync = append(mock.calls.Sync, callInfo) + mock.lockSync.Unlock() + return mock.SyncFunc() +} + +// SyncCalls gets all the calls that were made to Sync. +// Check the length with: +// +// len(mockedSegmentWriter.SyncCalls()) +func (mock *SegmentWriterMock) SyncCalls() []struct { +} { + var calls []struct { + } + mock.lockSync.RLock() + calls = mock.calls.Sync + mock.lockSync.RUnlock() + return calls +} + +// Write calls WriteFunc. +func (mock *SegmentWriterMock) Write(segment *cppbridge.HeadEncodedSegment) error { + if mock.WriteFunc == nil { + panic("SegmentWriterMock.WriteFunc: method is nil but SegmentWriter.Write was just called") + } + callInfo := struct { + Segment *cppbridge.HeadEncodedSegment + }{ + Segment: segment, + } + mock.lockWrite.Lock() + mock.calls.Write = append(mock.calls.Write, callInfo) + mock.lockWrite.Unlock() + return mock.WriteFunc(segment) +} + +// WriteCalls gets all the calls that were made to Write. +// Check the length with: +// +// len(mockedSegmentWriter.WriteCalls()) +func (mock *SegmentWriterMock) WriteCalls() []struct { + Segment *cppbridge.HeadEncodedSegment +} { + var calls []struct { + Segment *cppbridge.HeadEncodedSegment + } + mock.lockWrite.RLock() + calls = mock.calls.Write + mock.lockWrite.RUnlock() + return calls +} diff --git a/pp/go/storage/head/services/persistener.go b/pp/go/storage/head/services/persistener.go new file mode 100644 index 0000000000..e7e13ccc87 --- /dev/null +++ b/pp/go/storage/head/services/persistener.go @@ -0,0 +1,352 @@ +package services + +import ( + "slices" + "time" + + "github.com/jonboulle/clockwork" + "github.com/prometheus/client_golang/prometheus" + + "github.com/prometheus/prometheus/pp/go/cppbridge" + "github.com/prometheus/prometheus/pp/go/logger" + "github.com/prometheus/prometheus/pp/go/storage/catalog" + "github.com/prometheus/prometheus/pp/go/util" +) + +// defaultCoolingInterval the interval after which the rotation should have +// taken place to eliminate errors in the selection from the catalog. +const defaultCoolingInterval = 60 * time.Second + +// +// Persistener +// + +// Persistener converts and saves spent [Head]s. +type Persistener[ + TTask Task, + TShard, TGoShard Shard, + THeadBlockWriter HeadBlockWriter[TShard], + THead Head[TTask, TShard, TGoShard], +] struct { + catalog *catalog.Catalog + blockWriter THeadBlockWriter + writeNotifier WriteNotifier + + clock clockwork.Clock + tsdbRetentionPeriod time.Duration + retentionPeriod time.Duration + // stat + events prometheus.Counter + headPersistenceDuration prometheus.Histogram +} + +// NewPersistener init new [Persistener]. +func NewPersistener[ + TTask Task, + TShard, TGoShard Shard, + THeadBlockWriter HeadBlockWriter[TShard], + THead Head[TTask, TShard, TGoShard], +]( + hcatalog *catalog.Catalog, + blockWriter THeadBlockWriter, + writeNotifier WriteNotifier, + clock clockwork.Clock, + tsdbRetentionPeriod time.Duration, + retentionPeriod time.Duration, + registerer prometheus.Registerer, +) *Persistener[TTask, TShard, TGoShard, THeadBlockWriter, THead] { + factory := util.NewUnconflictRegisterer(registerer) + return &Persistener[TTask, TShard, TGoShard, THeadBlockWriter, THead]{ + catalog: hcatalog, + blockWriter: blockWriter, + writeNotifier: writeNotifier, + clock: clock, + tsdbRetentionPeriod: tsdbRetentionPeriod, + retentionPeriod: retentionPeriod, + events: factory.NewCounter( + prometheus.CounterOpts{ + Name: "prompp_head_event_count", + Help: "Number of head events", + ConstLabels: prometheus.Labels{"type": "persisted"}, + }, + ), + headPersistenceDuration: factory.NewHistogram( + prometheus.HistogramOpts{ + Name: "prompp_head_persistence_duration", + Help: "Block write duration in milliseconds.", + Buckets: []float64{ + 500, 1000, 2500, 5000, 7500, + 10000, 25000, 50000, 75000, 100000, + }, + }, + ), + } +} + +// Persist spent [Head]s. +// +//revive:disable-next-line:function-length // long but readable. +//revive:disable-next-line:cognitive-complexity // long but readable. +//revive:disable-next-line:cyclomatic // long but readable. +func (p *Persistener[TTask, TShard, TGoShard, THeadBlockWriter, THead]) Persist(heads []THead) (outdatedHeads []THead) { + shouldNotify := false + for _, head := range heads { + start := p.clock.Now() + if !head.IsReadOnly() { + continue + } + + logger.Debugf("[Persistener]: head %s start persist", head.ID()) + if record, err := p.catalog.Get(head.ID()); err != nil { + logger.Errorf("[Persistener]: failed get head %s from catalog: %v", head.ID(), err) + } else if record.Status() == catalog.StatusPersisted { + if p.persistedHeadIsOutdated(record.UpdatedAt()) { + logger.Debugf("[Persistener]: persisted head %s is outdated", head.ID()) + outdatedHeads = append(outdatedHeads, head) + } + + continue + } + + if p.HeadIsOutdated(head) { + logger.Debugf("[Persistener]: head %s is outdated", head.ID()) + if _, err := p.catalog.SetStatus(head.ID(), catalog.StatusPersisted); err != nil { + logger.Errorf("[Persistener]: set head status in catalog %s: %v", head.ID(), err) + continue + } + + outdatedHeads = append(outdatedHeads, head) + continue + } + + if err := p.flushHead(head); err != nil { + logger.Errorf("[Persistener]: failed flush head %s: %v", head.ID(), err) + continue + } + + if err := p.persistHead(head); err != nil { + logger.Errorf("[Persistener]: failed persist head %s: %v", head.ID(), err) + continue + } + + if _, err := p.catalog.SetStatus(head.ID(), catalog.StatusPersisted); err != nil { + logger.Errorf("[Persistener]: set head status in catalog %s: %v", head.ID(), err) + continue + } + + logger.Infof("[Persistener]: head %s persisted, duration: %v", head.ID(), p.clock.Since(start)) + p.events.Inc() + p.headPersistenceDuration.Observe(float64(p.clock.Since(start).Milliseconds())) + shouldNotify = true + } + + if shouldNotify { + p.writeNotifier.Notify() + } + + return outdatedHeads +} + +func (*Persistener[TTask, TShard, TGoShard, THeadBlockWriter, THead]) headTimeInterval( + head THead, +) cppbridge.TimeInterval { + timeInterval := cppbridge.NewInvalidTimeInterval() + for shard := range head.RangeShards() { + interval := shard.TimeInterval(false) + timeInterval.MinT = min(interval.MinT, timeInterval.MinT) + timeInterval.MaxT = max(interval.MaxT, timeInterval.MaxT) + } + return timeInterval +} + +// HeadIsOutdated check [Head] is outdated. +func (p *Persistener[TTask, TShard, TGoShard, THeadBlockWriter, THead]) HeadIsOutdated(head THead) bool { + return p.clock.Since(time.UnixMilli(p.headTimeInterval(head).MaxT)) >= p.tsdbRetentionPeriod +} + +func (p *Persistener[TTask, TShard, TGoShard, THeadBlockWriter, THead]) persistedHeadIsOutdated( + persistTimeMs int64, +) bool { + return p.clock.Since(time.UnixMilli(persistTimeMs)) >= p.retentionPeriod +} + +func (*Persistener[TTask, TShard, TGoShard, THeadBlockWriter, THead]) flushHead(head THead) error { + for shard := range head.RangeShards() { + if err := shard.WalFlush(); err != nil { + return err + } + } + + return nil +} + +func (p *Persistener[TTask, TShard, TGoShard, THeadBlockWriter, THead]) persistHead(head THead) error { + for shard := range head.RangeShards() { + if _, err := p.blockWriter.Write(shard); err != nil { + return err + } + } + + return nil +} + +// +// PersistenerService +// + +// PersistenerService service for persist spent [Head]s. +type PersistenerService[ + TTask Task, + TShard, TGoShard Shard, + THeadBlockWriter HeadBlockWriter[TShard], + THead Head[TTask, TShard, TGoShard], + TProxyHead ProxyHead[TTask, TShard, TGoShard, THead], + TLoader Loader[TTask, TShard, TGoShard, THead], +] struct { + persistener *Persistener[TTask, TShard, TGoShard, THeadBlockWriter, THead] + proxy TProxyHead + loader TLoader + catalog *catalog.Catalog + mediator Mediator + clock clockwork.Clock + tsdbRetentionPeriod time.Duration +} + +// NewPersistenerService init new [PersistenerService]. +func NewPersistenerService[ + TTask Task, + TShard, TGoShard Shard, + THeadBlockWriter HeadBlockWriter[TShard], + THead Head[TTask, TShard, TGoShard], + TProxyHead ProxyHead[TTask, TShard, TGoShard, THead], + TLoader Loader[TTask, TShard, TGoShard, THead], +]( + proxy TProxyHead, + loader TLoader, + hcatalog *catalog.Catalog, + blockWriter THeadBlockWriter, + writeNotifier WriteNotifier, + clock clockwork.Clock, + mediator Mediator, + tsdbRetentionPeriod time.Duration, + retentionPeriod time.Duration, + registerer prometheus.Registerer, +) *PersistenerService[TTask, TShard, TGoShard, THeadBlockWriter, THead, TProxyHead, TLoader] { + return &PersistenerService[TTask, TShard, TGoShard, THeadBlockWriter, THead, TProxyHead, TLoader]{ + persistener: NewPersistener[TTask, TShard, TGoShard, THeadBlockWriter, THead]( + hcatalog, + blockWriter, + writeNotifier, + clock, + tsdbRetentionPeriod, + retentionPeriod, + registerer, + ), + proxy: proxy, + loader: loader, + catalog: hcatalog, + mediator: mediator, + clock: clock, + tsdbRetentionPeriod: tsdbRetentionPeriod, + } +} + +// Execute starts the [PersistenerService]. +func (pg *PersistenerService[TTask, TShard, TGoShard, THeadBlockWriter, THead, TKeeper, TLoader]) Execute() { + logger.Infof("The PersistenerService is running.") + + for range pg.mediator.C() { + pg.ProcessHeads() + } + + logger.Infof("The PersistenerService stopped.") +} + +// ProcessHeads process persist [Head]s. +func (pg *PersistenerService[TTask, TShard, TGoShard, THeadBlockWriter, THead, TKeeper, TLoader]) ProcessHeads() { + heads := pg.proxy.Heads() + pg.persistHeads(heads) + pg.loadRotatedHeadsInKeeper(heads) +} + +func (pg *PersistenerService[TTask, TShard, TGoShard, THeadBlockWriter, THead, TKeeper, TLoader]) persistHeads( + heads []THead, +) { + pg.proxy.Remove(pg.persistener.Persist(heads)) +} + +// loadRotatedHeadsInKeeper loads rotated or unused [Head]s and adds them to the [Keeper]. +// +//revive:disable-next-line:cyclomatic // but readable +func (pg *PersistenerService[ + TTask, + TShard, + TGoShard, + THeadBlockWriter, + THead, + TKeeper, + TLoader, +]) loadRotatedHeadsInKeeper(keeperHeads []THead) { + if !pg.proxy.HasSlot() { + return + } + + headExists := func(id string) bool { + return slices.ContainsFunc(keeperHeads, func(head THead) bool { + return head.ID() == id + }) + } + + records := pg.catalog.List(func(record *catalog.Record) bool { + // in case the rotated status was not set due to an error + statusIsAppropriate := record.Status() == catalog.StatusNew || + record.Status() == catalog.StatusRotated || + record.Status() == catalog.StatusActive + + isOutdated := pg.clock.Since(time.UnixMilli(record.CreatedAt())) >= pg.tsdbRetentionPeriod + + return statusIsAppropriate && !headExists(record.ID()) && record.DeletedAt() == 0 && !isOutdated + }, catalog.LessByUpdateAt) + + aheadID := pg.proxy.Get().ID() + for _, record := range records { + // skip active head + if aheadID == record.ID() { + continue + } + + // skip the newly created head + if (record.Status() == catalog.StatusNew || record.Status() == catalog.StatusActive) && + pg.clock.Since(time.UnixMilli(record.CreatedAt())) < defaultCoolingInterval { + continue + } + + if !pg.proxy.HasSlot() { + break + } + + if !pg.loadAndAddHeadToKeeper(record) { + break + } + } +} + +// loadAndAddHeadToKeeper loads [Head] and adds them to the [Keeper]. +func (pg *PersistenerService[ + TTask, + TShard, + TGoShard, + THeadBlockWriter, + THead, + TKeeper, + TLoader, +]) loadAndAddHeadToKeeper(record *catalog.Record) bool { + head, _ := pg.loader.Load(record, 0) + head.SetReadOnly() + if err := pg.proxy.Add(head, time.Duration(record.CreatedAt())*time.Millisecond); err != nil { + _ = head.Close() + return false + } + + return true +} diff --git a/pp/go/storage/head/services/persistener_test.go b/pp/go/storage/head/services/persistener_test.go new file mode 100644 index 0000000000..f988ecf441 --- /dev/null +++ b/pp/go/storage/head/services/persistener_test.go @@ -0,0 +1,407 @@ +package services_test + +import ( + "errors" + "os" + "path/filepath" + "testing" + "time" + + "github.com/jonboulle/clockwork" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/prometheus/model/labels" + "github.com/prometheus/prometheus/pp/go/cppbridge" + "github.com/prometheus/prometheus/pp/go/storage" + "github.com/prometheus/prometheus/pp/go/storage/block" + "github.com/prometheus/prometheus/pp/go/storage/catalog" + "github.com/prometheus/prometheus/pp/go/storage/head/container" + "github.com/prometheus/prometheus/pp/go/storage/head/keeper" + "github.com/prometheus/prometheus/pp/go/storage/head/services" + "github.com/prometheus/prometheus/pp/go/storage/head/services/mock" + "github.com/prometheus/prometheus/pp/go/storage/head/shard" + "github.com/prometheus/prometheus/pp/go/storage/head/task" + "github.com/prometheus/prometheus/pp/go/storage/storagetest" + "github.com/stretchr/testify/suite" +) + +const ( + shardsCount = 2 + maxSegmentSize = 1024 + unloadDataStorageInterval = time.Duration(1) + + tsdbRetentionPeriod = time.Millisecond * 100 + + retentionPeriod = time.Minute * 5 +) + +type GenericPersistenceSuite struct { + suite.Suite + dataDir string + clock *clockwork.FakeClock + catalog *catalog.Catalog + proxy *storage.Proxy + blockWriter *mock.HeadBlockWriterMock[*shard.Shard] + writeNotifier *mock.WriteNotifierMock +} + +func (s *GenericPersistenceSuite) SetupTest() { + s.dataDir = s.createDataDirectory() + s.clock = clockwork.NewFakeClockAt(time.UnixMilli(0)) + s.createCatalog() + + h := s.mustCreateHead() + activeHeadContainer := container.NewWeighted(h) + removedHeadNotifier := &mock.WriteNotifierMock{NotifyFunc: func() {}} + hKeeper := keeper.NewKeeper[storage.Head](1, removedHeadNotifier) + s.proxy = storage.NewProxy(activeHeadContainer, hKeeper, func(*storage.Head) error { return nil }) + s.blockWriter = &mock.HeadBlockWriterMock[*shard.Shard]{} + s.writeNotifier = &mock.WriteNotifierMock{NotifyFunc: func() {}} +} + +func (s *GenericPersistenceSuite) createDataDirectory() string { + dataDir := filepath.Join(s.T().TempDir(), "data") + s.Require().NoError(os.MkdirAll(dataDir, os.ModeDir)) + return dataDir +} + +func (s *GenericPersistenceSuite) createHead() (*storage.Head, error) { + return storage.NewBuilder( + s.catalog, + s.dataDir, + maxSegmentSize, + prometheus.DefaultRegisterer, + unloadDataStorageInterval, + ).Build(0, shardsCount) +} + +func (s *GenericPersistenceSuite) mustCreateHead() *storage.Head { + h, err := s.createHead() + s.Require().NoError(err) + return h +} + +func (s *GenericPersistenceSuite) createCatalog() { + l, err := catalog.NewFileLogV2(filepath.Join(s.dataDir, "catalog.log")) + s.Require().NoError(err) + + s.catalog, err = catalog.New( + s.clock, + l, + &catalog.DefaultIDGenerator{}, + catalog.DefaultMaxLogFileSize, + nil, + ) + s.Require().NoError(err) +} + +type PersistenerSuite struct { + GenericPersistenceSuite + persistener *services.Persistener[ + *task.Generic[*shard.PerGoroutineShard], + *shard.Shard, + *shard.PerGoroutineShard, + *mock.HeadBlockWriterMock[*shard.Shard], + *storage.Head, + ] +} + +func (s *PersistenerSuite) SetupTest() { + s.GenericPersistenceSuite.SetupTest() + + s.persistener = services.NewPersistener[ + *task.Generic[*shard.PerGoroutineShard], + *shard.Shard, + *shard.PerGoroutineShard, + *mock.HeadBlockWriterMock[*shard.Shard], + *storage.Head, + ](s.catalog, s.blockWriter, s.writeNotifier, s.clock, tsdbRetentionPeriod, retentionPeriod, nil) +} + +func TestPersistenerSuite(t *testing.T) { + suite.Run(t, new(PersistenerSuite)) +} + +func (s *PersistenerSuite) TestNoHeads() { + // Arrange + + // Act + outdated := s.persistener.Persist(nil) + + // Assert + s.Equal([]*storage.Head(nil), outdated) + s.Empty(s.blockWriter.WriteCalls()) +} + +func (s *PersistenerSuite) TestNoPersistWritableHead() { + // Arrange + heads := []*storage.Head{s.mustCreateHead()} + + // Act + outdated := s.persistener.Persist(heads) + + // Assert + s.Equal([]*storage.Head(nil), outdated) + s.Empty(s.blockWriter.WriteCalls()) +} + +func (s *PersistenerSuite) TestNoPersistPersistedHead() { + // Arrange + head := s.mustCreateHead() + storagetest.MustAppendTimeSeries(&s.Suite, head, []storagetest.TimeSeries{ + { + Labels: labels.FromStrings("__name__", "value1"), + Samples: []cppbridge.Sample{ + {Timestamp: 0, Value: 0}, + }, + }, + }) + head.SetReadOnly() + _, err := s.catalog.SetStatus(head.ID(), catalog.StatusPersisted) + s.Require().NoError(err) + + s.clock.Advance(retentionPeriod - 1) + + // Act + outdated := s.persistener.Persist([]*storage.Head{head}) + + // Assert + s.Equal([]*storage.Head(nil), outdated) + s.Empty(s.blockWriter.WriteCalls()) +} + +func (s *PersistenerSuite) TestOutdatedPersistedHead() { + // Arrange + head := s.mustCreateHead() + storagetest.MustAppendTimeSeries(&s.Suite, head, []storagetest.TimeSeries{ + { + Labels: labels.FromStrings("__name__", "value1"), + Samples: []cppbridge.Sample{ + {Timestamp: 0, Value: 0}, + }, + }, + }) + head.SetReadOnly() + _, err := s.catalog.SetStatus(head.ID(), catalog.StatusPersisted) + s.Require().NoError(err) + + s.clock.Advance(retentionPeriod) + + // Act + outdated := s.persistener.Persist([]*storage.Head{head}) + + // Assert + s.Equal([]*storage.Head{head}, outdated) + s.Empty(s.blockWriter.WriteCalls()) +} + +func (s *PersistenerSuite) TestOutdatedHead() { + // Arrange + s.clock.Advance(tsdbRetentionPeriod) + + head := s.mustCreateHead() + storagetest.MustAppendTimeSeries(&s.Suite, head, []storagetest.TimeSeries{ + { + Labels: labels.FromStrings("__name__", "value1"), + Samples: []cppbridge.Sample{ + {Timestamp: 0, Value: 0}, + }, + }, + }) + head.SetReadOnly() + + // Act + outdated := s.persistener.Persist([]*storage.Head{head}) + + // Assert + s.Equal([]*storage.Head{head}, outdated) + s.Empty(s.blockWriter.WriteCalls()) +} + +func (s *PersistenerSuite) TestPersistHeadSuccess() { + // Arrange + s.clock.Advance(tsdbRetentionPeriod) + blockWriter := block.NewWriter[*shard.Shard]( + s.dataDir, + block.DefaultChunkSegmentSize, + 2*time.Hour, + prometheus.DefaultRegisterer, + ) + s.blockWriter.WriteFunc = func(shard *shard.Shard) ([]block.WrittenBlock, error) { + return blockWriter.Write(shard) + } + + head := s.mustCreateHead() + storagetest.MustAppendTimeSeries(&s.Suite, head, []storagetest.TimeSeries{ + { + Labels: labels.FromStrings("__name__", "value1"), + Samples: []cppbridge.Sample{ + {Timestamp: 1, Value: 0}, + }, + }, + }) + head.SetReadOnly() + + // Act + outdated := s.persistener.Persist([]*storage.Head{head}) + record, err := s.catalog.Get(head.ID()) + + // Assert + s.Equal([]*storage.Head(nil), outdated) + s.Len(s.blockWriter.WriteCalls(), 2) + s.Len(s.writeNotifier.NotifyCalls(), 1) + s.Require().NoError(err) + s.Equal(catalog.StatusPersisted, record.Status()) +} + +func (s *PersistenerSuite) TestPersistHeadErrorOnBlockWriterForSecondShard() { + // Arrange + s.clock.Advance(tsdbRetentionPeriod) + blockWriter := block.NewWriter[*shard.Shard]( + s.dataDir, + block.DefaultChunkSegmentSize, + 2*time.Hour, + prometheus.DefaultRegisterer, + ) + s.blockWriter.WriteFunc = func(shard *shard.Shard) ([]block.WrittenBlock, error) { + if len(s.blockWriter.WriteCalls()) == 2 { + return nil, errors.New("some error") + } + + return blockWriter.Write(shard) + } + + head := s.mustCreateHead() + storagetest.MustAppendTimeSeries(&s.Suite, head, []storagetest.TimeSeries{ + { + Labels: labels.FromStrings("__name__", "value1"), + Samples: []cppbridge.Sample{ + {Timestamp: 1, Value: 0}, + }, + }, + }) + head.SetReadOnly() + + // Act + outdated := s.persistener.Persist([]*storage.Head{head}) + record, err := s.catalog.Get(head.ID()) + + // Assert + s.Equal([]*storage.Head(nil), outdated) + s.Len(s.blockWriter.WriteCalls(), 2) + s.Empty(s.writeNotifier.NotifyCalls()) + s.Require().NoError(err) + s.Equal(catalog.StatusNew, record.Status()) +} + +type PersistenerServiceSuite struct { + GenericPersistenceSuite + loader *storage.Loader + service *services.PersistenerService[ + *task.Generic[*shard.PerGoroutineShard], + *shard.Shard, + *shard.PerGoroutineShard, + *mock.HeadBlockWriterMock[*shard.Shard], + *storage.Head, + *storage.Proxy, + *storage.Loader, + ] +} + +func (s *PersistenerServiceSuite) SetupTest() { + s.GenericPersistenceSuite.SetupTest() + + s.loader = storage.NewLoader(s.dataDir, maxSegmentSize, prometheus.DefaultRegisterer, unloadDataStorageInterval) + s.service = services.NewPersistenerService[ + *task.Generic[*shard.PerGoroutineShard], + *shard.Shard, + *shard.PerGoroutineShard, + *mock.HeadBlockWriterMock[*shard.Shard], + *storage.Head, + *storage.Proxy, + *storage.Loader, + ]( + s.proxy, + s.loader, + s.catalog, + s.blockWriter, + s.writeNotifier, + s.clock, + nil, + tsdbRetentionPeriod, + retentionPeriod, + nil, + ) +} + +func TestPersistenerServiceSuite(t *testing.T) { + suite.Run(t, new(PersistenerServiceSuite)) +} + +func (s *PersistenerServiceSuite) TestRemoveOutdatedHeadFromKeeper() { + // Arrange + s.clock.Advance(tsdbRetentionPeriod) + head := s.mustCreateHead() + storagetest.MustAppendTimeSeries(&s.Suite, head, []storagetest.TimeSeries{ + { + Labels: labels.FromStrings("__name__", "value1"), + Samples: []cppbridge.Sample{ + {Timestamp: 0, Value: 0}, + }, + }, + }) + head.SetReadOnly() + record, _ := s.catalog.SetStatus(head.ID(), catalog.StatusRotated) + _ = s.proxy.Add(head, time.Duration(s.clock.Now().Nanosecond())) + + // Act + s.service.ProcessHeads() + + // Assert + s.Empty(s.proxy.Heads()) + s.Equal(catalog.StatusPersisted, record.Status()) +} + +func (s *PersistenerServiceSuite) TestLoadHeadsInKeeper() { + // Arrange + head := s.mustCreateHead() + storagetest.MustAppendTimeSeries(&s.Suite, head, []storagetest.TimeSeries{ + { + Labels: labels.FromStrings("__name__", "value1"), + Samples: []cppbridge.Sample{ + {Timestamp: 1, Value: 0}, + }, + }, + }) + record, _ := s.catalog.SetStatus(head.ID(), catalog.StatusRotated) + s.Require().NoError(head.Close()) + + // Act + s.service.ProcessHeads() + + // Assert + s.Require().Len(s.proxy.Heads(), 1) + s.Equal(head.ID(), s.proxy.Heads()[0].ID()) + s.Equal(int64(0), record.CreatedAt()) +} + +func (s *PersistenerServiceSuite) TestHeadAlreadyExistsInKeeper() { + // Arrange + head := s.mustCreateHead() + storagetest.MustAppendTimeSeries(&s.Suite, head, []storagetest.TimeSeries{ + { + Labels: labels.FromStrings("__name__", "value1"), + Samples: []cppbridge.Sample{ + {Timestamp: 1, Value: 0}, + }, + }, + }) + _, _ = s.catalog.SetStatus(head.ID(), catalog.StatusRotated) + _ = s.proxy.Add(head, 0) + + // Act + s.service.ProcessHeads() + + // Assert + s.Require().Len(s.proxy.Heads(), 1) + s.Equal(head.ID(), s.proxy.Heads()[0].ID()) +} diff --git a/pp/go/storage/head/services/rotator.go b/pp/go/storage/head/services/rotator.go new file mode 100644 index 0000000000..c9e85003fc --- /dev/null +++ b/pp/go/storage/head/services/rotator.go @@ -0,0 +1,177 @@ +package services + +import ( + "context" + "fmt" + "time" + + "github.com/prometheus/client_golang/prometheus" + + "github.com/prometheus/prometheus/pp/go/logger" + "github.com/prometheus/prometheus/pp/go/util" +) + +// CopySeriesOnRotate copy active series from the current head to the new head during rotation. +var CopySeriesOnRotate = false + +// +// RotatorConfig +// + +// RotatorConfig config for [Rotator]. +type RotatorConfig interface { + // NumberOfShards returns current number of shards. + NumberOfShards() uint16 +} + +// +// Rotator +// + +// Rotator at the end of the specified interval, it creates a new [Head] and makes it active, +// and sends the old [Head] to the [Keeper]. +type Rotator[ + TTask Task, + TShard, TGoShard Shard, + THead Head[TTask, TShard, TGoShard], +] struct { + proxyHead ProxyHead[TTask, TShard, TGoShard, THead] + headBuilder HeadBuilder[TTask, TShard, TGoShard, THead] + m Mediator + cfg RotatorConfig + headInformer HeadInformer + headAddedSeriesCopier func(source, destination THead) + rotatedTrigger func() + + // stat + rotateCounter prometheus.Counter + events *prometheus.CounterVec + waitLockRotateDuration prometheus.Gauge + rotationDuration prometheus.Gauge +} + +// NewRotator init new [Rotator]. +func NewRotator[ + TTask Task, + TShard, TGoShard Shard, + THead Head[TTask, TShard, TGoShard], +]( + proxyHead ProxyHead[TTask, TShard, TGoShard, THead], + headBuilder HeadBuilder[TTask, TShard, TGoShard, THead], + m Mediator, + cfg RotatorConfig, + headInformer HeadInformer, + headAddedSeriesCopier func(source, destination THead), + rotatedTrigger func(), + registerer prometheus.Registerer, +) *Rotator[TTask, TShard, TGoShard, THead] { + factory := util.NewUnconflictRegisterer(registerer) + return &Rotator[TTask, TShard, TGoShard, THead]{ + proxyHead: proxyHead, + headBuilder: headBuilder, + m: m, + cfg: cfg, + headInformer: headInformer, + headAddedSeriesCopier: headAddedSeriesCopier, + rotatedTrigger: rotatedTrigger, + rotateCounter: factory.NewCounter( + prometheus.CounterOpts{ + Name: "prompp_rotator_rotate_count", + Help: "Total counter of rotate rotatable object.", + }, + ), + events: factory.NewCounterVec( + prometheus.CounterOpts{ + Name: "prompp_head_event_count", + Help: "Number of head events", + }, + []string{"type"}, + ), + waitLockRotateDuration: factory.NewGauge( + prometheus.GaugeOpts{ + Name: "prompp_rotator_wait_lock_rotate_duration", + Help: "The duration of the lock wait for rotation in nanoseconds", + }, + ), + rotationDuration: factory.NewGauge( + prometheus.GaugeOpts{ + Name: "prompp_rotator_rotate_duration", + Help: "The duration of the rotate in nanoseconds", + }, + ), + } +} + +// Execute starts the [Rotator]. +// +//revive:disable-next-line:confusing-naming // other type of Service. +func (s *Rotator[TTask, TShard, TGoShard, THead]) Execute(ctx context.Context) error { + logger.Infof("The Rotator is running.") + + for range s.m.C() { + if err := s.rotate(ctx, s.cfg.NumberOfShards()); err != nil { + logger.Errorf("rotation failed: %v", err) + } + + s.rotateCounter.Inc() + } + + logger.Infof("The Rotator stopped.") + + return nil +} + +// rotate it creates a new [Head] and makes it active, and sends the old [Head] to the [Keeper]. +// +//revive:disable-next-line:cyclomatic // long but readable. +func (s *Rotator[TTask, TShard, TGoShard, THead]) rotate( + ctx context.Context, + numberOfShards uint16, +) error { + start := time.Now() + oldHead := s.proxyHead.Get() + newHead, err := s.headBuilder.Build(oldHead.Generation()+1, numberOfShards) + if err != nil { + return fmt.Errorf("failed to build a new head: %w", err) + } + + if CopySeriesOnRotate && oldHead.NumberOfShards() == newHead.NumberOfShards() { + s.headAddedSeriesCopier(oldHead, newHead) + } + + if err = s.proxyHead.AddWithReplace(oldHead, s.headInformer.CreatedAt(oldHead.ID())); err != nil { + return fmt.Errorf("failed add to keeper old head: %w", err) + } + + startWait := time.Now() + if err = s.proxyHead.Replace(ctx, newHead); err != nil { + if errClose := newHead.Close(); errClose != nil { + logger.Errorf("failed close new head: %s : %v", newHead.ID(), errClose) + } + + return fmt.Errorf("failed to replace old to new head: %w", err) + } + s.waitLockRotateDuration.Set(float64(time.Since(startWait).Nanoseconds())) + + if err = s.headInformer.SetActiveStatus(newHead.ID()); err != nil { + logger.Warnf("failed set status active for head{%s}: %s", newHead.ID(), err) + } + + if err = MergeOutOfOrderChunksWithHead(oldHead); err != nil { + logger.Warnf("failed merge out of order chunks in data storage: %s", err) + } + + if err = CFSViaRange(oldHead); err != nil { + logger.Warnf("failed commit and flush to wal: %s", err) + } + + if err = s.headInformer.SetRotatedStatus(oldHead.ID()); err != nil { + logger.Warnf("failed set status rotated for head{%s}: %s", oldHead.ID(), err) + } + oldHead.SetReadOnly() + s.events.With(prometheus.Labels{"type": "rotated"}).Inc() + s.rotationDuration.Set(float64(time.Since(start).Nanoseconds())) + s.rotatedTrigger() + + return nil +} diff --git a/pp/go/storage/head/services/rotator_test.go b/pp/go/storage/head/services/rotator_test.go new file mode 100644 index 0000000000..093781eca8 --- /dev/null +++ b/pp/go/storage/head/services/rotator_test.go @@ -0,0 +1,332 @@ +package services_test + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/stretchr/testify/suite" + + "github.com/prometheus/prometheus/pp/go/cppbridge" + "github.com/prometheus/prometheus/pp/go/model" + "github.com/prometheus/prometheus/pp/go/storage" + "github.com/prometheus/prometheus/pp/go/storage/head/container" + "github.com/prometheus/prometheus/pp/go/storage/head/head" + "github.com/prometheus/prometheus/pp/go/storage/head/keeper" + "github.com/prometheus/prometheus/pp/go/storage/head/services" + "github.com/prometheus/prometheus/pp/go/storage/head/services/mock" + "github.com/prometheus/prometheus/pp/go/storage/head/shard" + "github.com/prometheus/prometheus/pp/go/storage/head/shard/wal" + "github.com/prometheus/prometheus/pp/go/storage/head/task" +) + +type RotatorSuite struct { + suite.Suite + + baseCtx context.Context +} + +func TestRotatorSuite(t *testing.T) { + suite.Run(t, new(RotatorSuite)) +} + +func (s *RotatorSuite) SetupSuite() { + s.baseCtx = context.Background() +} + +func (s *RotatorSuite) createHead( + headID string, + segmentWriters []*mock.SegmentWriterMock, + generation uint64, + numberOfShards uint16, +) *storage.Head { + shards := make([]*shard.Shard, numberOfShards) + for shardID, segmentWriter := range segmentWriters { + shards[shardID] = s.createShardOnMemory(segmentWriter, maxSegmentSize, uint16(shardID)) + } + + return head.NewHead( + headID, + shards, + shard.NewPerGoroutineShard[*storage.Wal], + nil, + generation, + nil, + ) +} + +func (*RotatorSuite) createShardOnMemory( + segmentWriter *mock.SegmentWriterMock, + maxSegmentSize uint32, + shardID uint16, +) *shard.Shard { + lss := shard.NewLSS() + // logShards is 0 for single encoder + shardWalEncoder := cppbridge.NewHeadWalEncoder(shardID, 0, lss.Target()) + + return shard.NewShard( + lss, + shard.NewDataStorage(), + nil, + nil, + wal.NewWal(shardWalEncoder, segmentWriter, maxSegmentSize, shardID, nil), + shardID, + ) +} + +func (*RotatorSuite) nameIDGenerator(n uint64) string { + return fmt.Sprintf("test-head-id-%d", n) +} + +func (*RotatorSuite) addLabelSetToHead(h *storage.Head) { + for sd := range h.RangeShards() { + sd.LSSWithRLock(func(target, _ *cppbridge.LabelSetStorage) error { + target.FindOrEmplace(model.LabelSetFromMap(map[string]string{"_name__": fmt.Sprintf("test0%d", sd.ShardID())})) + return nil + }) + } +} + +func (*RotatorSuite) getNumSeriesFromHead(h *storage.Head) uint32 { + numSeries := uint32(0) + for sd := range h.RangeShards() { + status := cppbridge.NewHeadStatus() + sd.LSSWithRLock(func(target, _ *cppbridge.LabelSetStorage) error { + status.FromLSS(target, 0) + return nil + }) + numSeries += status.NumSeries + } + + return numSeries +} + +func (s *RotatorSuite) TestRotate() { + trigger := make(chan struct{}, 1) + start := make(chan struct{}) + mediator := &mock.MediatorMock{ + CFunc: func() <-chan struct{} { + close(start) + return trigger + }, + } + + segmentWriters := make([]*mock.SegmentWriterMock, shardsCount) + for shardID := range shardsCount { + segmentWriters[shardID] = &mock.SegmentWriterMock{ + WriteFunc: func(*cppbridge.HeadEncodedSegment) error { return nil }, + FlushFunc: func() error { return nil }, + SyncFunc: func() error { return nil }, + CloseFunc: func() error { return nil }, + CurrentSizeFunc: func() int64 { return 0 }, + } + } + removedHeadNotifier := &mock.RemovedHeadNotifierMock{NotifyFunc: func() {}} + rotatedCounter := uint64(0) + rotatedTrigger := func() { rotatedCounter++ } + + proxyHead := storage.NewProxy( + container.NewWeighted( + s.createHead(s.nameIDGenerator(rotatedCounter), segmentWriters, rotatedCounter, shardsCount), + ), + keeper.NewKeeper[storage.Head](2, removedHeadNotifier), + func(*storage.Head) error { return nil }, + ) + + headBuilder := &mock.HeadBuilderMock[ + *task.Generic[*shard.PerGoroutineShard], + *shard.Shard, + *shard.PerGoroutineShard, + *storage.Head, + ]{BuildFunc: func(generation uint64, numberOfShards uint16) (*storage.Head, error) { + return s.createHead(s.nameIDGenerator(generation), segmentWriters, generation, numberOfShards), nil + }} + cfg := &mock.RotatorConfigMock{NumberOfShardsFunc: func() uint16 { return shardsCount }} + headInformer := &mock.HeadInformerMock{ + SetActiveStatusFunc: func(string) error { return nil }, + SetRotatedStatusFunc: func(string) error { return nil }, + CreatedAtFunc: func(string) time.Duration { return time.Duration(time.Now().UnixMilli()) }, + } + + rotator := services.NewRotator( + proxyHead, + headBuilder, + mediator, + cfg, + headInformer, + head.CopyAddedSeries[*shard.Shard, *shard.PerGoroutineShard](shard.CopyAddedSeries), + rotatedTrigger, + nil, + ) + + aHead := proxyHead.Get() + s.addLabelSetToHead(aHead) + + done := make(chan struct{}) + services.CopySeriesOnRotate = false + + s.T().Run("execute", func(t *testing.T) { + t.Parallel() + + err := rotator.Execute(s.baseCtx) + close(done) + s.NoError(err) + }) + + s.T().Run("tick", func(t *testing.T) { + t.Parallel() + + <-start + trigger <- struct{}{} + close(trigger) + <-done + + s.Require().NoError(proxyHead.Close()) + + s.Equal(uint64(1), rotatedCounter) + aHead := proxyHead.Get() + s.Equal(uint64(1), aHead.Generation()) + s.Equal(s.nameIDGenerator(rotatedCounter), aHead.ID()) + s.Equal(headInformer.SetActiveStatusCalls()[0].HeadID, aHead.ID()) + actualNumSeries := s.getNumSeriesFromHead(aHead) + s.Equal(uint32(0), actualNumSeries) + + for _, segmentWriter := range segmentWriters { + if !s.Len(segmentWriter.WriteCalls(), 1) { + return + } + if !s.Len(segmentWriter.FlushCalls(), 1) { + return + } + if !s.Len(segmentWriter.SyncCalls(), 1) { + return + } + + for _, call := range segmentWriter.WriteCalls() { + s.Equal(uint32(0), call.Segment.Samples()) + } + } + + rHeads := proxyHead.Heads() + s.Len(rHeads, 1) + s.Equal(s.nameIDGenerator(rotatedCounter-1), rHeads[0].ID()) + s.Equal(headInformer.SetRotatedStatusCalls()[0].HeadID, rHeads[0].ID()) + s.True(rHeads[0].IsReadOnly()) + }) +} + +func (s *RotatorSuite) TestCopySeriesOnRotate() { + trigger := make(chan struct{}, 1) + start := make(chan struct{}) + mediator := &mock.MediatorMock{ + CFunc: func() <-chan struct{} { + close(start) + return trigger + }, + } + + segmentWriters := make([]*mock.SegmentWriterMock, shardsCount) + for shardID := range shardsCount { + segmentWriters[shardID] = &mock.SegmentWriterMock{ + WriteFunc: func(*cppbridge.HeadEncodedSegment) error { return nil }, + FlushFunc: func() error { return nil }, + SyncFunc: func() error { return nil }, + CloseFunc: func() error { return nil }, + CurrentSizeFunc: func() int64 { return 0 }, + } + } + removedHeadNotifier := &mock.RemovedHeadNotifierMock{NotifyFunc: func() {}} + rotatedCounter := uint64(0) + rotatedTrigger := func() { rotatedCounter++ } + + proxyHead := storage.NewProxy( + container.NewWeighted( + s.createHead(s.nameIDGenerator(rotatedCounter), segmentWriters, rotatedCounter, shardsCount), + ), + keeper.NewKeeper[storage.Head](2, removedHeadNotifier), + func(*storage.Head) error { return nil }, + ) + + headBuilder := &mock.HeadBuilderMock[ + *task.Generic[*shard.PerGoroutineShard], + *shard.Shard, + *shard.PerGoroutineShard, + *storage.Head, + ]{BuildFunc: func(generation uint64, numberOfShards uint16) (*storage.Head, error) { + return s.createHead(s.nameIDGenerator(generation), segmentWriters, generation, numberOfShards), nil + }} + cfg := &mock.RotatorConfigMock{NumberOfShardsFunc: func() uint16 { return shardsCount }} + headInformer := &mock.HeadInformerMock{ + SetActiveStatusFunc: func(string) error { return nil }, + SetRotatedStatusFunc: func(string) error { return nil }, + CreatedAtFunc: func(string) time.Duration { return time.Duration(time.Now().UnixMilli()) }, + } + + rotator := services.NewRotator( + proxyHead, + headBuilder, + mediator, + cfg, + headInformer, + head.CopyAddedSeries[*shard.Shard, *shard.PerGoroutineShard](shard.CopyAddedSeries), + rotatedTrigger, + nil, + ) + + aHead := proxyHead.Get() + s.addLabelSetToHead(aHead) + expectedNumSeries := s.getNumSeriesFromHead(aHead) + + done := make(chan struct{}) + services.CopySeriesOnRotate = true + + s.T().Run("execute", func(t *testing.T) { + t.Parallel() + + err := rotator.Execute(s.baseCtx) + close(done) + s.NoError(err) + }) + + s.T().Run("tick", func(t *testing.T) { + t.Parallel() + + <-start + trigger <- struct{}{} + close(trigger) + <-done + + s.Require().NoError(proxyHead.Close()) + + s.Equal(uint64(1), rotatedCounter) + aHead := proxyHead.Get() + s.Equal(uint64(1), aHead.Generation()) + s.Equal(s.nameIDGenerator(rotatedCounter), aHead.ID()) + s.Equal(headInformer.SetActiveStatusCalls()[0].HeadID, aHead.ID()) + actualNumSeries := s.getNumSeriesFromHead(aHead) + s.Equal(expectedNumSeries, actualNumSeries) + + for _, segmentWriter := range segmentWriters { + if !s.Len(segmentWriter.WriteCalls(), 1) { + return + } + if !s.Len(segmentWriter.FlushCalls(), 1) { + return + } + if !s.Len(segmentWriter.SyncCalls(), 1) { + return + } + + for _, call := range segmentWriter.WriteCalls() { + s.Equal(uint32(0), call.Segment.Samples()) + } + } + + rHeads := proxyHead.Heads() + s.Len(rHeads, 1) + s.Equal(s.nameIDGenerator(rotatedCounter-1), rHeads[0].ID()) + s.Equal(headInformer.SetRotatedStatusCalls()[0].HeadID, rHeads[0].ID()) + s.True(rHeads[0].IsReadOnly()) + }) +} diff --git a/pp/go/storage/head/shard/data_storage.go b/pp/go/storage/head/shard/data_storage.go new file mode 100644 index 0000000000..3dafc62979 --- /dev/null +++ b/pp/go/storage/head/shard/data_storage.go @@ -0,0 +1,153 @@ +package shard + +import ( + "sync" + + "github.com/prometheus/prometheus/pp/go/cppbridge" +) + +// DataStorage samles storage with labels IDs. +type DataStorage struct { + dataStorage *cppbridge.HeadDataStorage + encoder *cppbridge.HeadEncoder + locker sync.RWMutex +} + +// NewDataStorage int new [DataStorage]. +func NewDataStorage() *DataStorage { + dataStorage := cppbridge.NewHeadDataStorage() + return &DataStorage{ + dataStorage: dataStorage, + encoder: cppbridge.NewHeadEncoderWithDataStorage(dataStorage), + locker: sync.RWMutex{}, + } +} + +// AllocatedMemory return size of allocated memory for [DataStorage]. +func (ds *DataStorage) AllocatedMemory() uint64 { + ds.locker.RLock() + am := ds.dataStorage.AllocatedMemory() + ds.locker.RUnlock() + + return am +} + +// AppendInnerSeriesSlice add InnerSeries to storage. +func (ds *DataStorage) AppendInnerSeriesSlice(innerSeriesSlice []*cppbridge.InnerSeries) { + ds.locker.Lock() + ds.encoder.EncodeInnerSeriesSlice(innerSeriesSlice) + ds.locker.Unlock() +} + +// DecodeSegment decode segment data from decoder [cppbridge.HeadWalDecoder] +// and add to encoder [cppbridge.HeadEncoder], returns createTs, encodeTs. +// +//revive:disable-next-line:confusing-results // returns createTs, encodeTs +//nolint:gocritic // returns createTs, encodeTs +func (ds *DataStorage) DecodeSegment(decoder *cppbridge.HeadWalDecoder, data []byte) (int64, int64, error) { + return decoder.DecodeToDataStorage(data, ds.encoder) +} + +// InstantQuery make instant query to data storage and returns samples. +func (ds *DataStorage) InstantQuery( + targetTimestamp, notFoundValueTimestampValue int64, + seriesIDs []uint32, +) ([]cppbridge.Sample, cppbridge.DataStorageQueryResult) { + ds.locker.RLock() + samples, res := ds.dataStorage.InstantQuery(targetTimestamp, notFoundValueTimestampValue, seriesIDs) + ds.locker.RUnlock() + + return samples, res +} + +// MergeOutOfOrderChunks merge chunks with out of order data chunks. +func (ds *DataStorage) MergeOutOfOrderChunks() { + ds.locker.Lock() + ds.encoder.MergeOutOfOrderChunks() + ds.locker.Unlock() +} + +// Query make query to data storage and returns serialazed chunks. +func (ds *DataStorage) Query( + query cppbridge.HeadDataStorageQuery, +) (*cppbridge.HeadDataStorageSerializedChunks, cppbridge.DataStorageQueryResult) { + ds.locker.RLock() + serializedChunks, res := ds.dataStorage.Query(query) + ds.locker.RUnlock() + + return serializedChunks, res +} + +// QueryFinal finishes all the queries after data load. +func (ds *DataStorage) QueryFinal(queriers []uintptr) { + ds.locker.RLock() + ds.dataStorage.QueryFinal(queriers) + ds.locker.RUnlock() +} + +// QueryStatus get head status from [DataStorage]. +func (ds *DataStorage) QueryStatus(status *cppbridge.HeadStatus) { + ds.locker.RLock() + status.FromDataStorage(ds.dataStorage) + ds.locker.RUnlock() +} + +// Raw returns raw [cppbridge.HeadDataStorage]. +func (ds *DataStorage) Raw() *cppbridge.HeadDataStorage { + return ds.dataStorage +} + +// WithLock calls fn on raw [cppbridge.HeadDataStorage] with write lock. +func (ds *DataStorage) WithLock(fn func(ds *cppbridge.HeadDataStorage) error) error { + ds.locker.Lock() + err := fn(ds.dataStorage) + ds.locker.Unlock() + + return err +} + +// WithRLock calls fn on raw [cppbridge.HeadDataStorage] with read lock. +func (ds *DataStorage) WithRLock(fn func(ds *cppbridge.HeadDataStorage) error) error { + ds.locker.RLock() + err := fn(ds.dataStorage) + ds.locker.RUnlock() + + return err +} + +// TimeInterval get time interval from [DataStorage]. +func (ds *DataStorage) TimeInterval(invalidateCache bool) cppbridge.TimeInterval { + ds.locker.RLock() + result := ds.dataStorage.TimeInterval(invalidateCache) + ds.locker.RUnlock() + + return result +} + +// CreateUnusedSeriesDataUnloader create unused series data unloader +func (ds *DataStorage) CreateUnusedSeriesDataUnloader() *cppbridge.UnusedSeriesDataUnloader { + return ds.dataStorage.CreateUnusedSeriesDataUnloader() +} + +// CreateLoader create series data unloader +func (ds *DataStorage) CreateLoader(queriers []uintptr) *cppbridge.UnloadedDataLoader { + return ds.dataStorage.CreateLoader(queriers) +} + +// CreateRevertableLoader create series data revertable unloader +func (ds *DataStorage) CreateRevertableLoader( + lss *cppbridge.LabelSetStorage, + lsIdBatchSize uint32, +) *cppbridge.UnloadedDataRevertableLoader { + return ds.dataStorage.CreateRevertableLoader(lss, lsIdBatchSize) +} + +// GetQueriedSeriesBitset gets the queried series bitset memory. +func (ds *DataStorage) GetQueriedSeriesBitset() []byte { + return ds.dataStorage.GetQueriedSeriesBitset() +} + +// SetQueriedSeriesBitset sets the queried series bitset. +func (ds *DataStorage) SetQueriedSeriesBitset(bitset []byte) bool { + return ds.dataStorage.SetQueriedSeriesBitset(bitset) +} diff --git a/pp/go/storage/head/shard/file_storage.go b/pp/go/storage/head/shard/file_storage.go new file mode 100644 index 0000000000..48348657a6 --- /dev/null +++ b/pp/go/storage/head/shard/file_storage.go @@ -0,0 +1,156 @@ +package shard + +import ( + "os" + + "github.com/prometheus/prometheus/pp/go/util" +) + +// FileStorage wrapper over [os.File] for convenient operation. +type FileStorage struct { + fileName string + file *os.File +} + +// NewFileStorage init new [FileStorage]. +func NewFileStorage(fileName string) *FileStorage { + return &FileStorage{fileName: fileName} +} + +// Close closes the [File], rendering it unusable for I/O. On files that support [File.SetDeadline], +// any pending I/O operations will be canceled and return immediately with an [ErrClosed] error. +// Close will return an error if it has already been called. +func (q *FileStorage) Close() error { + if q.file != nil { + return q.file.Close() + } + + return nil +} + +// IsEmpty returns true if file is empty. +func (q *FileStorage) IsEmpty() bool { + if q.file != nil { + if info, err := q.file.Stat(); err == nil { + return info.Size() == 0 + } + } + + return true +} + +// Open open file for [FileStorage] with flags. +func (q *FileStorage) Open(flags int) (err error) { + if q.file == nil { + q.file, err = os.OpenFile( //nolint:gosec // need this permissions + q.fileName, + flags, + 0o666, //revive:disable-line:add-constant // file permissions simple readable as octa-number + ) + } + + return err +} + +// Read reads up to len(b) bytes from the File and stores them in b. +// It returns the number of bytes read and any error encountered. At end of file, Read returns 0, io.EOF. +func (q *FileStorage) Read(p []byte) (n int, err error) { + return q.file.Read(p) +} + +// ReadAt reads len(b) bytes from the File starting at byte offset off. +// It returns the number of bytes read and the error, if any. +// ReadAt always returns a non-nil error when n < len(b). At end of file, that error is io.EOF. +func (q *FileStorage) ReadAt(p []byte, off int64) (n int, err error) { + return q.file.ReadAt(p, off) +} + +// Seek sets the offset for the next Read or Write on file to offset, +// interpreted according to whence: 0 means relative to the origin of the file, +// 1 means relative to the current offset, and 2 means relative to the end. +// It returns the new offset and an error, if any. +// The behavior of Seek on a file opened with [O_APPEND] is not specified. +func (q *FileStorage) Seek(offset int64, whence int) (int64, error) { + return q.file.Seek(offset, whence) +} + +// Sync commits the current contents of the file to stable storage. +// Typically, this means flushing the file system's in-memory copy of recently written data to disk. +func (q *FileStorage) Sync() error { + return q.file.Sync() +} + +// Truncate changes the size of the file. It does not change the I/O offset. +// If there is an error, it will be of type [*PathError]. +func (q *FileStorage) Truncate(size int64) error { + return q.file.Truncate(size) +} + +// Write writes len(b) bytes from b to the File. It returns the number of bytes written and an error, if any. +// Write returns a non-nil error when n != len(b). +func (q *FileStorage) Write(p []byte) (n int, err error) { + return q.file.Write(p) +} + +// +// AppendFileStorage +// + +// AppendFileStorage wrapper over [util.FileAppender] for convenient operation. +type AppendFileStorage struct { + fileName string + file *util.FileAppender +} + +// NewAppendFileStorage init new [AppendFileStorage]. +func NewAppendFileStorage(fileName string) *AppendFileStorage { + return &AppendFileStorage{fileName: fileName} +} + +// Close closes the [AppendFileStorage], rendering it unusable for I/O. On files that support [File.SetDeadline], +// any pending I/O operations will be canceled and return immediately with an [ErrClosed] error. +// Close will return an error if it has already been called. +func (q *AppendFileStorage) Close() error { + if q.file != nil { + return q.file.Close() + } + + return nil +} + +// IsEmpty returns true if file is empty. +func (q *AppendFileStorage) IsEmpty() bool { + if q.file != nil { + if info, err := q.file.Stat(); err == nil { + return info.Size() == 0 + } + } + + return true +} + +// Open open file for [AppendFileStorage]. +func (q *AppendFileStorage) Open() (err error) { + if q.file == nil { + q.file, err = util.CreateFileAppender(q.fileName, 0o666) + } + + return +} + +// Sync commits the current contents of the file to stable storage. +// Typically, this means flushing the file system's in-memory copy of recently written data to disk. +func (q *AppendFileStorage) Sync() error { + return q.file.Sync() +} + +// Reader returns a [StorageReader] from the [AppendFileStorage] for read file. +func (q *AppendFileStorage) Reader() (StorageReader, error) { + return os.Open(q.fileName) +} + +// Write writes len(b) bytes from b to the File. It returns the number of bytes written and an error, if any. +// Write returns a non-nil error when n != len(b). +func (q *AppendFileStorage) Write(p []byte) (n int, err error) { + return q.file.Write(p) +} diff --git a/pp/go/storage/head/shard/load_and_query_series_data_task.go b/pp/go/storage/head/shard/load_and_query_series_data_task.go new file mode 100644 index 0000000000..8d5562c6aa --- /dev/null +++ b/pp/go/storage/head/shard/load_and_query_series_data_task.go @@ -0,0 +1,38 @@ +package shard + +import ( + "sync" +) + +// Task the minimum required Task implementation. +type Task interface { + Wait() error +} + +// LoadAndQuerySeriesDataTask represents a task to load and query series data. +type LoadAndQuerySeriesDataTask struct { + queriers []uintptr + task Task + lock sync.Mutex +} + +// Add adds a querier to the task, if exists no task, it creates and enqueues a task. +func (t *LoadAndQuerySeriesDataTask) Add(querier uintptr, createAndEnqueueTask func() Task) Task { + t.lock.Lock() + defer t.lock.Unlock() + t.queriers = append(t.queriers, querier) + if len(t.queriers) == 1 { + t.task = createAndEnqueueTask() + } + + return t.task +} + +// Release executes and releases the queriers. +func (t *LoadAndQuerySeriesDataTask) Release(callback func([]uintptr)) { + t.lock.Lock() + callback(t.queriers) + t.queriers = nil + t.task = nil + t.lock.Unlock() +} diff --git a/pp/go/storage/head/shard/lss.go b/pp/go/storage/head/shard/lss.go new file mode 100644 index 0000000000..6a30a0d165 --- /dev/null +++ b/pp/go/storage/head/shard/lss.go @@ -0,0 +1,164 @@ +package shard + +import ( + "fmt" + "runtime" + "sync" + + "github.com/prometheus/prometheus/pp/go/cppbridge" + "github.com/prometheus/prometheus/pp/go/model" +) + +// LSS labelset storage for [shard]. +type LSS struct { + input *cppbridge.LabelSetStorage + target *cppbridge.LabelSetStorage + snapshot *cppbridge.LabelSetSnapshot + locker sync.RWMutex + once sync.Once +} + +// NewLSS init new [LSS]. +func NewLSS() *LSS { + return &LSS{ + input: cppbridge.NewLssStorage(), + target: cppbridge.NewQueryableLssStorage(), + } +} + +// AllocatedMemory return size of allocated memory for labelset storages. +func (l *LSS) AllocatedMemory() uint64 { + l.locker.RLock() + am := l.input.AllocatedMemory() + l.target.AllocatedMemory() + l.locker.RUnlock() + + return am +} + +// CopyAddedSeriesTo copy the label sets from the source lss to the destination lss that were added source lss. +func (l *LSS) CopyAddedSeriesTo(destination *LSS) { + l.locker.RLock() + snapshot := l.getSnapshot() + bitsetSeries := l.target.BitsetSeries() + l.locker.RUnlock() + + snapshot.CopyAddedSeries(bitsetSeries, destination.target) +} + +// Input returns input lss. +func (l *LSS) Input() *cppbridge.LabelSetStorage { + return l.input +} + +// QueryLabelNames add to dedup all the unique label names present in lss in sorted order. +func (l *LSS) QueryLabelNames( + shardID uint16, + matchers []model.LabelMatcher, + dedupAdd func(shardID uint16, snapshot *cppbridge.LabelSetSnapshot, values []string), +) error { + l.locker.RLock() + defer l.locker.RUnlock() + + queryLabelNamesResult := l.target.QueryLabelNames(matchers) + + if queryLabelNamesResult.Status() != cppbridge.LSSQueryStatusMatch { + return fmt.Errorf("no matches on shard: %d", shardID) + } + + dedupAdd(shardID, l.getSnapshot(), queryLabelNamesResult.Names()) + runtime.KeepAlive(queryLabelNamesResult) + + return nil +} + +// QueryLabelValues query labels values to [LSS] and add values to +// the dedup-container that matches the given label matchers. +func (l *LSS) QueryLabelValues( + shardID uint16, + name string, + matchers []model.LabelMatcher, + dedupAdd func(shardID uint16, snapshot *cppbridge.LabelSetSnapshot, values []string), +) error { + l.locker.RLock() + defer l.locker.RUnlock() + + queryLabelValuesResult := l.target.QueryLabelValues(name, matchers) + + if queryLabelValuesResult.Status() != cppbridge.LSSQueryStatusMatch { + return fmt.Errorf("no matches on shard: %d", shardID) + } + + dedupAdd(shardID, l.getSnapshot(), queryLabelValuesResult.Values()) + runtime.KeepAlive(queryLabelValuesResult) + + return nil +} + +// QuerySelector returns a created selector that matches the given label matchers. +func (l *LSS) QuerySelector(shardID uint16, matchers []model.LabelMatcher) ( + uintptr, + *cppbridge.LabelSetSnapshot, + error, +) { + l.locker.RLock() + defer l.locker.RUnlock() + + selector, status := l.target.QuerySelector(matchers) + switch status { + case cppbridge.LSSQueryStatusMatch: + return selector, l.getSnapshot(), nil + + case cppbridge.LSSQueryStatusNoMatch: + return 0, nil, nil + + default: + return 0, nil, fmt.Errorf( + "failed to query selector from shard: %d, query status: %d", shardID, status, + ) + } +} + +// QueryStatus get head status from [LSS]. +func (l *LSS) QueryStatus(status *cppbridge.HeadStatus, limit int) { + l.locker.RLock() + status.FromLSS(l.target, limit) + l.locker.RUnlock() +} + +// ResetSnapshot resets the current snapshot. Use only WithLock. +func (l *LSS) ResetSnapshot() { + l.snapshot = nil + l.once = sync.Once{} +} + +// Target returns main [LSS]. +func (l *LSS) Target() *cppbridge.LabelSetStorage { + return l.target +} + +// WithLock calls fn on raws [cppbridge.LabelSetStorage] with write lock. +func (l *LSS) WithLock(fn func(target, input *cppbridge.LabelSetStorage) error) error { + l.locker.Lock() + err := fn(l.target, l.input) + l.locker.Unlock() + + return err +} + +// WithRLock calls fn on raws [cppbridge.LabelSetStorage] with read lock. +func (l *LSS) WithRLock(fn func(target, input *cppbridge.LabelSetStorage) error) error { + l.locker.RLock() + err := fn(l.target, l.input) + l.locker.RUnlock() + + return err +} + +// getSnapshot return the actual snapshot. +func (l *LSS) getSnapshot() *cppbridge.LabelSetSnapshot { + l.once.Do(func() { + l.snapshot = l.target.CreateLabelSetSnapshot() + }) + + return l.snapshot +} diff --git a/pp/go/storage/head/shard/shard.go b/pp/go/storage/head/shard/shard.go new file mode 100644 index 0000000000..851b21de2d --- /dev/null +++ b/pp/go/storage/head/shard/shard.go @@ -0,0 +1,263 @@ +// Shard - TODO write description on package + +package shard + +import ( + "errors" + "fmt" + "time" + + "github.com/prometheus/prometheus/pp/go/cppbridge" +) + +// Wal the minimum required Wal implementation for a [Shard]. +type Wal interface { + // Close closes the wal segmentWriter. + Close() error + + // Commit finalize segment from encoder and write to wal. + Commit() error + + // CurrentSize returns current wal size. + CurrentSize() int64 + + // Flush flush all contetnt into wal. + Flush() error + + // Sync commits the current contents of the [Wal]. + Sync() error + + // Write append the incoming inner series to wal encoder. + Write(innerSeriesSlice []*cppbridge.InnerSeries) (bool, error) +} + +// +// Shard +// + +// Shard bridge to labelset storage, data storage and wal. +type Shard struct { + lss *LSS + dataStorage *DataStorage + unloadedDataStorage *UnloadedDataStorage + queriedSeriesStorage *QueriedSeriesStorage + loadAndQueryTask *LoadAndQuerySeriesDataTask + wal Wal + id uint16 +} + +// NewShard init new [Shard]. +func NewShard( + lss *LSS, + dataStorage *DataStorage, + unloadedDataStorage *UnloadedDataStorage, + queriedSeriesStorage *QueriedSeriesStorage, + wal Wal, + shardID uint16, +) *Shard { + return &Shard{ + id: shardID, + lss: lss, + dataStorage: dataStorage, + unloadedDataStorage: unloadedDataStorage, + queriedSeriesStorage: queriedSeriesStorage, + loadAndQueryTask: &LoadAndQuerySeriesDataTask{}, + wal: wal, + } +} + +// AppendInnerSeriesSlice add InnerSeries to [DataStorage]. +func (s *Shard) AppendInnerSeriesSlice(innerSeriesSlice []*cppbridge.InnerSeries) { + s.dataStorage.AppendInnerSeriesSlice(innerSeriesSlice) +} + +// Close closes the wal segmentWriter. +func (s *Shard) Close() error { + err := s.wal.Close() + + if s.unloadedDataStorage != nil { + err = errors.Join(err, s.unloadedDataStorage.Close()) + } + + if s.queriedSeriesStorage != nil { + err = errors.Join(err, s.queriedSeriesStorage.Close()) + } + + return err +} + +// DSAllocatedMemory return size of allocated memory for [DataStorage]. +func (s *Shard) DSAllocatedMemory() uint64 { + return s.dataStorage.AllocatedMemory() +} + +// DataStorage returns shard [DataStorage]. +func (s *Shard) DataStorage() *DataStorage { + return s.dataStorage +} + +// LSS returns shard labelset storage [LSS]. +func (s *Shard) LSS() *LSS { + return s.lss +} + +// LSSAllocatedMemory return size of allocated memory for labelset storages. +func (s *Shard) LSSAllocatedMemory() uint64 { + return s.lss.AllocatedMemory() +} + +// LSSWithLock calls fn on raws [cppbridge.LabelSetStorage] with write lock. +func (s *Shard) LSSWithLock(fn func(target, input *cppbridge.LabelSetStorage) error) error { + return s.lss.WithLock(fn) +} + +// LSSWithRLock calls fn on raws [cppbridge.LabelSetStorage] with read lock. +func (s *Shard) LSSWithRLock(fn func(target, input *cppbridge.LabelSetStorage) error) error { + return s.lss.WithRLock(fn) +} + +// LSSResetSnapshot resets the current snapshot. Use only WithLock. +func (s *Shard) LSSResetSnapshot() { + s.lss.ResetSnapshot() +} + +// MergeOutOfOrderChunks merge chunks with out of order data chunks in [DataStorage]. +func (s *Shard) MergeOutOfOrderChunks() { + s.dataStorage.MergeOutOfOrderChunks() +} + +// ShardID returns the shard ID. +func (s *Shard) ShardID() uint16 { + return s.id +} + +// WalCommit finalize segment from encoder and write to wal. +func (s *Shard) WalCommit() error { + return s.lss.WithRLock(func(_, _ *cppbridge.LabelSetStorage) error { + return s.wal.Commit() + }) +} + +// WalCurrentSize returns current [Wal] size. +func (s *Shard) WalCurrentSize() int64 { + return s.wal.CurrentSize() +} + +// WalFlush flush all contetnt into wal. +func (s *Shard) WalFlush() error { + return s.wal.Flush() +} + +// WalSync commits the current contents of the [Wal]. +func (s *Shard) WalSync() error { + return s.wal.Sync() +} + +// WalWrite append the incoming inner series to wal encoder. +func (s *Shard) WalWrite(innerSeriesSlice []*cppbridge.InnerSeries) (bool, error) { + return s.wal.Write(innerSeriesSlice) +} + +// TimeInterval get time interval from [DataStorage]. +func (s *Shard) TimeInterval(invalidateCache bool) cppbridge.TimeInterval { + return s.dataStorage.TimeInterval(invalidateCache) +} + +// UnloadedDataStorage get unloaded data storage +func (s *Shard) UnloadedDataStorage() *UnloadedDataStorage { + return s.unloadedDataStorage +} + +// QueriedSeriesStorage get queried series storage +func (s *Shard) QueriedSeriesStorage() *QueriedSeriesStorage { + return s.queriedSeriesStorage +} + +// LoadAndQuerySeriesDataTask get load and query series data task +func (s *Shard) LoadAndQuerySeriesDataTask() *LoadAndQuerySeriesDataTask { + return s.loadAndQueryTask +} + +// UnloadUnusedSeriesData unload unused series data +func (s *Shard) UnloadUnusedSeriesData() error { + if s.UnloadedDataStorage() == nil { + return nil + } + + unloader := s.DataStorage().CreateUnusedSeriesDataUnloader() + + var snapshot, queriedSeries []byte + _ = s.DataStorage().WithRLock(func(*cppbridge.HeadDataStorage) error { + snapshot = unloader.CreateSnapshot() + queriedSeries = s.DataStorage().GetQueriedSeriesBitset() + return nil + }) + + header, err := s.UnloadedDataStorage().WriteSnapshot(snapshot) + if err != nil { + return fmt.Errorf("unable to write unloaded series data snapshot: %v", err) + } + + _ = s.DataStorage().WithLock(func(*cppbridge.HeadDataStorage) error { + s.UnloadedDataStorage().WriteIndex(header) + unloader.Unload() + return nil + }) + + if err = s.QueriedSeriesStorage().Write(queriedSeries, time.Now().UnixMilli()); err != nil { + return fmt.Errorf("unable to write queried series data: %v", err) + } + + return nil +} + +// LoadAndQuerySeriesData loads the data and queries the series from the [DataStorage]. +func (s *Shard) LoadAndQuerySeriesData() (err error) { + var queriers []uintptr + s.loadAndQueryTask.Release(func(q []uintptr) { + queriers = q + err = s.DataStorage().WithLock(func(*cppbridge.HeadDataStorage) error { + loader := s.DataStorage().CreateLoader(queriers) + return s.UnloadedDataStorage().ForEachSnapshot(loader.Load) + }) + }) + if err != nil { + return err + } + + s.DataStorage().QueryFinal(queriers) + + return nil +} + +// +// PerGoroutineShard +// + +// PerGoroutineShard wrapper of shard with [PerGoroutineRelabeler] for goroutines. +type PerGoroutineShard struct { + relabeler *cppbridge.PerGoroutineRelabeler + *Shard +} + +// NewPerGoroutineShard init new [PerGoroutineShard]. +func NewPerGoroutineShard[TWal Wal](s *Shard, numberOfShards uint16) *PerGoroutineShard { + return &PerGoroutineShard{ + relabeler: cppbridge.NewPerGoroutineRelabeler(numberOfShards, s.ShardID()), + Shard: s, + } +} + +// Relabeler returns relabeler for shard goroutines. +func (s *PerGoroutineShard) Relabeler() *cppbridge.PerGoroutineRelabeler { + return s.relabeler +} + +// +// CopyAddedSeries +// + +// CopyAddedSeries copy the label sets from the source lss to the destination lss that were added source lss. +func CopyAddedSeries(source, destination *Shard) { + source.lss.CopyAddedSeriesTo(destination.lss) +} diff --git a/pp/go/storage/head/shard/unloaded_data_storage.go b/pp/go/storage/head/shard/unloaded_data_storage.go new file mode 100644 index 0000000000..eb4c50207f --- /dev/null +++ b/pp/go/storage/head/shard/unloaded_data_storage.go @@ -0,0 +1,365 @@ +package shard + +import ( + "errors" + "fmt" + "hash/crc32" + "io" + "os" + "unsafe" + + "github.com/prometheus/prometheus/pp/go/logger" +) + +const ( + // UnloadedDataStorageVersion file version for [UnloadedDataStorageVersion]. + UnloadedDataStorageVersion = 1 + + // QueriedSeriesStorageVersion file version for [QueriedSeriesStorage]. + QueriedSeriesStorageVersion = 1 +) + +// StorageFile wrapper over [os.File] for convenient operation. +type StorageFile interface { + Open(flags int) error + io.WriteCloser + io.ReadSeeker + io.ReaderAt + Sync() error + Truncate(size int64) error + IsEmpty() bool +} + +// StorageReader interface for reading from [os.File]. +type StorageReader interface { + io.Reader + io.ReaderAt + io.Closer +} + +// AppendFile interface for appending to [os.File]. +type AppendFile interface { + Open() error + io.Writer + io.Closer + Reader() (StorageReader, error) + Sync() error + IsEmpty() bool +} + +// UnloadedDataSnapshotHeader stubs for recording snapshots. +type UnloadedDataSnapshotHeader struct { + Crc32 uint32 + SnapshotSize uint32 +} + +// NewUnloadedDataSnapshotHeader init new [UnloadedDataSnapshotHeader]. +func NewUnloadedDataSnapshotHeader(snapshot []byte) UnloadedDataSnapshotHeader { + return UnloadedDataSnapshotHeader{ + Crc32: crc32.ChecksumIEEE(snapshot), + SnapshotSize: uint32(len(snapshot)), // #nosec G115 // no overflow + } +} + +// IsValid checks checksum if the header is valid. +func (h UnloadedDataSnapshotHeader) IsValid(snapshot []byte) bool { + return h.Crc32 == crc32.ChecksumIEEE(snapshot) +} + +// UnloadedDataStorage represents a unloaded data storage, unloads snapshots to the storage from [DataStorage]. +type UnloadedDataStorage struct { + storage AppendFile + snapshots []UnloadedDataSnapshotHeader + maxSnapshotSize uint32 +} + +// NewUnloadedDataStorage creates a new [UnloadedDataStorage]. +func NewUnloadedDataStorage(storage AppendFile) *UnloadedDataStorage { + return &UnloadedDataStorage{ + storage: storage, + } +} + +// WriteSnapshot writes a snapshot to the storage. +func (s *UnloadedDataStorage) WriteSnapshot(snapshot []byte) (UnloadedDataSnapshotHeader, error) { + if len(snapshot) == 0 { + return UnloadedDataSnapshotHeader{}, nil + } + + if err := s.storage.Open(); err != nil { + return UnloadedDataSnapshotHeader{}, err + } + + if len(s.snapshots) == 0 { + if err := s.WriteFormatVersion(); err != nil { + return UnloadedDataSnapshotHeader{}, err + } + } + + _, err := s.storage.Write(snapshot) + if err == nil { + err = s.storage.Sync() + } + return NewUnloadedDataSnapshotHeader(snapshot), err +} + +// WriteIndex writes an index to the storage. +func (s *UnloadedDataStorage) WriteIndex(header UnloadedDataSnapshotHeader) { + s.snapshots = append(s.snapshots, header) + s.maxSnapshotSize = max(header.SnapshotSize, s.maxSnapshotSize) +} + +// WriteFormatVersion writes the format version to the storage. +func (s *UnloadedDataStorage) WriteFormatVersion() error { + _, err := s.storage.Write([]byte{UnloadedDataStorageVersion}) + return err +} + +// ForEachSnapshot iterates over the snapshots and calls the callback function. +func (s *UnloadedDataStorage) ForEachSnapshot(f func(snapshot []byte, isLast bool)) error { + if len(s.snapshots) == 0 { + return nil + } + + reader, err := s.storage.Reader() + if err != nil { + return err + } + defer func() { + _ = reader.Close() + }() + + offset, err := s.validateFormatVersion(reader) + if err != nil { + return err + } + + snapshot := make([]byte, 0, s.maxSnapshotSize) + for index, header := range s.snapshots { + snapshot = snapshot[:header.SnapshotSize] + size, err := reader.ReadAt(snapshot, offset) + if size != int(header.SnapshotSize) { + return err + } + offset += int64(size) + + if !header.IsValid(snapshot) { + return fmt.Errorf("invalid snapshot at index %d", index) + } + + f(snapshot, index == len(s.snapshots)-1) + } + + return nil +} + +// validateFormatVersion validates the format version. +func (s *UnloadedDataStorage) validateFormatVersion(reader StorageReader) (offset int64, err error) { + version := []byte{0} + if _, err = reader.ReadAt(version, 0); err != nil { + return 0, err + } + + if version[0] != UnloadedDataStorageVersion { + return 0, fmt.Errorf("UnloadedDataStorage invalid version %d", version[0]) + } + + return int64(len(version)), nil +} + +// Close closes the storage. +func (s *UnloadedDataStorage) Close() (err error) { + if s.storage != nil { + err = s.storage.Close() + s.storage = nil + } + + return err +} + +// IsEmpty checks if the storage is empty. +func (s *UnloadedDataStorage) IsEmpty() bool { + return len(s.snapshots) == 0 +} + +// QueriedSeriesStorage represents a queried series storage, +// it contains two file stores that it swaps them as needed. +type QueriedSeriesStorage struct { + storages [2]StorageFile + validStorage StorageFile +} + +// NewQueriedSeriesStorage creates a new [QueriedSeriesStorage]. +func NewQueriedSeriesStorage(storage1, storage2 StorageFile) *QueriedSeriesStorage { + return &QueriedSeriesStorage{ + storages: [2]StorageFile{storage1, storage2}, //revive:disable-line:add-constant // 2 working files + } +} + +type queriedSeriesStorageHeader struct { + timestamp int64 + crc32 uint32 + size uint32 +} + +func (h *queriedSeriesStorageHeader) toSlice() []byte { + return (*(*[unsafe.Sizeof(queriedSeriesStorageHeader{})]byte)( + unsafe.Pointer(h), + ))[:] // #nosec G103 // it's meant to be that way +} + +func (h *queriedSeriesStorageHeader) CalculateCrc32(queriedSeriesBitset []byte) uint32 { + h.crc32 = 0 + + writer := crc32.NewIEEE() + _, _ = writer.Write(h.toSlice()) + _, _ = writer.Write(queriedSeriesBitset) + h.crc32 = writer.Sum32() + + return h.crc32 +} + +func (s *QueriedSeriesStorage) Write(queriedSeriesBitset []byte, timestamp int64) error { + storage := s.storages[0] + if err := storage.Open(os.O_RDWR | os.O_CREATE | os.O_TRUNC); err != nil { + s.changeActiveStorageIfNoValidStorage() + return err + } + + var headerBuffer [1 + unsafe.Sizeof(queriedSeriesStorageHeader{})]byte + headerBuffer[0] = UnloadedDataStorageVersion + + header := (*queriedSeriesStorageHeader)(unsafe.Pointer(&headerBuffer[1])) // #nosec G103 it's meant to be that way + header.timestamp = timestamp + header.size = uint32(len(queriedSeriesBitset)) // #nosec G115 // no overflow + header.CalculateCrc32(queriedSeriesBitset) + + if err := s.writeToStorage(storage, headerBuffer[:], queriedSeriesBitset); err != nil { + s.changeActiveStorageIfNoValidStorage() + return err + } + + s.validStorage = s.storages[0] + s.changeActiveStorage() + return nil +} + +func (*QueriedSeriesStorage) writeToStorage(storage StorageFile, headerBuffer, queriedSeriesBitset []byte) error { + if _, err := storage.Seek(0, io.SeekStart); err != nil { + return err + } + + if _, err := storage.Write(headerBuffer); err != nil { + return err + } + + if _, err := storage.Write(queriedSeriesBitset); err != nil { + return err + } + + if err := storage.Sync(); err != nil { + return err + } + + return storage.Truncate(int64(len(headerBuffer) + len(queriedSeriesBitset))) +} + +func (s *QueriedSeriesStorage) changeActiveStorage() { + s.storages[0], s.storages[1] = s.storages[1], s.storages[0] +} + +func (s *QueriedSeriesStorage) changeActiveStorageIfNoValidStorage() { + if s.validStorage == nil { + s.changeActiveStorage() + } +} + +func (s *QueriedSeriesStorage) Read() (data []byte, err error) { + readers, maxSize := s.readStorageHeaders() + data = make([]byte, 0, maxSize) + + for i := range readers { + data = data[:readers[i].size] + + if len(data) > 0 { + if _, err = io.ReadFull(readers[i].storage, data); err != nil { + logger.Warnf("failed to read data from queried series storage: %v", err) + continue + } + } + + if storageCrc32 := readers[i].crc32; storageCrc32 != readers[i].CalculateCrc32(data) { + logger.Warnf("invalid queried series storage crc32: %d != %d", storageCrc32, readers[i].crc32) + continue + } + + s.validStorage = readers[i].storage + if readers[i].storage == s.storages[0] { + s.changeActiveStorage() + } + + return data, nil + } + + return nil, errors.New("no valid queried series storage") +} + +func (s *QueriedSeriesStorage) readStorageHeaders() (result []storageHeaderReader, maxSize uint32) { + for _, storage := range s.storages { + reader := storageHeaderReader{storage: storage} + + if err := reader.read(); err == nil { + result = append(result, reader) + maxSize = max(maxSize, reader.size) + } else if !os.IsNotExist(err) && !errors.Is(err, io.EOF) { + logger.Warnf("failed to read header: %v", err) + } + } + + //revive:disable-next-line:add-constant // 2 working files + if len(result) == 2 && result[0].timestamp < result[1].timestamp { + result[0], result[1] = result[1], result[0] + } + + return result, maxSize +} + +// Close closes the storage. +func (s *QueriedSeriesStorage) Close() error { + return errors.Join(s.storages[0].Close(), s.storages[1].Close()) +} + +type storageHeaderReader struct { + queriedSeriesStorageHeader + storage StorageFile +} + +func (s *storageHeaderReader) read() error { + if err := s.storage.Open(os.O_RDWR); err != nil { + return err + } + + if err := s.readAndValidateFormatVersion(); err != nil { + return err + } + + _, err := io.ReadFull(s.storage, s.toSlice()) + return err +} + +func (s *storageHeaderReader) readAndValidateFormatVersion() error { + if _, err := s.storage.Seek(0, io.SeekStart); err != nil { + return err + } + + version := []byte{0} + if _, err := s.storage.Read(version); err != nil { + return err + } + + if version[0] != QueriedSeriesStorageVersion { + return fmt.Errorf("QueriedSeriesStorage invalid version %d", version[0]) + } + + return nil +} diff --git a/pp/go/storage/head/shard/unloaded_data_storage_test.go b/pp/go/storage/head/shard/unloaded_data_storage_test.go new file mode 100644 index 0000000000..5640032832 --- /dev/null +++ b/pp/go/storage/head/shard/unloaded_data_storage_test.go @@ -0,0 +1,534 @@ +package shard + +import ( + "bytes" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/suite" +) + +// BufferReaderAtWriterCloser implementation [AppendFile]. +type BufferReaderAtWriterCloser struct { + buffer []byte +} + +// IsEmpty implementation [AppendFile]. +func (*BufferReaderAtWriterCloser) IsEmpty() bool { + return true +} + +// Open implementation [AppendFile]. +func (*BufferReaderAtWriterCloser) Open() error { + return nil +} + +// Read implementation [AppendFile]. +func (*BufferReaderAtWriterCloser) Read([]byte) (n int, err error) { + return 0, nil +} + +// Reader implementation [AppendFile]. +func (s *BufferReaderAtWriterCloser) Reader() (StorageReader, error) { + return s, nil +} + +// Sync implementation [AppendFile]. +func (*BufferReaderAtWriterCloser) Sync() error { + return nil +} + +// ReadAt implementation [AppendFile]. +func (s *BufferReaderAtWriterCloser) ReadAt(p []byte, off int64) (n int, err error) { + return bytes.NewReader(s.buffer).ReadAt(p, off) +} + +// Write implementation [AppendFile]. +func (s *BufferReaderAtWriterCloser) Write(p []byte) (n int, err error) { + s.buffer = append(s.buffer, p...) + return len(p), nil +} + +// Close implementation [AppendFile]. +func (*BufferReaderAtWriterCloser) Close() error { + return nil +} + +type UnloadedDataStorageSuite struct { + suite.Suite + storageBuffer *BufferReaderAtWriterCloser + storage *UnloadedDataStorage +} + +func TestUnloadedDataStorageSuite(t *testing.T) { + suite.Run(t, new(UnloadedDataStorageSuite)) +} + +func (s *UnloadedDataStorageSuite) SetupTest() { + s.storageBuffer = &BufferReaderAtWriterCloser{} + s.storage = NewUnloadedDataStorage(s.storageBuffer) +} + +func (s *UnloadedDataStorageSuite) Write(snapshot []byte) { + header, _ := s.storage.WriteSnapshot(snapshot) + s.storage.WriteIndex(header) +} + +func (s *UnloadedDataStorageSuite) readSnapshots() ([]string, error) { + var snapshots []string + return snapshots, s.storage.ForEachSnapshot(func(snapshot []byte, _ bool) { + snapshots = append(snapshots, string(snapshot)) + }) +} + +func (s *UnloadedDataStorageSuite) TestWriteEmptySnapshot() { + // Arrange + + // Act + header, err := s.storage.WriteSnapshot(nil) + + // Assert + s.Require().NoError(err) + s.Equal(UnloadedDataSnapshotHeader{}, header) +} + +func (s *UnloadedDataStorageSuite) TestReadEmptySnapshots() { + // Arrange + s.storageBuffer.buffer = []byte{UnloadedDataStorageVersion} + + // Act + snapshots, err := s.readSnapshots() + + // Assert + s.Require().NoError(err) + s.Equal([]string(nil), snapshots) +} + +func (s *UnloadedDataStorageSuite) TestReadOneSnapshot() { + // Arrange + s.Write([]byte("12345")) + + // Act + snapshots, err := s.readSnapshots() + + // Assert + s.Require().NoError(err) + s.Equal([]string{"12345"}, snapshots) +} + +func (s *UnloadedDataStorageSuite) TestReadMultipleSnapshots() { + // Arrange + s.Write([]byte("123")) + s.Write([]byte("45678")) + s.Write([]byte("90")) + + // Act + snapshots, err := s.readSnapshots() + + // Assert + s.Require().NoError(err) + s.Equal([]string{"123", "45678", "90"}, snapshots) +} + +func (s *UnloadedDataStorageSuite) TestReadEof() { + // Arrange + s.Write([]byte("123")) + s.storageBuffer.buffer = s.storageBuffer.buffer[:len(s.storageBuffer.buffer)-1] + + // Act + snapshots, err := s.readSnapshots() + + // Assert + s.Require().ErrorIs(err, io.EOF) + s.Equal([]string(nil), snapshots) +} + +func (s *UnloadedDataStorageSuite) TestReadVersionError() { + // Arrange + s.Write([]byte("123")) + s.storageBuffer.buffer = nil + + // Act + snapshots, err := s.readSnapshots() + + // Assert + s.Require().Equal(io.EOF, err) + s.Equal([]string(nil), snapshots) +} + +func (s *UnloadedDataStorageSuite) TestInvalidVersion() { + // Arrange + s.Write([]byte("123")) + var invalidVersion byte = UnloadedDataStorageVersion + 1 + s.storageBuffer.buffer = []byte{invalidVersion} + + // Act + snapshots, err := s.readSnapshots() + + // Assert + s.Equal([]string(nil), snapshots) + s.Equal(fmt.Errorf("UnloadedDataStorage invalid version %d", invalidVersion), err) +} + +func (s *UnloadedDataStorageSuite) TestReadInvalidSnapshot() { + // Arrange + s.Write([]byte("123")) + s.Write([]byte("45678")) + s.storageBuffer.buffer[4] = 0x00 + + // Act + snapshots, err := s.readSnapshots() + + // Assert + s.Equal([]string{"123"}, snapshots) + s.Equal(fmt.Errorf("invalid snapshot at index 1"), err) +} + +type QueriedSeriesStorageSuite struct { + suite.Suite + file1 *FileStorage + file2 *FileStorage + storage *QueriedSeriesStorage +} + +func TestQueriedSeriesStorageWriterSuite(t *testing.T) { + suite.Run(t, new(QueriedSeriesStorageSuite)) +} + +func (s *QueriedSeriesStorageSuite) SetupTest() { + tempDir := s.T().TempDir() + s.file1 = &FileStorage{fileName: filepath.Join(tempDir, "file1")} + s.file2 = &FileStorage{fileName: filepath.Join(tempDir, "file2")} + s.storage = NewQueriedSeriesStorage(s.file1, s.file2) +} + +func (s *QueriedSeriesStorageSuite) TearDownTest() { + s.Require().NoError(s.storage.Close()) +} + +func (s *QueriedSeriesStorageSuite) writeFile(file *FileStorage, data []byte) { + s.Require().NoError(file.Open(os.O_RDWR | os.O_CREATE | os.O_TRUNC)) + _, err := file.Write(data) + s.Require().NoError(err) +} + +func (s *QueriedSeriesStorageSuite) readFile(file *FileStorage) []byte { + _, err := file.Seek(0, io.SeekStart) + s.Require().NoError(err) + + data, err := io.ReadAll(file) + s.Require().NoError(err) + + return data +} + +func (s *QueriedSeriesStorageSuite) TestOpenErrorOnWrite() { + // Arrange + s.file1.fileName = "" + + // Act + err := s.storage.Write([]byte("12345"), 1234567890) + + // Assert + s.Require().Error(err) + s.Nil(s.storage.validStorage) + s.Equal(s.file2, s.storage.storages[0]) +} + +func (s *QueriedSeriesStorageSuite) TestWriteInFirstStorage() { + // Arrange + + // Act + err := s.storage.Write([]byte("12345"), 1234567890) + + // Assert + s.Require().NoError(err) + s.Equal([]byte{ + QueriedSeriesStorageVersion, // version + 0xd2, 0x02, 0x96, 0x49, 0x00, 0x00, 0x00, 0x00, // timestamp + 0x4e, 0x78, 0xf9, 0xf3, // crc32 + 0x05, 0x00, 0x00, 0x00, // size + '1', '2', '3', '4', '5', // content + }, s.readFile(s.file1)) + s.Nil(s.file2.file) +} + +func (s *QueriedSeriesStorageSuite) TestWriteInAllStorages() { + // Arrange + + // Act + err1 := s.storage.Write([]byte("12345"), 1234567890) + err2 := s.storage.Write([]byte("67890"), 987654321) + + // Assert + s.NoError(err1) + s.NoError(err2) + s.Equal([]byte{ + QueriedSeriesStorageVersion, // version + 0xd2, 0x02, 0x96, 0x49, 0x00, 0x00, 0x00, 0x00, // timestamp + 0x4e, 0x78, 0xf9, 0xf3, // crc32 + 0x05, 0x00, 0x00, 0x00, // size + '1', '2', '3', '4', '5', // content + }, s.readFile(s.file1)) + s.Equal([]byte{ + QueriedSeriesStorageVersion, // version + 0xb1, 0x68, 0xde, 0x3a, 0x00, 0x00, 0x00, 0x00, // timestamp + 0x21, 0x33, 0xf7, 0xb8, // crc32 + 0x05, 0x00, 0x00, 0x00, // size + '6', '7', '8', '9', '0', // content + }, s.readFile(s.file2)) +} + +func (s *QueriedSeriesStorageSuite) TestMultipleWriteInFirstStorage() { + // Arrange + + // Act + _ = s.storage.Write([]byte("12345"), 1234567890) + _ = s.storage.Write([]byte("67890"), 987654321) + _ = s.storage.Write([]byte("67890"), 987654321) + _ = s.storage.Write([]byte("12345"), 1234567890) + + // Assert + s.Equal([]byte{ + QueriedSeriesStorageVersion, // version + 0xb1, 0x68, 0xde, 0x3a, 0x00, 0x00, 0x00, 0x00, // timestamp + 0x21, 0x33, 0xf7, 0xb8, // crc32 + 0x05, 0x00, 0x00, 0x00, // size + '6', '7', '8', '9', '0', // content + }, s.readFile(s.file1)) + s.Equal([]byte{ + QueriedSeriesStorageVersion, // version + 0xd2, 0x02, 0x96, 0x49, 0x00, 0x00, 0x00, 0x00, // timestamp + 0x4e, 0x78, 0xf9, 0xf3, // crc32 + 0x05, 0x00, 0x00, 0x00, // size + '1', '2', '3', '4', '5', // content + }, s.readFile(s.file2)) +} + +func (s *QueriedSeriesStorageSuite) TestOpenErrorInRead() { + // Arrange + s.file1.fileName = "" + + // Act + data, err := s.storage.Read() + + // Assert + s.Equal([]byte(nil), data) + s.Error(err) +} + +func (s *QueriedSeriesStorageSuite) TestChangeActiveFileOnOpenErrorWithoutValidFile() { + // Arrange + s.file1.fileName = "" + + // Act + writeErr1 := s.storage.Write([]byte("12345"), 1234567890) + writeErr2 := s.storage.Write([]byte("12345"), 1234567890) + + // Assert + s.Require().Error(writeErr1) + s.Require().NoError(writeErr2) + s.Equal([]byte{ + QueriedSeriesStorageVersion, // version + 0xd2, 0x02, 0x96, 0x49, 0x00, 0x00, 0x00, 0x00, // timestamp + 0x4e, 0x78, 0xf9, 0xf3, // crc32 + 0x05, 0x00, 0x00, 0x00, // size + '1', '2', '3', '4', '5', // content + }, s.readFile(s.file2)) +} + +func (s *QueriedSeriesStorageSuite) TestNoChangeActiveFileOnOpenErrorWithValidFile() { + // Arrange + s.file2.fileName = "" + s.writeFile(s.file1, []byte{ + QueriedSeriesStorageVersion, // version + 0xd2, 0x02, 0x96, 0x49, 0x00, 0x00, 0x00, 0x00, // timestamp + 0x4e, 0x78, 0xf9, 0xf3, // crc32 + 0x05, 0x00, 0x00, 0x00, + '1', '2', '3', '4', '5', + }) + + // Act + data, readErr := s.storage.Read() + writeErr1 := s.storage.Write([]byte("67890"), 987654321) + writeErr2 := s.storage.Write([]byte("67890"), 987654321) + + // Assert + s.Require().NoError(readErr) + s.Equal([]byte("12345"), data) + s.Require().Error(writeErr1) + s.Require().Error(writeErr2) +} + +func (s *QueriedSeriesStorageSuite) TestReadEmptyFiles() { + // Arrange + + // Act + data, err := s.storage.Read() + + // Assert + s.Equal([]byte(nil), data) + s.Equal(errors.New("no valid queried series storage"), err) +} + +func (s *QueriedSeriesStorageSuite) TestInvalidVersionInAllStorages() { + // Arrange + s.writeFile(s.file1, []byte{QueriedSeriesStorageVersion + 1}) + s.writeFile(s.file2, []byte{QueriedSeriesStorageVersion + 1}) + + // Act + data, err := s.storage.Read() + + // Assert + s.Equal([]byte(nil), data) + s.Equal(errors.New("no valid queried series storage"), err) +} + +func (s *QueriedSeriesStorageSuite) TestInvalidHeaderInAllStorages() { + // Arrange + invalidHeader := []byte{ + QueriedSeriesStorageVersion, // version + 0xd2, 0x02, 0x96, 0x49, 0x00, 0x00, 0x00, 0x00, // timestamp + 0x4e, 0x78, 0xf9, 0xf3, // crc32 + 0x05, 0x00, 0x00, + } + s.writeFile(s.file1, invalidHeader) + s.writeFile(s.file2, invalidHeader) + + // Act + data, err := s.storage.Read() + + // Assert + s.Equal([]byte(nil), data) + s.Equal(errors.New("no valid queried series storage"), err) +} + +func (s *QueriedSeriesStorageSuite) TestInvalidDataInAllStorages() { + // Arrange + invalidData := []byte{ + QueriedSeriesStorageVersion, // version + 0xd2, 0x02, 0x96, 0x49, 0x00, 0x00, 0x00, 0x00, // timestamp + 0x4e, 0x78, 0xf9, 0xf3, // crc32 + 0x05, 0x00, 0x00, 0x00, + '1', '2', '3', '4', + } + s.writeFile(s.file1, invalidData) + s.writeFile(s.file2, invalidData) + + // Act + data, err := s.storage.Read() + + // Assert + s.Equal([]byte(nil), data) + s.Equal(errors.New("no valid queried series storage"), err) +} + +func (s *QueriedSeriesStorageSuite) TestInvalidCrc32InAllStorages() { + // Arrange + invalidCrc32 := []byte{ + QueriedSeriesStorageVersion, // version + 0xd2, 0x02, 0x96, 0x49, 0x00, 0x00, 0x00, 0x00, // timestamp + 0x4e, 0x78, 0xf9, 0xf2, // crc32 + 0x05, 0x00, 0x00, 0x00, + '1', '2', '3', '4', '5', + } + s.writeFile(s.file1, invalidCrc32) + s.writeFile(s.file2, invalidCrc32) + + // Act + data, err := s.storage.Read() + + // Assert + s.Equal([]byte(nil), data) + s.Equal(errors.New("no valid queried series storage"), err) +} + +func (s *QueriedSeriesStorageSuite) TestReadFromFirstStorageAndChangeActiveStorage() { + // Arrange + s.writeFile(s.file1, []byte{ + QueriedSeriesStorageVersion, // version + 0xd2, 0x02, 0x96, 0x49, 0x00, 0x00, 0x00, 0x00, // timestamp + 0x4e, 0x78, 0xf9, 0xf3, // crc32 + 0x05, 0x00, 0x00, 0x00, + '1', '2', '3', '4', '5', + }) + + // Act + data, readErr := s.storage.Read() + _ = s.storage.Write([]byte("67890"), 987654321) + + // Assert + s.Require().NoError(readErr) + s.Equal([]byte("12345"), data) + s.Equal([]byte{ + QueriedSeriesStorageVersion, // version + 0xb1, 0x68, 0xde, 0x3a, 0x00, 0x00, 0x00, 0x00, // timestamp + 0x21, 0x33, 0xf7, 0xb8, // crc32 + 0x05, 0x00, 0x00, 0x00, // size + '6', '7', '8', '9', '0', // content + }, s.readFile(s.file2)) +} + +func (s *QueriedSeriesStorageSuite) TestReadFromSecondStorage() { + // Arrange + s.writeFile(s.file2, []byte{ + QueriedSeriesStorageVersion, // version + 0xd2, 0x02, 0x96, 0x49, 0x00, 0x00, 0x00, 0x00, // timestamp + 0x4e, 0x78, 0xf9, 0xf3, // crc32 + 0x05, 0x00, 0x00, 0x00, + '1', '2', '3', '4', '5', + }) + + // Act + data, err := s.storage.Read() + + // Assert + s.Require().NoError(err) + s.Equal([]byte("12345"), data) +} + +func (s *QueriedSeriesStorageSuite) TestReadFromStorageWithMaxTimestamp() { + // Arrange + s.writeFile(s.file1, []byte{ + QueriedSeriesStorageVersion, // version + 0xb1, 0x68, 0xde, 0x3a, 0x00, 0x00, 0x00, 0x00, // timestamp + 0x21, 0x33, 0xf7, 0xb8, // crc32 + 0x05, 0x00, 0x00, 0x00, // size + '6', '7', '8', '9', '0', // content + }) + s.writeFile(s.file2, []byte{ + QueriedSeriesStorageVersion, // version + 0xd3, 0x02, 0x96, 0x49, 0x00, 0x00, 0x00, 0x00, // timestamp + 0xfd, 0x12, 0xf7, 0xe0, // crc32 + 0x04, 0x00, 0x00, 0x00, + '6', '7', '8', '9', + }) + + // Act + data, err := s.storage.Read() + + // Assert + s.Require().NoError(err) + s.Equal([]byte("6789"), data) +} + +func (s *QueriedSeriesStorageSuite) TestReadEmptyContent() { + // Arrange + s.writeFile(s.file1, []byte{ + QueriedSeriesStorageVersion, // version + 0xb1, 0x68, 0xde, 0x3a, 0x00, 0x00, 0x00, 0x00, // timestamp + 0x41, 0x01, 0x44, 0x30, // crc32 + 0x00, 0x00, 0x00, 0x00, // size + }) + + // Act + data, err := s.storage.Read() + + // Assert + s.Require().NoError(err) + s.Equal([]byte{}, data) +} diff --git a/pp/go/storage/head/shard/wal/reader/byte_reader.go b/pp/go/storage/head/shard/wal/reader/byte_reader.go new file mode 100644 index 0000000000..e7fc00ef31 --- /dev/null +++ b/pp/go/storage/head/shard/wal/reader/byte_reader.go @@ -0,0 +1,30 @@ +package reader + +import "io" + +// ByteReader reads from the reader 1 byte at a time. +type ByteReader struct { + r io.Reader + buf []byte + n int +} + +// NewByteReader init new [byteReader] +func NewByteReader(r io.Reader) *ByteReader { + return &ByteReader{ + r: r, + buf: make([]byte, 1), + } +} + +// ReadByte reads from the reader 1 byte. +func (r *ByteReader) ReadByte() (byte, error) { + n, err := io.ReadFull(r.r, r.buf) + if err != nil { + return 0, err + } + + r.n += n + + return r.buf[0], nil +} diff --git a/pp/go/storage/head/shard/wal/reader/byte_reader_test.go b/pp/go/storage/head/shard/wal/reader/byte_reader_test.go new file mode 100644 index 0000000000..da7c4de11e --- /dev/null +++ b/pp/go/storage/head/shard/wal/reader/byte_reader_test.go @@ -0,0 +1,43 @@ +package reader_test + +import ( + "bytes" + "io" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/prometheus/prometheus/pp/go/storage/head/shard/wal/reader" +) + +func TestRead(t *testing.T) { + bb := &bytes.Buffer{} + br := reader.NewByteReader(bb) + data := []byte{1, 42, 3, 0} + + _, err := bb.Write(data) + require.NoError(t, err) + + for _, expectedV := range data { + actualV, errRead := br.ReadByte() + require.NoError(t, errRead) + require.Equal(t, expectedV, actualV) + } + + _, err = br.ReadByte() + require.ErrorIs(t, err, io.EOF) +} + +func BenchmarkBR1(b *testing.B) { + bb := &bytes.Buffer{} + br := reader.NewByteReader(bb) + + buf := []byte{1, 2, 3} + + for i := 0; i < b.N; i++ { + _, _ = bb.Write(buf) + _, _ = br.ReadByte() + _, _ = br.ReadByte() + _, _ = br.ReadByte() + } +} diff --git a/pp/go/storage/head/shard/wal/reader/header.go b/pp/go/storage/head/shard/wal/reader/header.go new file mode 100644 index 0000000000..f5b2be57c2 --- /dev/null +++ b/pp/go/storage/head/shard/wal/reader/header.go @@ -0,0 +1,29 @@ +package reader + +import ( + "encoding/binary" + "fmt" + "io" +) + +// ReadHeader read header from reader. +// +//revive:disable-next-line:function-result-limit there is no point in packing it into a structure. +func ReadHeader(reader io.Reader) (fileFormatVersion, encoderVersion uint8, n int, err error) { + br := NewByteReader(reader) + fileFormatVersionU64, err := binary.ReadUvarint(br) + if err != nil { + return 0, 0, n, fmt.Errorf("failed to read file format version: %w", err) + } + fileFormatVersion = uint8(fileFormatVersionU64) // #nosec G115 // no overflow + n = br.n + + encoderVersionU64, err := binary.ReadUvarint(br) + if err != nil { + return 0, 0, n, fmt.Errorf("failed to read encoder version: %w", err) + } + encoderVersion = uint8(encoderVersionU64) // #nosec G115 // no overflow + n = br.n + + return fileFormatVersion, encoderVersion, n, nil +} diff --git a/pp/go/storage/head/shard/wal/reader/header_test.go b/pp/go/storage/head/shard/wal/reader/header_test.go new file mode 100644 index 0000000000..0cc3066d0c --- /dev/null +++ b/pp/go/storage/head/shard/wal/reader/header_test.go @@ -0,0 +1,24 @@ +package reader_test + +import ( + "bytes" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/prometheus/prometheus/pp/go/storage/head/shard/wal/reader" +) + +func TestWriteHeader(t *testing.T) { + buf := &bytes.Buffer{} + expected := []byte{21, 42} + + _, err := buf.Write(expected) + require.NoError(t, err) + + fileFormatVersion, encoderVersion, _, err := reader.ReadHeader(buf) + require.NoError(t, err) + + require.Equal(t, expected[0], fileFormatVersion) + require.Equal(t, expected[1], encoderVersion) +} diff --git a/pp/go/storage/head/shard/wal/reader/segment.go b/pp/go/storage/head/shard/wal/reader/segment.go new file mode 100644 index 0000000000..42bcafaff8 --- /dev/null +++ b/pp/go/storage/head/shard/wal/reader/segment.go @@ -0,0 +1,91 @@ +package reader + +import ( + "encoding/binary" + "fmt" + "hash/crc32" + "io" +) + +// Segment encoded segment from wal. +type Segment struct { + data []byte + sampleCount uint32 +} + +// NewSegment init new [Segment]. +func NewSegment() *Segment { + return &Segment{} +} + +// Bytes returns the data read. +func (s *Segment) Bytes() []byte { + return s.data +} + +// Length returns the length of slice byte. +func (s *Segment) Length() int { + return len(s.data) +} + +// ReadFrom reads [Segment] data from r [io.Reader]. The return value n is the number of bytes read. +// Any error encountered during the read is also returned. +func (s *Segment) ReadFrom(r io.Reader) (int64, error) { + return readSegment(r, s) +} + +// Reset [Segment] data. +func (s *Segment) Reset() { + s.data = s.data[:0] + s.sampleCount = 0 +} + +// Samples returns count of samples in [Segment]. +func (s *Segment) Samples() uint32 { + return s.sampleCount +} + +// resize [Segment] data. +func (s *Segment) resize(size int) { + if cap(s.data) < size { + s.data = make([]byte, size) + } else { + s.data = s.data[:size] + } +} + +// readSegment read and decode [Segment] from [io.Reader] and returns. +func readSegment(reader io.Reader, segment *Segment) (int64, error) { + br := NewByteReader(reader) + size, err := binary.ReadUvarint(br) + if err != nil { + return int64(br.n), fmt.Errorf("failed to read segment size: %w", err) + } + + crc32HashU64, err := binary.ReadUvarint(br) + if err != nil { + return int64(br.n), fmt.Errorf("failed to read segment crc32 hash: %w", err) + } + crc32Hash := uint32(crc32HashU64) // #nosec G115 // no overflow + + sampleCountU64, err := binary.ReadUvarint(br) + if err != nil { + return int64(br.n), fmt.Errorf("failed to read segment sample count: %w", err) + } + segment.sampleCount = uint32(sampleCountU64) // #nosec G115 // no overflow + + segment.resize(int(size)) // #nosec G115 // no overflow + n, err := io.ReadFull(reader, segment.data) + if err != nil { + return int64(br.n), fmt.Errorf("failed to read segment data: %w", err) + } + n += br.n + + if crc32Hash != crc32.ChecksumIEEE(segment.data) { + return int64(n), fmt.Errorf( + "crc32 did not match, want: %d, have: %d", crc32Hash, crc32.ChecksumIEEE(segment.data), + ) + } + + return int64(n), nil +} diff --git a/pp/go/storage/head/shard/wal/reader/segment_test.go b/pp/go/storage/head/shard/wal/reader/segment_test.go new file mode 100644 index 0000000000..83c661d4bd --- /dev/null +++ b/pp/go/storage/head/shard/wal/reader/segment_test.go @@ -0,0 +1,122 @@ +package reader_test + +import ( + "bytes" + "encoding/binary" + "hash/crc32" + "io" + "testing" + + "github.com/go-faker/faker/v4" + "github.com/stretchr/testify/suite" + + "github.com/prometheus/prometheus/pp/go/storage/head/shard/wal/reader" +) + +type SegmentSuite struct { + suite.Suite +} + +func TestSegmentSuite(t *testing.T) { + suite.Run(t, new(SegmentSuite)) +} + +func (s *SegmentSuite) TestHappyPath() { + data := []byte(faker.Paragraph()) + segmentSamples := uint32(42) + toWrite := []byte{} + toWrite = append(toWrite, binary.AppendUvarint(nil, uint64(len(data)))...) + toWrite = append(toWrite, binary.AppendUvarint(nil, uint64(crc32.ChecksumIEEE(data)))...) + toWrite = append(toWrite, byte(segmentSamples)) + toWrite = append(toWrite, data...) + + buf := &bytes.Buffer{} + _, err := buf.Write(toWrite) + s.Require().NoError(err) + + segment := reader.NewSegment() + _, err = segment.ReadFrom(buf) + s.Require().NoError(err) + + s.Require().Equal(len(data), segment.Length()) + s.Require().Equal(segmentSamples, segment.Samples()) + s.Require().Equal(data, segment.Bytes()) +} + +func (s *SegmentSuite) TestReuseSegment() { + data := []byte(faker.Paragraph()) + segmentSamples := uint32(42) + toWrite := []byte{} + toWrite = append(toWrite, binary.AppendUvarint(nil, uint64(len(data)))...) + toWrite = append(toWrite, binary.AppendUvarint(nil, uint64(crc32.ChecksumIEEE(data)))...) + toWrite = append(toWrite, byte(segmentSamples)) + toWrite = append(toWrite, data...) + + buf := &bytes.Buffer{} + _, err := buf.Write(toWrite) + s.Require().NoError(err) + + segment := reader.NewSegment() + _, err = segment.ReadFrom(buf) + s.Require().NoError(err) + + s.Require().Equal(len(data), segment.Length()) + s.Require().Equal(segmentSamples, segment.Samples()) + s.Require().Equal(data, segment.Bytes()) + + buf.Reset() + toWrite = toWrite[:0] + data = []byte(faker.Paragraph()) + toWrite = append(toWrite, binary.AppendUvarint(nil, uint64(len(data)))...) + toWrite = append(toWrite, binary.AppendUvarint(nil, uint64(crc32.ChecksumIEEE(data)))...) + toWrite = append(toWrite, byte(segmentSamples)) + toWrite = append(toWrite, data...) + + _, err = buf.Write(toWrite) + s.Require().NoError(err) + + segment.Reset() + _, err = segment.ReadFrom(buf) + s.Require().NoError(err) + + s.Require().Equal(len(data), segment.Length()) + s.Require().Equal(segmentSamples, segment.Samples()) + s.Require().Equal(data, segment.Bytes()) +} + +func (s *SegmentSuite) TestCrc32Error() { + data := []byte(faker.Paragraph()) + segmentSamples := uint32(42) + toWrite := []byte{} + toWrite = append(toWrite, binary.AppendUvarint(nil, uint64(len(data)))...) + toWrite = append(toWrite, binary.AppendUvarint(nil, uint64(0))...) + toWrite = append(toWrite, byte(segmentSamples)) + toWrite = append(toWrite, data...) + + buf := &bytes.Buffer{} + _, err := buf.Write(toWrite) + s.Require().NoError(err) + + segment := reader.NewSegment() + _, err = segment.ReadFrom(buf) + s.Require().Error(err) +} + +func (s *SegmentSuite) TestCutSegment() { + data := []byte(faker.Paragraph()) + segmentCrc32 := crc32.ChecksumIEEE(data) + segmentSamples := uint32(42) + toWrite := []byte{} + toWrite = append(toWrite, binary.AppendUvarint(nil, uint64(len(data)))...) + toWrite = append(toWrite, binary.AppendUvarint(nil, uint64(segmentCrc32))...) + toWrite = append(toWrite, byte(segmentSamples)) + toWrite = append(toWrite, data[:len(data)-2]...) + + buf := &bytes.Buffer{} + _, err := buf.Write(toWrite) + s.Require().NoError(err) + + segment := reader.NewSegment() + _, err = segment.ReadFrom(buf) + s.Require().ErrorIs(err, io.ErrUnexpectedEOF) +} diff --git a/pp/go/storage/head/shard/wal/wal.go b/pp/go/storage/head/shard/wal/wal.go new file mode 100644 index 0000000000..26de043725 --- /dev/null +++ b/pp/go/storage/head/shard/wal/wal.go @@ -0,0 +1,237 @@ +package wal + +import ( + "errors" + "fmt" + "io" + "strconv" + "sync" + + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/prometheus/pp/go/cppbridge" + "github.com/prometheus/prometheus/pp/go/util" +) + +//go:generate -command moq go run github.com/matryer/moq --rm --skip-ensure --pkg wal_test --out +//go:generate moq wal_moq_test.go . SegmentWriter Encoder EncodedSegment + +const ( + // FileFormatVersion wal file version. + FileFormatVersion = 1 +) + +// ErrWalIsCorrupted errror when wal is corrupted. +var ErrWalIsCorrupted = errors.New("wal is corrupted") + +// SegmentWriter writer for wal segments. +type SegmentWriter[TSegment EncodedSegment] interface { + // CurrentSize return current shard wal size. + CurrentSize() int64 + + // Write encoded segment to writer. + Write(segment TSegment) error + + // Flush write all buffered segments. + Flush() error + + // Sync commits the current contents of the [SegmentWriter]. + Sync() error + + // Close closes the storage. + Close() error +} + +// Encoder the minimum required Encoder implementation for a [Wal]. +type Encoder[TSegment EncodedSegment] interface { + // Encode encodes the inner series into a segment. + Encode(innerSeriesSlice []*cppbridge.InnerSeries) (uint32, error) + + // Finalize finalizes the encoder and returns the encoded segment. + Finalize() (TSegment, error) +} + +// EncodedSegment the minimum required Segment implementation for a [Wal]. +type EncodedSegment interface { + // Size returns the size of the segment. + Size() int64 + + // CRC32 returns the CRC32 of the segment. + CRC32() uint32 + + // Samples returns the number of samples in the segment. + Samples() uint32 + + // WriteTo implements [io.WriterTo] interface. + io.WriterTo +} + +// Wal write-ahead log for [Shard]. +type Wal[TSegment EncodedSegment, TWriter SegmentWriter[TSegment]] struct { + encoder Encoder[TSegment] // *cppbridge.HeadWalEncoder + segmentWriter TWriter + locker sync.Mutex + maxSegmentSize uint32 + corrupted bool + limitExhausted bool + closed bool + // stat + samplesPerSegment prometheus.Counter + sizePerSegment prometheus.Counter + segments prometheus.Gauge +} + +// NewWal init new [Wal]. +func NewWal[TSegment EncodedSegment, TWriter SegmentWriter[TSegment]]( + encoder Encoder[TSegment], + segmentWriter TWriter, + maxSegmentSize uint32, + shardID uint16, + registerer prometheus.Registerer, +) *Wal[TSegment, TWriter] { + factory := util.NewUnconflictRegisterer(registerer) + ls := prometheus.Labels{"shard_id": strconv.FormatUint(uint64(shardID), 10)} + w := &Wal[TSegment, TWriter]{ + encoder: encoder, + segmentWriter: segmentWriter, + locker: sync.Mutex{}, + maxSegmentSize: maxSegmentSize, + samplesPerSegment: factory.NewCounter(prometheus.CounterOpts{ + Name: "prompp_shard_wal_samples_per_segment_sum", + Help: "Number of samples per segment.", + ConstLabels: ls, + }), + sizePerSegment: factory.NewCounter(prometheus.CounterOpts{ + Name: "prompp_shard_wal_size_per_segment_sum", + Help: "Size of segment.", + ConstLabels: ls, + }), + segments: factory.NewGauge(prometheus.GaugeOpts{ + Name: "prompp_shard_wal_segments", + Help: "Number of segments.", + ConstLabels: ls, + }), + } + + w.segments.Set(0) + + return w +} + +// NewCorruptedWal init new corrupted [Wal]. +func NewCorruptedWal[ + TSegment EncodedSegment, + TWriter SegmentWriter[TSegment], +]() *Wal[TSegment, TWriter] { + return &Wal[TSegment, TWriter]{ + locker: sync.Mutex{}, + corrupted: true, + } +} + +// Close closes the wal segmentWriter. +func (w *Wal[TSegment, TWriter]) Close() error { + if w.corrupted { + return nil + } + + w.locker.Lock() + defer w.locker.Unlock() + + if w.closed { + return nil + } + + if err := w.segmentWriter.Close(); err != nil { + return err + } + + w.closed = true + + return nil +} + +// Commit finalize segment from encoder and write to [SegmentWriter]. +// It is necessary to lock the LSS for reading for the commit. +func (w *Wal[TSegment, TWriter]) Commit() error { + if w.corrupted { + return ErrWalIsCorrupted + } + + w.locker.Lock() + defer w.locker.Unlock() + + segment, err := w.encoder.Finalize() + if err != nil { + return fmt.Errorf("failed to finalize segment: %w", err) + } + w.samplesPerSegment.Add(float64(segment.Samples())) + w.sizePerSegment.Add(float64(segment.Size())) + w.segments.Inc() + w.limitExhausted = false + + if err = w.segmentWriter.Write(segment); err != nil { + return fmt.Errorf("failed to write segment: %w", err) + } + + return nil +} + +// CurrentSize returns current wal size. +func (w *Wal[TSegment, TWriter]) CurrentSize() int64 { + if w.corrupted { + return 0 + } + + return w.segmentWriter.CurrentSize() +} + +// Flush wal [SegmentWriter], write all buffered data to storage. +func (w *Wal[TSegment, TWriter]) Flush() error { + if w.corrupted { + return nil + } + + w.locker.Lock() + defer w.locker.Unlock() + + return w.segmentWriter.Flush() +} + +// Sync commits the current contents of the [SegmentWriter]. +func (w *Wal[TSegment, TWriter]) Sync() error { + if w.corrupted { + return ErrWalIsCorrupted + } + + w.locker.Lock() + defer w.locker.Unlock() + + return w.segmentWriter.Sync() +} + +// Write the incoming inner series to wal encoder. +func (w *Wal[TSegment, TWriter]) Write(innerSeriesSlice []*cppbridge.InnerSeries) (bool, error) { + if w.corrupted { + return false, ErrWalIsCorrupted + } + + w.locker.Lock() + defer w.locker.Unlock() + + samples, err := w.encoder.Encode(innerSeriesSlice) + if err != nil { + return false, fmt.Errorf("failed to encode inner series: %w", err) + } + + if w.maxSegmentSize == 0 { + return false, nil + } + + // memoize reaching of limits to deduplicate triggers + if !w.limitExhausted && samples >= w.maxSegmentSize { + w.limitExhausted = true + return true, nil + } + + return false, nil +} diff --git a/pp/go/storage/head/shard/wal/wal_moq_test.go b/pp/go/storage/head/shard/wal/wal_moq_test.go new file mode 100644 index 0000000000..c333066433 --- /dev/null +++ b/pp/go/storage/head/shard/wal/wal_moq_test.go @@ -0,0 +1,493 @@ +// Code generated by moq; DO NOT EDIT. +// github.com/matryer/moq + +package wal_test + +import ( + "github.com/prometheus/prometheus/pp/go/cppbridge" + "github.com/prometheus/prometheus/pp/go/storage/head/shard/wal" + "io" + "sync" +) + +// SegmentWriterMock is a mock implementation of wal.SegmentWriter. +// +// func TestSomethingThatUsesSegmentWriter(t *testing.T) { +// +// // make and configure a mocked wal.SegmentWriter +// mockedSegmentWriter := &SegmentWriterMock{ +// CloseFunc: func() error { +// panic("mock out the Close method") +// }, +// CurrentSizeFunc: func() int64 { +// panic("mock out the CurrentSize method") +// }, +// FlushFunc: func() error { +// panic("mock out the Flush method") +// }, +// SyncFunc: func() error { +// panic("mock out the Sync method") +// }, +// WriteFunc: func(segment TSegment) error { +// panic("mock out the Write method") +// }, +// } +// +// // use mockedSegmentWriter in code that requires wal.SegmentWriter +// // and then make assertions. +// +// } +type SegmentWriterMock[TSegment wal.EncodedSegment] struct { + // CloseFunc mocks the Close method. + CloseFunc func() error + + // CurrentSizeFunc mocks the CurrentSize method. + CurrentSizeFunc func() int64 + + // FlushFunc mocks the Flush method. + FlushFunc func() error + + // SyncFunc mocks the Sync method. + SyncFunc func() error + + // WriteFunc mocks the Write method. + WriteFunc func(segment TSegment) error + + // calls tracks calls to the methods. + calls struct { + // Close holds details about calls to the Close method. + Close []struct { + } + // CurrentSize holds details about calls to the CurrentSize method. + CurrentSize []struct { + } + // Flush holds details about calls to the Flush method. + Flush []struct { + } + // Sync holds details about calls to the Sync method. + Sync []struct { + } + // Write holds details about calls to the Write method. + Write []struct { + // Segment is the segment argument value. + Segment TSegment + } + } + lockClose sync.RWMutex + lockCurrentSize sync.RWMutex + lockFlush sync.RWMutex + lockSync sync.RWMutex + lockWrite sync.RWMutex +} + +// Close calls CloseFunc. +func (mock *SegmentWriterMock[TSegment]) Close() error { + if mock.CloseFunc == nil { + panic("SegmentWriterMock.CloseFunc: method is nil but SegmentWriter.Close was just called") + } + callInfo := struct { + }{} + mock.lockClose.Lock() + mock.calls.Close = append(mock.calls.Close, callInfo) + mock.lockClose.Unlock() + return mock.CloseFunc() +} + +// CloseCalls gets all the calls that were made to Close. +// Check the length with: +// +// len(mockedSegmentWriter.CloseCalls()) +func (mock *SegmentWriterMock[TSegment]) CloseCalls() []struct { +} { + var calls []struct { + } + mock.lockClose.RLock() + calls = mock.calls.Close + mock.lockClose.RUnlock() + return calls +} + +// CurrentSize calls CurrentSizeFunc. +func (mock *SegmentWriterMock[TSegment]) CurrentSize() int64 { + if mock.CurrentSizeFunc == nil { + panic("SegmentWriterMock.CurrentSizeFunc: method is nil but SegmentWriter.CurrentSize was just called") + } + callInfo := struct { + }{} + mock.lockCurrentSize.Lock() + mock.calls.CurrentSize = append(mock.calls.CurrentSize, callInfo) + mock.lockCurrentSize.Unlock() + return mock.CurrentSizeFunc() +} + +// CurrentSizeCalls gets all the calls that were made to CurrentSize. +// Check the length with: +// +// len(mockedSegmentWriter.CurrentSizeCalls()) +func (mock *SegmentWriterMock[TSegment]) CurrentSizeCalls() []struct { +} { + var calls []struct { + } + mock.lockCurrentSize.RLock() + calls = mock.calls.CurrentSize + mock.lockCurrentSize.RUnlock() + return calls +} + +// Flush calls FlushFunc. +func (mock *SegmentWriterMock[TSegment]) Flush() error { + if mock.FlushFunc == nil { + panic("SegmentWriterMock.FlushFunc: method is nil but SegmentWriter.Flush was just called") + } + callInfo := struct { + }{} + mock.lockFlush.Lock() + mock.calls.Flush = append(mock.calls.Flush, callInfo) + mock.lockFlush.Unlock() + return mock.FlushFunc() +} + +// FlushCalls gets all the calls that were made to Flush. +// Check the length with: +// +// len(mockedSegmentWriter.FlushCalls()) +func (mock *SegmentWriterMock[TSegment]) FlushCalls() []struct { +} { + var calls []struct { + } + mock.lockFlush.RLock() + calls = mock.calls.Flush + mock.lockFlush.RUnlock() + return calls +} + +// Sync calls SyncFunc. +func (mock *SegmentWriterMock[TSegment]) Sync() error { + if mock.SyncFunc == nil { + panic("SegmentWriterMock.SyncFunc: method is nil but SegmentWriter.Sync was just called") + } + callInfo := struct { + }{} + mock.lockSync.Lock() + mock.calls.Sync = append(mock.calls.Sync, callInfo) + mock.lockSync.Unlock() + return mock.SyncFunc() +} + +// SyncCalls gets all the calls that were made to Sync. +// Check the length with: +// +// len(mockedSegmentWriter.SyncCalls()) +func (mock *SegmentWriterMock[TSegment]) SyncCalls() []struct { +} { + var calls []struct { + } + mock.lockSync.RLock() + calls = mock.calls.Sync + mock.lockSync.RUnlock() + return calls +} + +// Write calls WriteFunc. +func (mock *SegmentWriterMock[TSegment]) Write(segment TSegment) error { + if mock.WriteFunc == nil { + panic("SegmentWriterMock.WriteFunc: method is nil but SegmentWriter.Write was just called") + } + callInfo := struct { + Segment TSegment + }{ + Segment: segment, + } + mock.lockWrite.Lock() + mock.calls.Write = append(mock.calls.Write, callInfo) + mock.lockWrite.Unlock() + return mock.WriteFunc(segment) +} + +// WriteCalls gets all the calls that were made to Write. +// Check the length with: +// +// len(mockedSegmentWriter.WriteCalls()) +func (mock *SegmentWriterMock[TSegment]) WriteCalls() []struct { + Segment TSegment +} { + var calls []struct { + Segment TSegment + } + mock.lockWrite.RLock() + calls = mock.calls.Write + mock.lockWrite.RUnlock() + return calls +} + +// EncoderMock is a mock implementation of wal.Encoder. +// +// func TestSomethingThatUsesEncoder(t *testing.T) { +// +// // make and configure a mocked wal.Encoder +// mockedEncoder := &EncoderMock{ +// EncodeFunc: func(innerSeriesSlice []*cppbridge.InnerSeries) (uint32, error) { +// panic("mock out the Encode method") +// }, +// FinalizeFunc: func() (TSegment, error) { +// panic("mock out the Finalize method") +// }, +// } +// +// // use mockedEncoder in code that requires wal.Encoder +// // and then make assertions. +// +// } +type EncoderMock[TSegment wal.EncodedSegment] struct { + // EncodeFunc mocks the Encode method. + EncodeFunc func(innerSeriesSlice []*cppbridge.InnerSeries) (uint32, error) + + // FinalizeFunc mocks the Finalize method. + FinalizeFunc func() (TSegment, error) + + // calls tracks calls to the methods. + calls struct { + // Encode holds details about calls to the Encode method. + Encode []struct { + // InnerSeriesSlice is the innerSeriesSlice argument value. + InnerSeriesSlice []*cppbridge.InnerSeries + } + // Finalize holds details about calls to the Finalize method. + Finalize []struct { + } + } + lockEncode sync.RWMutex + lockFinalize sync.RWMutex +} + +// Encode calls EncodeFunc. +func (mock *EncoderMock[TSegment]) Encode(innerSeriesSlice []*cppbridge.InnerSeries) (uint32, error) { + if mock.EncodeFunc == nil { + panic("EncoderMock.EncodeFunc: method is nil but Encoder.Encode was just called") + } + callInfo := struct { + InnerSeriesSlice []*cppbridge.InnerSeries + }{ + InnerSeriesSlice: innerSeriesSlice, + } + mock.lockEncode.Lock() + mock.calls.Encode = append(mock.calls.Encode, callInfo) + mock.lockEncode.Unlock() + return mock.EncodeFunc(innerSeriesSlice) +} + +// EncodeCalls gets all the calls that were made to Encode. +// Check the length with: +// +// len(mockedEncoder.EncodeCalls()) +func (mock *EncoderMock[TSegment]) EncodeCalls() []struct { + InnerSeriesSlice []*cppbridge.InnerSeries +} { + var calls []struct { + InnerSeriesSlice []*cppbridge.InnerSeries + } + mock.lockEncode.RLock() + calls = mock.calls.Encode + mock.lockEncode.RUnlock() + return calls +} + +// Finalize calls FinalizeFunc. +func (mock *EncoderMock[TSegment]) Finalize() (TSegment, error) { + if mock.FinalizeFunc == nil { + panic("EncoderMock.FinalizeFunc: method is nil but Encoder.Finalize was just called") + } + callInfo := struct { + }{} + mock.lockFinalize.Lock() + mock.calls.Finalize = append(mock.calls.Finalize, callInfo) + mock.lockFinalize.Unlock() + return mock.FinalizeFunc() +} + +// FinalizeCalls gets all the calls that were made to Finalize. +// Check the length with: +// +// len(mockedEncoder.FinalizeCalls()) +func (mock *EncoderMock[TSegment]) FinalizeCalls() []struct { +} { + var calls []struct { + } + mock.lockFinalize.RLock() + calls = mock.calls.Finalize + mock.lockFinalize.RUnlock() + return calls +} + +// EncodedSegmentMock is a mock implementation of wal.EncodedSegment. +// +// func TestSomethingThatUsesEncodedSegment(t *testing.T) { +// +// // make and configure a mocked wal.EncodedSegment +// mockedEncodedSegment := &EncodedSegmentMock{ +// CRC32Func: func() uint32 { +// panic("mock out the CRC32 method") +// }, +// SamplesFunc: func() uint32 { +// panic("mock out the Samples method") +// }, +// SizeFunc: func() int64 { +// panic("mock out the Size method") +// }, +// WriteToFunc: func(w io.Writer) (int64, error) { +// panic("mock out the WriteTo method") +// }, +// } +// +// // use mockedEncodedSegment in code that requires wal.EncodedSegment +// // and then make assertions. +// +// } +type EncodedSegmentMock struct { + // CRC32Func mocks the CRC32 method. + CRC32Func func() uint32 + + // SamplesFunc mocks the Samples method. + SamplesFunc func() uint32 + + // SizeFunc mocks the Size method. + SizeFunc func() int64 + + // WriteToFunc mocks the WriteTo method. + WriteToFunc func(w io.Writer) (int64, error) + + // calls tracks calls to the methods. + calls struct { + // CRC32 holds details about calls to the CRC32 method. + CRC32 []struct { + } + // Samples holds details about calls to the Samples method. + Samples []struct { + } + // Size holds details about calls to the Size method. + Size []struct { + } + // WriteTo holds details about calls to the WriteTo method. + WriteTo []struct { + // W is the w argument value. + W io.Writer + } + } + lockCRC32 sync.RWMutex + lockSamples sync.RWMutex + lockSize sync.RWMutex + lockWriteTo sync.RWMutex +} + +// CRC32 calls CRC32Func. +func (mock *EncodedSegmentMock) CRC32() uint32 { + if mock.CRC32Func == nil { + panic("EncodedSegmentMock.CRC32Func: method is nil but EncodedSegment.CRC32 was just called") + } + callInfo := struct { + }{} + mock.lockCRC32.Lock() + mock.calls.CRC32 = append(mock.calls.CRC32, callInfo) + mock.lockCRC32.Unlock() + return mock.CRC32Func() +} + +// CRC32Calls gets all the calls that were made to CRC32. +// Check the length with: +// +// len(mockedEncodedSegment.CRC32Calls()) +func (mock *EncodedSegmentMock) CRC32Calls() []struct { +} { + var calls []struct { + } + mock.lockCRC32.RLock() + calls = mock.calls.CRC32 + mock.lockCRC32.RUnlock() + return calls +} + +// Samples calls SamplesFunc. +func (mock *EncodedSegmentMock) Samples() uint32 { + if mock.SamplesFunc == nil { + panic("EncodedSegmentMock.SamplesFunc: method is nil but EncodedSegment.Samples was just called") + } + callInfo := struct { + }{} + mock.lockSamples.Lock() + mock.calls.Samples = append(mock.calls.Samples, callInfo) + mock.lockSamples.Unlock() + return mock.SamplesFunc() +} + +// SamplesCalls gets all the calls that were made to Samples. +// Check the length with: +// +// len(mockedEncodedSegment.SamplesCalls()) +func (mock *EncodedSegmentMock) SamplesCalls() []struct { +} { + var calls []struct { + } + mock.lockSamples.RLock() + calls = mock.calls.Samples + mock.lockSamples.RUnlock() + return calls +} + +// Size calls SizeFunc. +func (mock *EncodedSegmentMock) Size() int64 { + if mock.SizeFunc == nil { + panic("EncodedSegmentMock.SizeFunc: method is nil but EncodedSegment.Size was just called") + } + callInfo := struct { + }{} + mock.lockSize.Lock() + mock.calls.Size = append(mock.calls.Size, callInfo) + mock.lockSize.Unlock() + return mock.SizeFunc() +} + +// SizeCalls gets all the calls that were made to Size. +// Check the length with: +// +// len(mockedEncodedSegment.SizeCalls()) +func (mock *EncodedSegmentMock) SizeCalls() []struct { +} { + var calls []struct { + } + mock.lockSize.RLock() + calls = mock.calls.Size + mock.lockSize.RUnlock() + return calls +} + +// WriteTo calls WriteToFunc. +func (mock *EncodedSegmentMock) WriteTo(w io.Writer) (int64, error) { + if mock.WriteToFunc == nil { + panic("EncodedSegmentMock.WriteToFunc: method is nil but EncodedSegment.WriteTo was just called") + } + callInfo := struct { + W io.Writer + }{ + W: w, + } + mock.lockWriteTo.Lock() + mock.calls.WriteTo = append(mock.calls.WriteTo, callInfo) + mock.lockWriteTo.Unlock() + return mock.WriteToFunc(w) +} + +// WriteToCalls gets all the calls that were made to WriteTo. +// Check the length with: +// +// len(mockedEncodedSegment.WriteToCalls()) +func (mock *EncodedSegmentMock) WriteToCalls() []struct { + W io.Writer +} { + var calls []struct { + W io.Writer + } + mock.lockWriteTo.RLock() + calls = mock.calls.WriteTo + mock.lockWriteTo.RUnlock() + return calls +} diff --git a/pp/go/storage/head/shard/wal/wal_reader.go b/pp/go/storage/head/shard/wal/wal_reader.go new file mode 100644 index 0000000000..d4a1f31caa --- /dev/null +++ b/pp/go/storage/head/shard/wal/wal_reader.go @@ -0,0 +1,60 @@ +package wal + +import ( + "bufio" + "errors" + "fmt" + "io" +) + +//go:generate -command moq go run github.com/matryer/moq --rm --skip-ensure --pkg wal_test --out +//go:generate moq wal_reader_moq_test.go . ReadSegment + +// ReadSegment the minimum required [Segment] implementation for a [Wal]. +type ReadSegment interface { + // ReadFrom reads [ReadSegment] data from r [io.Reader]. The return value n is the number of bytes read. + // Any error encountered during the read is also returned. + ReadFrom(r io.Reader) (int64, error) + + // Reset [ReadSegment] data. + Reset() +} + +// SegmentWalReader buffered reader [ReadSegment]s from wal. +type SegmentWalReader[TReadSegment ReadSegment] struct { + reader *bufio.Reader + segmentCtor func() TReadSegment +} + +// NewSegmentWalReader init new [SegmentWalReader]. +func NewSegmentWalReader[TReadSegment ReadSegment]( + r io.Reader, + segmentCtor func() TReadSegment, +) *SegmentWalReader[TReadSegment] { + return &SegmentWalReader[TReadSegment]{ + reader: bufio.NewReaderSize(r, 1024*1024*4), + segmentCtor: segmentCtor, + } +} + +// ForEachSegment reads [ReadSegment]s from the reader and for each [ReadSegment] a [do] is called for each, +// if an error occurs during reading it will return and reading will stop. +func (r *SegmentWalReader[TReadSegment]) ForEachSegment(do func(TReadSegment) error) error { + segment := r.segmentCtor() + for { + segment.Reset() + + if _, err := segment.ReadFrom(r.reader); err != nil { + if errors.Is(err, io.EOF) { + break + } + return fmt.Errorf("failed to read segment: %w", err) + } + + if err := do(segment); err != nil { + return err + } + } + + return nil +} diff --git a/pp/go/storage/head/shard/wal/wal_reader_moq_test.go b/pp/go/storage/head/shard/wal/wal_reader_moq_test.go new file mode 100644 index 0000000000..4343b2e695 --- /dev/null +++ b/pp/go/storage/head/shard/wal/wal_reader_moq_test.go @@ -0,0 +1,108 @@ +// Code generated by moq; DO NOT EDIT. +// github.com/matryer/moq + +package wal_test + +import ( + "io" + "sync" +) + +// ReadSegmentMock is a mock implementation of wal.ReadSegment. +// +// func TestSomethingThatUsesReadSegment(t *testing.T) { +// +// // make and configure a mocked wal.ReadSegment +// mockedReadSegment := &ReadSegmentMock{ +// ReadFromFunc: func(r io.Reader) (int64, error) { +// panic("mock out the ReadFrom method") +// }, +// ResetFunc: func() { +// panic("mock out the Reset method") +// }, +// } +// +// // use mockedReadSegment in code that requires wal.ReadSegment +// // and then make assertions. +// +// } +type ReadSegmentMock struct { + // ReadFromFunc mocks the ReadFrom method. + ReadFromFunc func(r io.Reader) (int64, error) + + // ResetFunc mocks the Reset method. + ResetFunc func() + + // calls tracks calls to the methods. + calls struct { + // ReadFrom holds details about calls to the ReadFrom method. + ReadFrom []struct { + // R is the r argument value. + R io.Reader + } + // Reset holds details about calls to the Reset method. + Reset []struct { + } + } + lockReadFrom sync.RWMutex + lockReset sync.RWMutex +} + +// ReadFrom calls ReadFromFunc. +func (mock *ReadSegmentMock) ReadFrom(r io.Reader) (int64, error) { + if mock.ReadFromFunc == nil { + panic("ReadSegmentMock.ReadFromFunc: method is nil but ReadSegment.ReadFrom was just called") + } + callInfo := struct { + R io.Reader + }{ + R: r, + } + mock.lockReadFrom.Lock() + mock.calls.ReadFrom = append(mock.calls.ReadFrom, callInfo) + mock.lockReadFrom.Unlock() + return mock.ReadFromFunc(r) +} + +// ReadFromCalls gets all the calls that were made to ReadFrom. +// Check the length with: +// +// len(mockedReadSegment.ReadFromCalls()) +func (mock *ReadSegmentMock) ReadFromCalls() []struct { + R io.Reader +} { + var calls []struct { + R io.Reader + } + mock.lockReadFrom.RLock() + calls = mock.calls.ReadFrom + mock.lockReadFrom.RUnlock() + return calls +} + +// Reset calls ResetFunc. +func (mock *ReadSegmentMock) Reset() { + if mock.ResetFunc == nil { + panic("ReadSegmentMock.ResetFunc: method is nil but ReadSegment.Reset was just called") + } + callInfo := struct { + }{} + mock.lockReset.Lock() + mock.calls.Reset = append(mock.calls.Reset, callInfo) + mock.lockReset.Unlock() + mock.ResetFunc() +} + +// ResetCalls gets all the calls that were made to Reset. +// Check the length with: +// +// len(mockedReadSegment.ResetCalls()) +func (mock *ReadSegmentMock) ResetCalls() []struct { +} { + var calls []struct { + } + mock.lockReset.RLock() + calls = mock.calls.Reset + mock.lockReset.RUnlock() + return calls +} diff --git a/pp/go/storage/head/shard/wal/wal_reader_test.go b/pp/go/storage/head/shard/wal/wal_reader_test.go new file mode 100644 index 0000000000..25a9588f8a --- /dev/null +++ b/pp/go/storage/head/shard/wal/wal_reader_test.go @@ -0,0 +1,159 @@ +package wal_test + +import ( + "bytes" + "errors" + "io" + "testing" + + "github.com/go-faker/faker/v4" + "github.com/prometheus/prometheus/pp/go/storage/head/shard/wal" + "github.com/stretchr/testify/suite" +) + +type SegmentWalReaderSuite struct { + suite.Suite +} + +func TestSegmentWalReaderSuite(t *testing.T) { + suite.Run(t, new(SegmentWalReaderSuite)) +} + +func (s *SegmentWalReaderSuite) TestHappyPath() { + buf := &bytes.Buffer{} + data := []byte(faker.Paragraph()) + data = data[:(len(data)/10)*10] + _, err := buf.Write(data) + s.Require().NoError(err) + + swr := wal.NewSegmentWalReader(buf, newTestSegment) + limiter := 0 + actual := make([]byte, 0, len(data)) + err = swr.ForEachSegment(func(rsm *testSegment) error { + actual = append(actual, rsm.Bytes()...) + + // protect from infinite loop + limiter++ + if limiter == 1000 { + return errors.New("limiter") + } + + return nil + }) + s.Require().NoError(err) + + s.Equal(data, actual) +} + +func (s *SegmentWalReaderSuite) TestForEachSegmentError() { + buf := &bytes.Buffer{} + data := []byte(faker.Paragraph()) + data = data[:(len(data)/10)*10] + _, err := buf.Write(data) + s.Require().NoError(err) + + swr := wal.NewSegmentWalReader(buf, newTestSegment) + limiter := 0 + actual := make([]byte, 0, len(data)) + expectedError := errors.New("test error") + err = swr.ForEachSegment(func(rsm *testSegment) error { + actual = append(actual, rsm.Bytes()...) + + // protect from infinite loop + limiter++ + if limiter == 1 { + return expectedError + } + + return nil + }) + s.Require().ErrorIs(err, expectedError) +} + +func (s *SegmentWalReaderSuite) TestForEachSegmentReadError() { + buf := &bytes.Buffer{} + data := []byte(faker.Paragraph()) + data = data[:(len(data)/10)*10] + _, err := buf.Write(data) + s.Require().NoError(err) + + expectedError := errors.New("test error") + swr := wal.NewSegmentWalReader(buf, newTestSegmentWithError(expectedError)) + limiter := 0 + actual := make([]byte, 0, len(data)) + err = swr.ForEachSegment(func(rsm *testSegment) error { + actual = append(actual, rsm.Bytes()...) + + // protect from infinite loop + limiter++ + if limiter == 1 { + return errors.New("another error") + } + + return nil + }) + s.Require().ErrorIs(err, expectedError) +} + +// +// testSegment +// + +// testSegment implements [ReadSegment]. +type testSegment struct { + buf []byte + *ReadSegmentMock +} + +// newTestSegment init new [testSegment]. +func newTestSegment() *testSegment { + s := &testSegment{ + buf: make([]byte, 10), + } + + s.ReadSegmentMock = &ReadSegmentMock{ + ReadFromFunc: func(r io.Reader) (int64, error) { + n, err := io.ReadFull(r, s.buf) + return int64(n), err + }, + ResetFunc: func() { + for i := range s.buf { + s.buf[i] = 0 + } + }, + } + + return s +} + +// newTestSegmentWithError init new [testSegment] with error. +func newTestSegmentWithError(err error) func() *testSegment { + return func() *testSegment { + s := &testSegment{ + buf: make([]byte, 10), + } + + s.ReadSegmentMock = &ReadSegmentMock{ + ReadFromFunc: func(r io.Reader) (int64, error) { + n, errRead := io.ReadFull(r, s.buf) + if errRead != nil { + return int64(n), errRead + } + + return int64(n), err + }, + ResetFunc: func() { + for i := range s.buf { + s.buf[i] = 0 + } + }, + } + + return s + } +} + +// Bytes returns data. +func (s *testSegment) Bytes() []byte { + return s.buf +} diff --git a/pp/go/storage/head/shard/wal/wal_test.go b/pp/go/storage/head/shard/wal/wal_test.go new file mode 100644 index 0000000000..9942de8107 --- /dev/null +++ b/pp/go/storage/head/shard/wal/wal_test.go @@ -0,0 +1,281 @@ +package wal_test + +import ( + "errors" + "testing" + + "github.com/stretchr/testify/suite" + + "github.com/prometheus/prometheus/pp/go/cppbridge" + "github.com/prometheus/prometheus/pp/go/storage/head/shard/wal" +) + +type WalSuite struct { + suite.Suite +} + +func TestWalSuite(t *testing.T) { + suite.Run(t, new(WalSuite)) +} + +func (s *WalSuite) TestCurrentSize() { + expectedWalSize := int64(42) + enc := &EncoderMock[*EncodedSegmentMock]{} + segmentWriter := &SegmentWriterMock[*EncodedSegmentMock]{ + CurrentSizeFunc: func() int64 { return expectedWalSize }, + CloseFunc: func() error { return nil }, + } + maxSegmentSize := uint32(100) + + wl := wal.NewWal(enc, segmentWriter, maxSegmentSize, 0, nil) + + s.Equal(expectedWalSize, wl.CurrentSize()) + + s.Require().NoError(wl.Close()) + s.Len(segmentWriter.CloseCalls(), 1) +} + +func (s *WalSuite) TestClose() { + enc := &EncoderMock[*EncodedSegmentMock]{} + segmentWriter := &SegmentWriterMock[*EncodedSegmentMock]{ + CloseFunc: func() error { return nil }, + } + maxSegmentSize := uint32(100) + + wl := wal.NewWal(enc, segmentWriter, maxSegmentSize, 0, nil) + + s.Require().NoError(wl.Close()) + s.Len(segmentWriter.CloseCalls(), 1) + + s.Require().NoError(wl.Close()) + s.Len(segmentWriter.CloseCalls(), 1) +} + +func (s *WalSuite) TestCloseError() { + expectedError := errors.New("test error") + enc := &EncoderMock[*EncodedSegmentMock]{} + segmentWriter := &SegmentWriterMock[*EncodedSegmentMock]{ + CloseFunc: func() error { return expectedError }, + } + maxSegmentSize := uint32(100) + + wl := wal.NewWal(enc, segmentWriter, maxSegmentSize, 0, nil) + + s.Require().ErrorIs(wl.Close(), expectedError) + s.Len(segmentWriter.CloseCalls(), 1) +} + +func (s *WalSuite) TestCommit() { + enc := &EncoderMock[*EncodedSegmentMock]{ + FinalizeFunc: func() (*EncodedSegmentMock, error) { + return &EncodedSegmentMock{ + SizeFunc: func() int64 { return 42 }, + SamplesFunc: func() uint32 { return 42 }, + }, nil + }, + } + segmentWriter := &SegmentWriterMock[*EncodedSegmentMock]{ + WriteFunc: func(*EncodedSegmentMock) error { return nil }, + } + maxSegmentSize := uint32(100) + + wl := wal.NewWal(enc, segmentWriter, maxSegmentSize, 0, nil) + + s.Require().NoError(wl.Commit()) + s.Len(enc.FinalizeCalls(), 1) + s.Len(segmentWriter.WriteCalls(), 1) +} + +func (s *WalSuite) TestCommitEncodeError() { + expectedError := errors.New("test error") + enc := &EncoderMock[*EncodedSegmentMock]{ + FinalizeFunc: func() (*EncodedSegmentMock, error) { return &EncodedSegmentMock{}, expectedError }, + } + segmentWriter := &SegmentWriterMock[*EncodedSegmentMock]{ + WriteFunc: func(*EncodedSegmentMock) error { return nil }, + } + maxSegmentSize := uint32(100) + + wl := wal.NewWal(enc, segmentWriter, maxSegmentSize, 0, nil) + + s.Require().ErrorIs(wl.Commit(), expectedError) + s.Len(enc.FinalizeCalls(), 1) + s.Empty(segmentWriter.WriteCalls()) +} + +func (s *WalSuite) TestCommitWriteError() { + expectedError := errors.New("test error") + enc := &EncoderMock[*EncodedSegmentMock]{ + FinalizeFunc: func() (*EncodedSegmentMock, error) { + return &EncodedSegmentMock{ + SizeFunc: func() int64 { return 42 }, + SamplesFunc: func() uint32 { return 42 }, + }, nil + }, + } + segmentWriter := &SegmentWriterMock[*EncodedSegmentMock]{ + WriteFunc: func(*EncodedSegmentMock) error { return expectedError }, + } + maxSegmentSize := uint32(100) + + wl := wal.NewWal(enc, segmentWriter, maxSegmentSize, 0, nil) + + s.Require().ErrorIs(wl.Commit(), expectedError) + s.Len(enc.FinalizeCalls(), 1) + s.Len(segmentWriter.WriteCalls(), 1) +} + +func (s *WalSuite) TestFlush() { + enc := &EncoderMock[*EncodedSegmentMock]{} + segmentWriter := &SegmentWriterMock[*EncodedSegmentMock]{ + FlushFunc: func() error { return nil }, + } + maxSegmentSize := uint32(100) + + wl := wal.NewWal(enc, segmentWriter, maxSegmentSize, 0, nil) + + s.Require().NoError(wl.Flush()) + s.Len(segmentWriter.FlushCalls(), 1) +} + +func (s *WalSuite) TestFlushError() { + expectedError := errors.New("test error") + enc := &EncoderMock[*EncodedSegmentMock]{} + segmentWriter := &SegmentWriterMock[*EncodedSegmentMock]{ + FlushFunc: func() error { return expectedError }, + } + maxSegmentSize := uint32(100) + + wl := wal.NewWal(enc, segmentWriter, maxSegmentSize, 0, nil) + + s.Require().ErrorIs(wl.Flush(), expectedError) + s.Len(segmentWriter.FlushCalls(), 1) +} + +func (s *WalSuite) TestSync() { + enc := &EncoderMock[*EncodedSegmentMock]{} + segmentWriter := &SegmentWriterMock[*EncodedSegmentMock]{ + SyncFunc: func() error { return nil }, + } + maxSegmentSize := uint32(100) + + wl := wal.NewWal(enc, segmentWriter, maxSegmentSize, 0, nil) + + s.Require().NoError(wl.Sync()) + s.Len(segmentWriter.SyncCalls(), 1) +} + +func (s *WalSuite) TestSyncError() { + expectedError := errors.New("test error") + enc := &EncoderMock[*EncodedSegmentMock]{} + segmentWriter := &SegmentWriterMock[*EncodedSegmentMock]{ + SyncFunc: func() error { return expectedError }, + } + maxSegmentSize := uint32(100) + + wl := wal.NewWal(enc, segmentWriter, maxSegmentSize, 0, nil) + + s.Require().ErrorIs(wl.Sync(), expectedError) + s.Len(segmentWriter.SyncCalls(), 1) +} + +func (s *WalSuite) TestWrite() { + enc := &EncoderMock[*EncodedSegmentMock]{ + EncodeFunc: func([]*cppbridge.InnerSeries) (uint32, error) { return 100, nil }, + } + segmentWriter := &SegmentWriterMock[*EncodedSegmentMock]{ + CloseFunc: func() error { return nil }, + } + + maxSegmentSize := uint32(0) + wl := wal.NewWal(enc, segmentWriter, maxSegmentSize, 0, nil) + + limitExhausted, err := wl.Write([]*cppbridge.InnerSeries{}) + s.Require().NoError(err) + s.Len(enc.EncodeCalls(), 1) + s.False(limitExhausted) + + s.Require().NoError(wl.Close()) + s.Len(segmentWriter.CloseCalls(), 1) +} + +func (s *WalSuite) TestWriteLimitExhausted() { + maxSegmentSize := uint32(100) + enc := &EncoderMock[*EncodedSegmentMock]{ + EncodeFunc: func([]*cppbridge.InnerSeries) (uint32, error) { return maxSegmentSize, nil }, + } + segmentWriter := &SegmentWriterMock[*EncodedSegmentMock]{ + CloseFunc: func() error { return nil }, + } + + wl := wal.NewWal(enc, segmentWriter, maxSegmentSize, 0, nil) + + limitExhausted, err := wl.Write([]*cppbridge.InnerSeries{}) + s.Require().NoError(err) + s.Len(enc.EncodeCalls(), 1) + s.True(limitExhausted) + + s.Require().NoError(wl.Close()) + s.Len(segmentWriter.CloseCalls(), 1) +} + +func (s *WalSuite) TestWriteLimitNotExhausted() { + maxSegmentSize := uint32(100) + enc := &EncoderMock[*EncodedSegmentMock]{ + EncodeFunc: func([]*cppbridge.InnerSeries) (uint32, error) { return maxSegmentSize / 2, nil }, + } + segmentWriter := &SegmentWriterMock[*EncodedSegmentMock]{ + CloseFunc: func() error { return nil }, + } + + wl := wal.NewWal(enc, segmentWriter, maxSegmentSize, 0, nil) + + limitExhausted, err := wl.Write([]*cppbridge.InnerSeries{}) + s.Require().NoError(err) + s.Len(enc.EncodeCalls(), 1) + s.False(limitExhausted) + + s.Require().NoError(wl.Close()) + s.Len(segmentWriter.CloseCalls(), 1) +} + +func (s *WalSuite) TestWriteError() { + maxSegmentSize := uint32(100) + expectedError := errors.New("test error") + enc := &EncoderMock[*EncodedSegmentMock]{ + EncodeFunc: func([]*cppbridge.InnerSeries) (uint32, error) { return maxSegmentSize / 2, expectedError }, + } + segmentWriter := &SegmentWriterMock[*EncodedSegmentMock]{ + CloseFunc: func() error { return nil }, + } + + wl := wal.NewWal(enc, segmentWriter, maxSegmentSize, 0, nil) + + limitExhausted, err := wl.Write([]*cppbridge.InnerSeries{}) + s.Require().ErrorIs(err, expectedError) + s.Len(enc.EncodeCalls(), 1) + s.False(limitExhausted) + + s.Require().NoError(wl.Close()) + s.Len(segmentWriter.CloseCalls(), 1) +} + +func (s *WalSuite) TestCorrupted() { + wl := wal.NewCorruptedWal[*EncodedSegmentMock, *SegmentWriterMock[*EncodedSegmentMock]]() + s.Equal(int64(0), wl.CurrentSize()) + + limitExhausted, err := wl.Write([]*cppbridge.InnerSeries{}) + s.Require().ErrorIs(err, wal.ErrWalIsCorrupted) + s.False(limitExhausted) + + err = wl.Commit() + s.Require().ErrorIs(err, wal.ErrWalIsCorrupted) + + err = wl.Flush() + s.Require().NoError(err) + + err = wl.Sync() + s.Require().ErrorIs(err, wal.ErrWalIsCorrupted) + + s.Require().NoError(wl.Close()) +} diff --git a/pp/go/storage/head/shard/wal/writer/buffered.go b/pp/go/storage/head/shard/wal/writer/buffered.go new file mode 100644 index 0000000000..145331a13b --- /dev/null +++ b/pp/go/storage/head/shard/wal/writer/buffered.go @@ -0,0 +1,151 @@ +package writer + +import ( + "bytes" + "fmt" + "io" + "os" + "sync/atomic" +) + +//go:generate -command moq go run github.com/matryer/moq --rm --skip-ensure --pkg writer_test --out +//go:generate moq buffered_moq_test.go . FileInfo SegmentIsWrittenNotifier FileWriter + +// FileInfo alias for [os.FileInfo]. +type FileInfo = os.FileInfo + +// SegmentIsWrittenNotifier notify when new segment write. +type SegmentIsWrittenNotifier interface { + NotifySegmentIsWritten() + NotifySegmentWrite(shardID uint16) +} + +// FileWriter writer implementation [os.File]. +type FileWriter interface { + io.WriteCloser + Sync() error + Stat() (FileInfo, error) +} + +// SegmentWriterFN encode to slice byte and write to [io.Writer]. +type SegmentWriterFN[TSegment any] func(writer io.Writer, segment TSegment) (n int, err error) + +// Buffered writer for segments. +type Buffered[TSegment any] struct { + shardID uint16 + segments []TSegment + buffer *bytes.Buffer + notifier SegmentIsWrittenNotifier + swriter SegmentWriterFN[TSegment] + writer FileWriter + currentSize int64 + writeCompleted bool +} + +// NewBuffered init new [Buffered]. +func NewBuffered[TSegment any]( + shardID uint16, + writer FileWriter, + swriter SegmentWriterFN[TSegment], + notifier SegmentIsWrittenNotifier, +) (*Buffered[TSegment], error) { + info, err := writer.Stat() + if err != nil { + return nil, err + } + + return &Buffered[TSegment]{ + shardID: shardID, + buffer: bytes.NewBuffer(nil), + notifier: notifier, + swriter: swriter, + writer: writer, + currentSize: info.Size(), + writeCompleted: true, + }, nil +} + +// Close closes the writer [WriteSyncCloser]. +func (w *Buffered[TSegment]) Close() error { + return w.writer.Close() +} + +// CurrentSize return current shard wal size. +func (w *Buffered[TSegment]) CurrentSize() int64 { + return atomic.LoadInt64(&w.currentSize) +} + +// Flush buffer and collected segments to [WriteSyncCloser]. +func (w *Buffered[TSegment]) Flush() error { + if !w.writeCompleted { + if err := w.flushBuffer(); err != nil { + return fmt.Errorf("flush and sync: %w", err) + } + } + + for index, segment := range w.segments { + if encoded, err := w.writeToBufferAndFlush(segment); err != nil { + if encoded { + index++ + } + // shift encoded segments to the left + copy(w.segments, w.segments[index:]) + w.segments = w.segments[:len(w.segments)-index] + return fmt.Errorf("flush segment: %w", err) + } + } + + if len(w.segments) != 0 && cap(w.segments) >= len(w.segments)*2 { //revive:disable-line:add-constant // x2 + w.segments = make([]TSegment, 0, len(w.segments)) + } else { + clear(w.segments) + w.segments = w.segments[:0] + } + + return nil +} + +// Sync commits the current contents of the [WriteSyncCloser] and notify [SegmentIsWrittenNotifier]. +func (w *Buffered[TSegment]) Sync() error { + if err := w.writer.Sync(); err != nil { + return fmt.Errorf("writer sync: %w", err) + } + + w.notifier.NotifySegmentIsWritten() + w.writeCompleted = true + return nil +} + +// Write to buffer [Buffered] incoming [Segment]. +func (w *Buffered[TSegment]) Write(segment TSegment) error { + w.segments = append(w.segments, segment) + return nil +} + +// flushBuffer write the contents from buffer to [WriteSyncCloser]. +func (w *Buffered[TSegment]) flushBuffer() error { + n, err := w.buffer.WriteTo(w.writer) + atomic.AddInt64(&w.currentSize, n) + if err != nil { + return fmt.Errorf("buffer write: %w", err) + } + + return nil +} + +// writeToBufferAndFlush write [Segment] as slice byte to buffer and flush to [WriteSyncCloser]. +func (w *Buffered[TSegment]) writeToBufferAndFlush(segment TSegment) (encoded bool, err error) { + if _, err := w.swriter(w.buffer, segment); err != nil { + w.buffer.Reset() + return false, fmt.Errorf("encode segment: %w", err) + } + + w.writeCompleted = false + w.notifier.NotifySegmentWrite(w.shardID) + + if err := w.flushBuffer(); err != nil { + return true, err + } + + return true, nil +} diff --git a/pp/go/storage/head/shard/wal/writer/buffered_moq_test.go b/pp/go/storage/head/shard/wal/writer/buffered_moq_test.go new file mode 100644 index 0000000000..0c89658a4f --- /dev/null +++ b/pp/go/storage/head/shard/wal/writer/buffered_moq_test.go @@ -0,0 +1,523 @@ +// Code generated by moq; DO NOT EDIT. +// github.com/matryer/moq + +package writer_test + +import ( + "github.com/prometheus/prometheus/pp/go/storage/head/shard/wal/writer" + "io/fs" + "sync" + "time" +) + +// FileInfoMock is a mock implementation of writer.FileInfo. +// +// func TestSomethingThatUsesFileInfo(t *testing.T) { +// +// // make and configure a mocked writer.FileInfo +// mockedFileInfo := &FileInfoMock{ +// IsDirFunc: func() bool { +// panic("mock out the IsDir method") +// }, +// ModTimeFunc: func() time.Time { +// panic("mock out the ModTime method") +// }, +// ModeFunc: func() fs.FileMode { +// panic("mock out the Mode method") +// }, +// NameFunc: func() string { +// panic("mock out the Name method") +// }, +// SizeFunc: func() int64 { +// panic("mock out the Size method") +// }, +// SysFunc: func() any { +// panic("mock out the Sys method") +// }, +// } +// +// // use mockedFileInfo in code that requires writer.FileInfo +// // and then make assertions. +// +// } +type FileInfoMock struct { + // IsDirFunc mocks the IsDir method. + IsDirFunc func() bool + + // ModTimeFunc mocks the ModTime method. + ModTimeFunc func() time.Time + + // ModeFunc mocks the Mode method. + ModeFunc func() fs.FileMode + + // NameFunc mocks the Name method. + NameFunc func() string + + // SizeFunc mocks the Size method. + SizeFunc func() int64 + + // SysFunc mocks the Sys method. + SysFunc func() any + + // calls tracks calls to the methods. + calls struct { + // IsDir holds details about calls to the IsDir method. + IsDir []struct { + } + // ModTime holds details about calls to the ModTime method. + ModTime []struct { + } + // Mode holds details about calls to the Mode method. + Mode []struct { + } + // Name holds details about calls to the Name method. + Name []struct { + } + // Size holds details about calls to the Size method. + Size []struct { + } + // Sys holds details about calls to the Sys method. + Sys []struct { + } + } + lockIsDir sync.RWMutex + lockModTime sync.RWMutex + lockMode sync.RWMutex + lockName sync.RWMutex + lockSize sync.RWMutex + lockSys sync.RWMutex +} + +// IsDir calls IsDirFunc. +func (mock *FileInfoMock) IsDir() bool { + if mock.IsDirFunc == nil { + panic("FileInfoMock.IsDirFunc: method is nil but FileInfo.IsDir was just called") + } + callInfo := struct { + }{} + mock.lockIsDir.Lock() + mock.calls.IsDir = append(mock.calls.IsDir, callInfo) + mock.lockIsDir.Unlock() + return mock.IsDirFunc() +} + +// IsDirCalls gets all the calls that were made to IsDir. +// Check the length with: +// +// len(mockedFileInfo.IsDirCalls()) +func (mock *FileInfoMock) IsDirCalls() []struct { +} { + var calls []struct { + } + mock.lockIsDir.RLock() + calls = mock.calls.IsDir + mock.lockIsDir.RUnlock() + return calls +} + +// ModTime calls ModTimeFunc. +func (mock *FileInfoMock) ModTime() time.Time { + if mock.ModTimeFunc == nil { + panic("FileInfoMock.ModTimeFunc: method is nil but FileInfo.ModTime was just called") + } + callInfo := struct { + }{} + mock.lockModTime.Lock() + mock.calls.ModTime = append(mock.calls.ModTime, callInfo) + mock.lockModTime.Unlock() + return mock.ModTimeFunc() +} + +// ModTimeCalls gets all the calls that were made to ModTime. +// Check the length with: +// +// len(mockedFileInfo.ModTimeCalls()) +func (mock *FileInfoMock) ModTimeCalls() []struct { +} { + var calls []struct { + } + mock.lockModTime.RLock() + calls = mock.calls.ModTime + mock.lockModTime.RUnlock() + return calls +} + +// Mode calls ModeFunc. +func (mock *FileInfoMock) Mode() fs.FileMode { + if mock.ModeFunc == nil { + panic("FileInfoMock.ModeFunc: method is nil but FileInfo.Mode was just called") + } + callInfo := struct { + }{} + mock.lockMode.Lock() + mock.calls.Mode = append(mock.calls.Mode, callInfo) + mock.lockMode.Unlock() + return mock.ModeFunc() +} + +// ModeCalls gets all the calls that were made to Mode. +// Check the length with: +// +// len(mockedFileInfo.ModeCalls()) +func (mock *FileInfoMock) ModeCalls() []struct { +} { + var calls []struct { + } + mock.lockMode.RLock() + calls = mock.calls.Mode + mock.lockMode.RUnlock() + return calls +} + +// Name calls NameFunc. +func (mock *FileInfoMock) Name() string { + if mock.NameFunc == nil { + panic("FileInfoMock.NameFunc: method is nil but FileInfo.Name was just called") + } + callInfo := struct { + }{} + mock.lockName.Lock() + mock.calls.Name = append(mock.calls.Name, callInfo) + mock.lockName.Unlock() + return mock.NameFunc() +} + +// NameCalls gets all the calls that were made to Name. +// Check the length with: +// +// len(mockedFileInfo.NameCalls()) +func (mock *FileInfoMock) NameCalls() []struct { +} { + var calls []struct { + } + mock.lockName.RLock() + calls = mock.calls.Name + mock.lockName.RUnlock() + return calls +} + +// Size calls SizeFunc. +func (mock *FileInfoMock) Size() int64 { + if mock.SizeFunc == nil { + panic("FileInfoMock.SizeFunc: method is nil but FileInfo.Size was just called") + } + callInfo := struct { + }{} + mock.lockSize.Lock() + mock.calls.Size = append(mock.calls.Size, callInfo) + mock.lockSize.Unlock() + return mock.SizeFunc() +} + +// SizeCalls gets all the calls that were made to Size. +// Check the length with: +// +// len(mockedFileInfo.SizeCalls()) +func (mock *FileInfoMock) SizeCalls() []struct { +} { + var calls []struct { + } + mock.lockSize.RLock() + calls = mock.calls.Size + mock.lockSize.RUnlock() + return calls +} + +// Sys calls SysFunc. +func (mock *FileInfoMock) Sys() any { + if mock.SysFunc == nil { + panic("FileInfoMock.SysFunc: method is nil but FileInfo.Sys was just called") + } + callInfo := struct { + }{} + mock.lockSys.Lock() + mock.calls.Sys = append(mock.calls.Sys, callInfo) + mock.lockSys.Unlock() + return mock.SysFunc() +} + +// SysCalls gets all the calls that were made to Sys. +// Check the length with: +// +// len(mockedFileInfo.SysCalls()) +func (mock *FileInfoMock) SysCalls() []struct { +} { + var calls []struct { + } + mock.lockSys.RLock() + calls = mock.calls.Sys + mock.lockSys.RUnlock() + return calls +} + +// SegmentIsWrittenNotifierMock is a mock implementation of writer.SegmentIsWrittenNotifier. +// +// func TestSomethingThatUsesSegmentIsWrittenNotifier(t *testing.T) { +// +// // make and configure a mocked writer.SegmentIsWrittenNotifier +// mockedSegmentIsWrittenNotifier := &SegmentIsWrittenNotifierMock{ +// NotifySegmentIsWrittenFunc: func() { +// panic("mock out the NotifySegmentIsWritten method") +// }, +// NotifySegmentWriteFunc: func(shardID uint16) { +// panic("mock out the NotifySegmentWrite method") +// }, +// } +// +// // use mockedSegmentIsWrittenNotifier in code that requires writer.SegmentIsWrittenNotifier +// // and then make assertions. +// +// } +type SegmentIsWrittenNotifierMock struct { + // NotifySegmentIsWrittenFunc mocks the NotifySegmentIsWritten method. + NotifySegmentIsWrittenFunc func() + + // NotifySegmentWriteFunc mocks the NotifySegmentWrite method. + NotifySegmentWriteFunc func(shardID uint16) + + // calls tracks calls to the methods. + calls struct { + // NotifySegmentIsWritten holds details about calls to the NotifySegmentIsWritten method. + NotifySegmentIsWritten []struct { + } + // NotifySegmentWrite holds details about calls to the NotifySegmentWrite method. + NotifySegmentWrite []struct { + // ShardID is the shardID argument value. + ShardID uint16 + } + } + lockNotifySegmentIsWritten sync.RWMutex + lockNotifySegmentWrite sync.RWMutex +} + +// NotifySegmentIsWritten calls NotifySegmentIsWrittenFunc. +func (mock *SegmentIsWrittenNotifierMock) NotifySegmentIsWritten() { + if mock.NotifySegmentIsWrittenFunc == nil { + panic("SegmentIsWrittenNotifierMock.NotifySegmentIsWrittenFunc: method is nil but SegmentIsWrittenNotifier.NotifySegmentIsWritten was just called") + } + callInfo := struct { + }{} + mock.lockNotifySegmentIsWritten.Lock() + mock.calls.NotifySegmentIsWritten = append(mock.calls.NotifySegmentIsWritten, callInfo) + mock.lockNotifySegmentIsWritten.Unlock() + mock.NotifySegmentIsWrittenFunc() +} + +// NotifySegmentIsWrittenCalls gets all the calls that were made to NotifySegmentIsWritten. +// Check the length with: +// +// len(mockedSegmentIsWrittenNotifier.NotifySegmentIsWrittenCalls()) +func (mock *SegmentIsWrittenNotifierMock) NotifySegmentIsWrittenCalls() []struct { +} { + var calls []struct { + } + mock.lockNotifySegmentIsWritten.RLock() + calls = mock.calls.NotifySegmentIsWritten + mock.lockNotifySegmentIsWritten.RUnlock() + return calls +} + +// NotifySegmentWrite calls NotifySegmentWriteFunc. +func (mock *SegmentIsWrittenNotifierMock) NotifySegmentWrite(shardID uint16) { + if mock.NotifySegmentWriteFunc == nil { + panic("SegmentIsWrittenNotifierMock.NotifySegmentWriteFunc: method is nil but SegmentIsWrittenNotifier.NotifySegmentWrite was just called") + } + callInfo := struct { + ShardID uint16 + }{ + ShardID: shardID, + } + mock.lockNotifySegmentWrite.Lock() + mock.calls.NotifySegmentWrite = append(mock.calls.NotifySegmentWrite, callInfo) + mock.lockNotifySegmentWrite.Unlock() + mock.NotifySegmentWriteFunc(shardID) +} + +// NotifySegmentWriteCalls gets all the calls that were made to NotifySegmentWrite. +// Check the length with: +// +// len(mockedSegmentIsWrittenNotifier.NotifySegmentWriteCalls()) +func (mock *SegmentIsWrittenNotifierMock) NotifySegmentWriteCalls() []struct { + ShardID uint16 +} { + var calls []struct { + ShardID uint16 + } + mock.lockNotifySegmentWrite.RLock() + calls = mock.calls.NotifySegmentWrite + mock.lockNotifySegmentWrite.RUnlock() + return calls +} + +// FileWriterMock is a mock implementation of writer.FileWriter. +// +// func TestSomethingThatUsesFileWriter(t *testing.T) { +// +// // make and configure a mocked writer.FileWriter +// mockedFileWriter := &FileWriterMock{ +// CloseFunc: func() error { +// panic("mock out the Close method") +// }, +// StatFunc: func() (writer.FileInfo, error) { +// panic("mock out the Stat method") +// }, +// SyncFunc: func() error { +// panic("mock out the Sync method") +// }, +// WriteFunc: func(p []byte) (int, error) { +// panic("mock out the Write method") +// }, +// } +// +// // use mockedFileWriter in code that requires writer.FileWriter +// // and then make assertions. +// +// } +type FileWriterMock struct { + // CloseFunc mocks the Close method. + CloseFunc func() error + + // StatFunc mocks the Stat method. + StatFunc func() (writer.FileInfo, error) + + // SyncFunc mocks the Sync method. + SyncFunc func() error + + // WriteFunc mocks the Write method. + WriteFunc func(p []byte) (int, error) + + // calls tracks calls to the methods. + calls struct { + // Close holds details about calls to the Close method. + Close []struct { + } + // Stat holds details about calls to the Stat method. + Stat []struct { + } + // Sync holds details about calls to the Sync method. + Sync []struct { + } + // Write holds details about calls to the Write method. + Write []struct { + // P is the p argument value. + P []byte + } + } + lockClose sync.RWMutex + lockStat sync.RWMutex + lockSync sync.RWMutex + lockWrite sync.RWMutex +} + +// Close calls CloseFunc. +func (mock *FileWriterMock) Close() error { + if mock.CloseFunc == nil { + panic("FileWriterMock.CloseFunc: method is nil but FileWriter.Close was just called") + } + callInfo := struct { + }{} + mock.lockClose.Lock() + mock.calls.Close = append(mock.calls.Close, callInfo) + mock.lockClose.Unlock() + return mock.CloseFunc() +} + +// CloseCalls gets all the calls that were made to Close. +// Check the length with: +// +// len(mockedFileWriter.CloseCalls()) +func (mock *FileWriterMock) CloseCalls() []struct { +} { + var calls []struct { + } + mock.lockClose.RLock() + calls = mock.calls.Close + mock.lockClose.RUnlock() + return calls +} + +// Stat calls StatFunc. +func (mock *FileWriterMock) Stat() (writer.FileInfo, error) { + if mock.StatFunc == nil { + panic("FileWriterMock.StatFunc: method is nil but FileWriter.Stat was just called") + } + callInfo := struct { + }{} + mock.lockStat.Lock() + mock.calls.Stat = append(mock.calls.Stat, callInfo) + mock.lockStat.Unlock() + return mock.StatFunc() +} + +// StatCalls gets all the calls that were made to Stat. +// Check the length with: +// +// len(mockedFileWriter.StatCalls()) +func (mock *FileWriterMock) StatCalls() []struct { +} { + var calls []struct { + } + mock.lockStat.RLock() + calls = mock.calls.Stat + mock.lockStat.RUnlock() + return calls +} + +// Sync calls SyncFunc. +func (mock *FileWriterMock) Sync() error { + if mock.SyncFunc == nil { + panic("FileWriterMock.SyncFunc: method is nil but FileWriter.Sync was just called") + } + callInfo := struct { + }{} + mock.lockSync.Lock() + mock.calls.Sync = append(mock.calls.Sync, callInfo) + mock.lockSync.Unlock() + return mock.SyncFunc() +} + +// SyncCalls gets all the calls that were made to Sync. +// Check the length with: +// +// len(mockedFileWriter.SyncCalls()) +func (mock *FileWriterMock) SyncCalls() []struct { +} { + var calls []struct { + } + mock.lockSync.RLock() + calls = mock.calls.Sync + mock.lockSync.RUnlock() + return calls +} + +// Write calls WriteFunc. +func (mock *FileWriterMock) Write(p []byte) (int, error) { + if mock.WriteFunc == nil { + panic("FileWriterMock.WriteFunc: method is nil but FileWriter.Write was just called") + } + callInfo := struct { + P []byte + }{ + P: p, + } + mock.lockWrite.Lock() + mock.calls.Write = append(mock.calls.Write, callInfo) + mock.lockWrite.Unlock() + return mock.WriteFunc(p) +} + +// WriteCalls gets all the calls that were made to Write. +// Check the length with: +// +// len(mockedFileWriter.WriteCalls()) +func (mock *FileWriterMock) WriteCalls() []struct { + P []byte +} { + var calls []struct { + P []byte + } + mock.lockWrite.RLock() + calls = mock.calls.Write + mock.lockWrite.RUnlock() + return calls +} diff --git a/pp/go/storage/head/shard/wal/writer/buffered_test.go b/pp/go/storage/head/shard/wal/writer/buffered_test.go new file mode 100644 index 0000000000..d184496e9e --- /dev/null +++ b/pp/go/storage/head/shard/wal/writer/buffered_test.go @@ -0,0 +1,348 @@ +package writer_test + +import ( + "bytes" + "errors" + "io" + "os" + "testing" + + "github.com/go-faker/faker/v4" + "github.com/stretchr/testify/suite" + + "github.com/prometheus/prometheus/pp/go/storage/head/shard/wal/writer" +) + +type BufferedSuite struct { + suite.Suite +} + +func TestBufferedSuite(t *testing.T) { + suite.Run(t, new(BufferedSuite)) +} + +func (s *BufferedSuite) TestWriteFlushSync() { + actual := &bytes.Buffer{} + sfile := s.openfile(actual) + + swn := &SegmentIsWrittenNotifierMock{ + NotifySegmentIsWrittenFunc: func() {}, + NotifySegmentWriteFunc: func(uint16) {}, + } + + shardID := uint16(0) + wrBuf, err := writer.NewBuffered(shardID, sfile, writer.WriteSegment[*EncodedSegmentMock], swn) + s.Require().NoError(err) + s.Equal(int64(0), wrBuf.CurrentSize()) + + segment, expectedSegment := s.genSegment() + + err = wrBuf.Write(segment) + s.Require().NoError(err) + s.Empty(sfile.WriteCalls()) + + err = wrBuf.Flush() + s.Require().NoError(err) + s.Equal(int64(len(expectedSegment)), wrBuf.CurrentSize()) + s.Equal(expectedSegment, actual.Bytes()) + + err = wrBuf.Sync() + s.Require().NoError(err) + s.Len(sfile.SyncCalls(), 1) + s.Len(swn.NotifySegmentIsWrittenCalls(), 1) + + err = wrBuf.Close() + s.Require().NoError(err) + s.Len(sfile.CloseCalls(), 1) +} + +func (s *BufferedSuite) TestDoubleWriteAndFlush() { + actual := &bytes.Buffer{} + sfile := s.openfile(actual) + + numberOfSegments := 0 + swn := &SegmentIsWrittenNotifierMock{ + NotifySegmentIsWrittenFunc: func() {}, + NotifySegmentWriteFunc: func(uint16) { numberOfSegments++ }, + } + + shardID := uint16(0) + wrBuf, err := writer.NewBuffered(shardID, sfile, writer.WriteSegment[*EncodedSegmentMock], swn) + s.Require().NoError(err) + s.Equal(int64(0), wrBuf.CurrentSize()) + + segment, expectedSegment := s.genSegment() + expectedLen := len(expectedSegment) + + err = wrBuf.Write(segment) + s.Require().NoError(err) + s.Empty(sfile.WriteCalls()) + + err = wrBuf.Flush() + s.Require().NoError(err) + s.Equal(int64(expectedLen), wrBuf.CurrentSize()) + s.Equal(expectedSegment, actual.Bytes()) + + actual.Reset() + segment, expectedSegment = s.genSegment() + + err = wrBuf.Write(segment) + s.Require().NoError(err) + + err = wrBuf.Flush() + s.Require().NoError(err) + s.Equal(int64(len(expectedSegment)+expectedLen), wrBuf.CurrentSize()) + s.Equal(expectedSegment, actual.Bytes()) + + err = wrBuf.Sync() + s.Require().NoError(err) + s.Len(sfile.SyncCalls(), 1) + s.Len(swn.NotifySegmentIsWrittenCalls(), 1) + s.Len(swn.NotifySegmentWriteCalls(), 2) + s.Equal(2, numberOfSegments) + + err = wrBuf.Close() + s.Require().NoError(err) + s.Len(sfile.CloseCalls(), 1) +} + +func (s *BufferedSuite) TestBuffered() { + actual := &bytes.Buffer{} + sfile := s.openfile(actual) + + swn := &SegmentIsWrittenNotifierMock{ + NotifySegmentIsWrittenFunc: func() {}, + NotifySegmentWriteFunc: func(uint16) {}, + } + + shardID := uint16(0) + wrBuf, err := writer.NewBuffered(shardID, sfile, writer.WriteSegment[*EncodedSegmentMock], swn) + s.Require().NoError(err) + s.Equal(int64(0), wrBuf.CurrentSize()) + + expectedSegments := []byte{} + expectedSize := 0 + for range 10 { + segment, expectedSegment := s.genSegment() + err = wrBuf.Write(segment) + s.Require().NoError(err) + s.Empty(sfile.WriteCalls()) + expectedSegments = append(expectedSegments, expectedSegment...) + expectedSize += len(expectedSegment) + } + + err = wrBuf.Flush() + s.Require().NoError(err) + s.Equal(int64(expectedSize), wrBuf.CurrentSize()) + s.Equal(expectedSegments, actual.Bytes()) + s.Empty(swn.NotifySegmentIsWrittenCalls()) + s.Len(swn.NotifySegmentWriteCalls(), 10) + + err = wrBuf.Sync() + s.Require().NoError(err) + s.Len(sfile.SyncCalls(), 1) + s.Len(swn.NotifySegmentIsWrittenCalls(), 1) + + err = wrBuf.Close() + s.Require().NoError(err) + s.Len(sfile.CloseCalls(), 1) +} + +func (s *BufferedSuite) TestStatError() { + actual := &bytes.Buffer{} + sfile := s.openfile(actual) + sfile.StatFunc = func() (os.FileInfo, error) { return nil, errors.New("some error") } + + swn := &SegmentIsWrittenNotifierMock{ + NotifySegmentIsWrittenFunc: func() {}, + NotifySegmentWriteFunc: func(uint16) {}, + } + + shardID := uint16(0) + wrBuf, err := writer.NewBuffered(shardID, sfile, writer.WriteSegment[*EncodedSegmentMock], swn) + s.Require().Error(err) + s.Require().Nil(wrBuf) +} + +func (s *BufferedSuite) TestSyncError() { + actual := &bytes.Buffer{} + sfile := s.openfile(actual) + sfile.SyncFunc = func() error { return errors.New("some error") } + + swn := &SegmentIsWrittenNotifierMock{ + NotifySegmentIsWrittenFunc: func() {}, + NotifySegmentWriteFunc: func(uint16) {}, + } + + shardID := uint16(0) + wrBuf, err := writer.NewBuffered(shardID, sfile, writer.WriteSegment[*EncodedSegmentMock], swn) + s.Require().NoError(err) + s.Equal(int64(0), wrBuf.CurrentSize()) + + segment, expectedSegment := s.genSegment() + + err = wrBuf.Write(segment) + s.Require().NoError(err) + s.Empty(sfile.WriteCalls()) + + err = wrBuf.Flush() + s.Require().NoError(err) + s.Equal(int64(len(expectedSegment)), wrBuf.CurrentSize()) + s.Equal(expectedSegment, actual.Bytes()) + + err = wrBuf.Sync() + s.Require().Error(err) + s.Len(sfile.SyncCalls(), 1) + s.Empty(swn.NotifySegmentIsWrittenCalls()) +} + +func (s *BufferedSuite) TestWriteToBufferWithError() { + actual := &bytes.Buffer{} + sfile := s.openfile(actual) + + swn := &SegmentIsWrittenNotifierMock{ + NotifySegmentIsWrittenFunc: func() {}, + NotifySegmentWriteFunc: func(uint16) {}, + } + + scount := 0 + writeSegment := func(w io.Writer, segment *EncodedSegmentMock) (n int, err error) { + if scount == 5 { + scount++ + return 0, errors.New("some error") + } + + scount++ + return writer.WriteSegment(w, segment) + } + + shardID := uint16(0) + wrBuf, err := writer.NewBuffered(shardID, sfile, writeSegment, swn) + s.Require().NoError(err) + s.Equal(int64(0), wrBuf.CurrentSize()) + + expectedSegments := []byte{} + expectedSize := 0 + for range 10 { + segment, expectedSegment := s.genSegment() + err = wrBuf.Write(segment) + s.Require().NoError(err) + s.Empty(sfile.WriteCalls()) + expectedSegments = append(expectedSegments, expectedSegment...) + expectedSize += len(expectedSegment) + } + + err = wrBuf.Flush() + s.Require().Error(err) + + err = wrBuf.Flush() + s.Require().NoError(err) + s.Equal(int64(expectedSize), wrBuf.CurrentSize()) + s.Equal(expectedSegments, actual.Bytes()) + s.Empty(swn.NotifySegmentIsWrittenCalls()) + + err = wrBuf.Sync() + s.Require().NoError(err) + s.Len(sfile.SyncCalls(), 1) + s.Len(swn.NotifySegmentIsWrittenCalls(), 1) + + err = wrBuf.Close() + s.Require().NoError(err) + s.Len(sfile.CloseCalls(), 1) +} + +func (s *BufferedSuite) TestFlushWithError() { + actual := &bytes.Buffer{} + sfile := s.openfile(actual) + + swn := &SegmentIsWrittenNotifierMock{ + NotifySegmentIsWrittenFunc: func() {}, + NotifySegmentWriteFunc: func(uint16) {}, + } + + scount := 0 + sfile.WriteFunc = func(p []byte) (int, error) { + if scount == 4 || scount == 5 { + scount++ + return 0, errors.New("some error") + } + + scount++ + + return actual.Write(p) + } + + shardID := uint16(0) + wrBuf, err := writer.NewBuffered(shardID, sfile, writer.WriteSegment[*EncodedSegmentMock], swn) + s.Require().NoError(err) + s.Equal(int64(0), wrBuf.CurrentSize()) + + expectedSegments := []byte{} + expectedSize := 0 + for range 10 { + segment, expectedSegment := s.genSegment() + err = wrBuf.Write(segment) + s.Require().NoError(err) + s.Empty(sfile.WriteCalls()) + expectedSegments = append(expectedSegments, expectedSegment...) + expectedSize += len(expectedSegment) + } + + err = wrBuf.Flush() + s.Require().Error(err) + + err = wrBuf.Flush() + s.Require().Error(err) + + err = wrBuf.Flush() + s.Require().NoError(err) + s.Equal(int64(expectedSize), wrBuf.CurrentSize()) + s.Equal(expectedSegments, actual.Bytes()) + s.Empty(swn.NotifySegmentIsWrittenCalls()) + + err = wrBuf.Sync() + s.Require().NoError(err) + s.Len(sfile.SyncCalls(), 1) + s.Len(swn.NotifySegmentIsWrittenCalls(), 1) + + err = wrBuf.Close() + s.Require().NoError(err) + s.Len(sfile.CloseCalls(), 1) +} + +func (*BufferedSuite) openfile(buf *bytes.Buffer) *FileWriterMock { + return &FileWriterMock{ + CloseFunc: func() error { return nil }, + StatFunc: func() (os.FileInfo, error) { return &FileInfoMock{SizeFunc: func() int64 { return 0 }}, nil }, + SyncFunc: func() error { return nil }, + WriteFunc: buf.Write, + } +} + +func (s *BufferedSuite) genSegment() (segment *EncodedSegmentMock, expected []byte) { + segmentCrc32 := uint32(0) + segmentSamples := uint32(42) + data := []byte(faker.Paragraph()) + + segment = &EncodedSegmentMock{ + CRC32Func: func() uint32 { + return segmentCrc32 + }, + SamplesFunc: func() uint32 { + return segmentSamples + }, + SizeFunc: func() int64 { + return int64(len(data)) + }, + WriteToFunc: func(w io.Writer) (int64, error) { + n, errWr := w.Write(data) + return int64(n), errWr + }, + } + + buf := &bytes.Buffer{} + _, err := writer.WriteSegment(buf, segment) + s.Require().NoError(err) + + return segment, buf.Bytes() +} diff --git a/pp/go/storage/head/shard/wal/writer/header.go b/pp/go/storage/head/shard/wal/writer/header.go new file mode 100644 index 0000000000..778e0ea5b5 --- /dev/null +++ b/pp/go/storage/head/shard/wal/writer/header.go @@ -0,0 +1,30 @@ +package writer + +import ( + "encoding/binary" + "fmt" + "io" +) + +// WriteHeader encode Header to slice byte and write to [io.Writer]. +func WriteHeader(writer io.Writer, fileFormatVersion, encoderVersion uint8) (n int, err error) { + var buf [binary.MaxVarintLen32]byte + var size int + var bytesWritten int + + size = binary.PutUvarint(buf[:], uint64(fileFormatVersion)) + bytesWritten, err = writer.Write(buf[:size]) + if err != nil { + return n, fmt.Errorf("failed to write file format version: %w", err) + } + n += bytesWritten + + size = binary.PutUvarint(buf[:], uint64(encoderVersion)) + bytesWritten, err = writer.Write(buf[:size]) + if err != nil { + return n, fmt.Errorf("failed to write encoder version: %w", err) + } + n += bytesWritten + + return n, nil +} diff --git a/pp/go/storage/head/shard/wal/writer/header_test.go b/pp/go/storage/head/shard/wal/writer/header_test.go new file mode 100644 index 0000000000..e829c3af72 --- /dev/null +++ b/pp/go/storage/head/shard/wal/writer/header_test.go @@ -0,0 +1,23 @@ +package writer_test + +import ( + "bytes" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/prometheus/prometheus/pp/go/storage/head/shard/wal/writer" +) + +func TestWriteHeader(t *testing.T) { + buf := &bytes.Buffer{} + fileFormatVersion := uint8(21) + encoderVersion := uint8(42) + expected := []byte{fileFormatVersion, encoderVersion} + + n, err := writer.WriteHeader(buf, fileFormatVersion, encoderVersion) + require.NoError(t, err) + + require.Equal(t, len(expected), n) + require.Equal(t, expected, buf.Bytes()) +} diff --git a/pp/go/storage/head/shard/wal/writer/notifier.go b/pp/go/storage/head/shard/wal/writer/notifier.go new file mode 100644 index 0000000000..d661154bdd --- /dev/null +++ b/pp/go/storage/head/shard/wal/writer/notifier.go @@ -0,0 +1,45 @@ +package writer + +import ( + "slices" + "sync" +) + +// SegmentWriteNotifier notifies that the segment has been written. +type SegmentWriteNotifier struct { + locker sync.Mutex + shards []uint32 + setLastAppendedSegmentID func(segmentID uint32) +} + +// NewSegmentWriteNotifier init new [SegmentWriteNotifier]. +func NewSegmentWriteNotifier( + numberOfShards uint16, + setLastAppendedSegmentID func(segmentID uint32), +) *SegmentWriteNotifier { + return &SegmentWriteNotifier{ + shards: make([]uint32, numberOfShards), + setLastAppendedSegmentID: setLastAppendedSegmentID, + } +} + +// NotifySegmentIsWritten notify that the segment has been written for shard. +func (swn *SegmentWriteNotifier) NotifySegmentIsWritten() { + swn.locker.Lock() + defer swn.locker.Unlock() + minNumberOfSegments := slices.Min(swn.shards) + if minNumberOfSegments > 0 { + swn.setLastAppendedSegmentID(minNumberOfSegments - 1) + } +} + +func (swn *SegmentWriteNotifier) NotifySegmentWrite(shardID uint16) { + swn.locker.Lock() + defer swn.locker.Unlock() + swn.shards[shardID]++ +} + +// Set for shard number of segments. +func (swn *SegmentWriteNotifier) Set(shardID uint16, numberOfSegments uint32) { + swn.shards[shardID] = numberOfSegments +} diff --git a/pp/go/storage/head/shard/wal/writer/notifier_test.go b/pp/go/storage/head/shard/wal/writer/notifier_test.go new file mode 100644 index 0000000000..eda6d4f64a --- /dev/null +++ b/pp/go/storage/head/shard/wal/writer/notifier_test.go @@ -0,0 +1,69 @@ +package writer_test + +import ( + "math" + "testing" + + "github.com/prometheus/prometheus/pp/go/storage/head/shard/wal/writer" + "github.com/stretchr/testify/suite" +) + +type SegmentWriteNotifierSuite struct { + suite.Suite +} + +func TestSegmentWriteNotifierSuite(t *testing.T) { + suite.Run(t, new(SegmentWriteNotifierSuite)) +} + +func (s *SegmentWriteNotifierSuite) TestHappyPath() { + actualSegmentID := uint32(math.MaxUint32) + + numberOfShards := uint16(2) + swn := writer.NewSegmentWriteNotifier(numberOfShards, func(segmentID uint32) { actualSegmentID = segmentID }) + + for id := range numberOfShards { + swn.NotifySegmentWrite(id) + } + swn.NotifySegmentIsWritten() + + s.Equal(uint32(0), actualSegmentID) +} + +func (s *SegmentWriteNotifierSuite) TestNotifyOnlyOneShard() { + actualSegmentID := uint32(math.MaxUint32) + + numberOfShards := uint16(2) + swn := writer.NewSegmentWriteNotifier(numberOfShards, func(segmentID uint32) { actualSegmentID = segmentID }) + + swn.NotifySegmentWrite(0) + swn.NotifySegmentIsWritten() + + s.Equal(uint32(math.MaxUint32), actualSegmentID) +} + +func (s *SegmentWriteNotifierSuite) TestSetAndNotifyOnlyOneShard() { + actualSegmentID := uint32(math.MaxUint32) + + numberOfShards := uint16(2) + swn := writer.NewSegmentWriteNotifier(numberOfShards, func(segmentID uint32) { actualSegmentID = segmentID }) + swn.Set(0, 42) + + swn.NotifySegmentWrite(0) + swn.NotifySegmentIsWritten() + + s.Equal(uint32(math.MaxUint32), actualSegmentID) +} + +func (s *SegmentWriteNotifierSuite) TestSetAndNotifyOnlyOneShard_2() { + actualSegmentID := uint32(math.MaxUint32) + + numberOfShards := uint16(2) + swn := writer.NewSegmentWriteNotifier(numberOfShards, func(segmentID uint32) { actualSegmentID = segmentID }) + swn.Set(1, 42) + + swn.NotifySegmentWrite(0) + swn.NotifySegmentIsWritten() + + s.Equal(uint32(0), actualSegmentID) +} diff --git a/pp/go/storage/head/shard/wal/writer/segment.go b/pp/go/storage/head/shard/wal/writer/segment.go new file mode 100644 index 0000000000..35ca87bf61 --- /dev/null +++ b/pp/go/storage/head/shard/wal/writer/segment.go @@ -0,0 +1,56 @@ +package writer + +import ( + "encoding/binary" + "fmt" + "io" +) + +//go:generate -command moq go run github.com/matryer/moq --rm --skip-ensure --pkg writer_test --out +//go:generate moq segment_moq_test.go . EncodedSegment + +// EncodedSegment the minimum required Segment implementation for a [WriteSegment]. +type EncodedSegment interface { + CRC32() uint32 + Samples() uint32 + Size() int64 + io.WriterTo +} + +// WriteSegment encode [EncodedSegment] to slice byte and write to [io.Writer]. +func WriteSegment[TSegment EncodedSegment](writer io.Writer, segment TSegment) (n int, err error) { + var buf [binary.MaxVarintLen32]byte + var size int + var bytesWritten int + + segmentSize := uint64(segment.Size()) // #nosec G115 // no overflow + size = binary.PutUvarint(buf[:], segmentSize) + bytesWritten, err = writer.Write(buf[:size]) + if err != nil { + return n, fmt.Errorf("failed to write segment size: %w", err) + } + n += bytesWritten + + size = binary.PutUvarint(buf[:], uint64(segment.CRC32())) + bytesWritten, err = writer.Write(buf[:size]) + if err != nil { + return n, fmt.Errorf("failed to write segment crc32 hash: %w", err) + } + n += bytesWritten + + size = binary.PutUvarint(buf[:], uint64(segment.Samples())) + bytesWritten, err = writer.Write(buf[:size]) + if err != nil { + return n, fmt.Errorf("failed to write segment sample count: %w", err) + } + n += bytesWritten + + var bytesWritten64 int64 + bytesWritten64, err = segment.WriteTo(writer) + if err != nil { + return n, fmt.Errorf("failed to write segment data: %w", err) + } + n += int(bytesWritten64) + + return n, nil +} diff --git a/pp/go/storage/head/shard/wal/writer/segment_moq_test.go b/pp/go/storage/head/shard/wal/writer/segment_moq_test.go new file mode 100644 index 0000000000..33b8bd2971 --- /dev/null +++ b/pp/go/storage/head/shard/wal/writer/segment_moq_test.go @@ -0,0 +1,182 @@ +// Code generated by moq; DO NOT EDIT. +// github.com/matryer/moq + +package writer_test + +import ( + "io" + "sync" +) + +// EncodedSegmentMock is a mock implementation of writer.EncodedSegment. +// +// func TestSomethingThatUsesEncodedSegment(t *testing.T) { +// +// // make and configure a mocked writer.EncodedSegment +// mockedEncodedSegment := &EncodedSegmentMock{ +// CRC32Func: func() uint32 { +// panic("mock out the CRC32 method") +// }, +// SamplesFunc: func() uint32 { +// panic("mock out the Samples method") +// }, +// SizeFunc: func() int64 { +// panic("mock out the Size method") +// }, +// WriteToFunc: func(w io.Writer) (int64, error) { +// panic("mock out the WriteTo method") +// }, +// } +// +// // use mockedEncodedSegment in code that requires writer.EncodedSegment +// // and then make assertions. +// +// } +type EncodedSegmentMock struct { + // CRC32Func mocks the CRC32 method. + CRC32Func func() uint32 + + // SamplesFunc mocks the Samples method. + SamplesFunc func() uint32 + + // SizeFunc mocks the Size method. + SizeFunc func() int64 + + // WriteToFunc mocks the WriteTo method. + WriteToFunc func(w io.Writer) (int64, error) + + // calls tracks calls to the methods. + calls struct { + // CRC32 holds details about calls to the CRC32 method. + CRC32 []struct { + } + // Samples holds details about calls to the Samples method. + Samples []struct { + } + // Size holds details about calls to the Size method. + Size []struct { + } + // WriteTo holds details about calls to the WriteTo method. + WriteTo []struct { + // W is the w argument value. + W io.Writer + } + } + lockCRC32 sync.RWMutex + lockSamples sync.RWMutex + lockSize sync.RWMutex + lockWriteTo sync.RWMutex +} + +// CRC32 calls CRC32Func. +func (mock *EncodedSegmentMock) CRC32() uint32 { + if mock.CRC32Func == nil { + panic("EncodedSegmentMock.CRC32Func: method is nil but EncodedSegment.CRC32 was just called") + } + callInfo := struct { + }{} + mock.lockCRC32.Lock() + mock.calls.CRC32 = append(mock.calls.CRC32, callInfo) + mock.lockCRC32.Unlock() + return mock.CRC32Func() +} + +// CRC32Calls gets all the calls that were made to CRC32. +// Check the length with: +// +// len(mockedEncodedSegment.CRC32Calls()) +func (mock *EncodedSegmentMock) CRC32Calls() []struct { +} { + var calls []struct { + } + mock.lockCRC32.RLock() + calls = mock.calls.CRC32 + mock.lockCRC32.RUnlock() + return calls +} + +// Samples calls SamplesFunc. +func (mock *EncodedSegmentMock) Samples() uint32 { + if mock.SamplesFunc == nil { + panic("EncodedSegmentMock.SamplesFunc: method is nil but EncodedSegment.Samples was just called") + } + callInfo := struct { + }{} + mock.lockSamples.Lock() + mock.calls.Samples = append(mock.calls.Samples, callInfo) + mock.lockSamples.Unlock() + return mock.SamplesFunc() +} + +// SamplesCalls gets all the calls that were made to Samples. +// Check the length with: +// +// len(mockedEncodedSegment.SamplesCalls()) +func (mock *EncodedSegmentMock) SamplesCalls() []struct { +} { + var calls []struct { + } + mock.lockSamples.RLock() + calls = mock.calls.Samples + mock.lockSamples.RUnlock() + return calls +} + +// Size calls SizeFunc. +func (mock *EncodedSegmentMock) Size() int64 { + if mock.SizeFunc == nil { + panic("EncodedSegmentMock.SizeFunc: method is nil but EncodedSegment.Size was just called") + } + callInfo := struct { + }{} + mock.lockSize.Lock() + mock.calls.Size = append(mock.calls.Size, callInfo) + mock.lockSize.Unlock() + return mock.SizeFunc() +} + +// SizeCalls gets all the calls that were made to Size. +// Check the length with: +// +// len(mockedEncodedSegment.SizeCalls()) +func (mock *EncodedSegmentMock) SizeCalls() []struct { +} { + var calls []struct { + } + mock.lockSize.RLock() + calls = mock.calls.Size + mock.lockSize.RUnlock() + return calls +} + +// WriteTo calls WriteToFunc. +func (mock *EncodedSegmentMock) WriteTo(w io.Writer) (int64, error) { + if mock.WriteToFunc == nil { + panic("EncodedSegmentMock.WriteToFunc: method is nil but EncodedSegment.WriteTo was just called") + } + callInfo := struct { + W io.Writer + }{ + W: w, + } + mock.lockWriteTo.Lock() + mock.calls.WriteTo = append(mock.calls.WriteTo, callInfo) + mock.lockWriteTo.Unlock() + return mock.WriteToFunc(w) +} + +// WriteToCalls gets all the calls that were made to WriteTo. +// Check the length with: +// +// len(mockedEncodedSegment.WriteToCalls()) +func (mock *EncodedSegmentMock) WriteToCalls() []struct { + W io.Writer +} { + var calls []struct { + W io.Writer + } + mock.lockWriteTo.RLock() + calls = mock.calls.WriteTo + mock.lockWriteTo.RUnlock() + return calls +} diff --git a/pp/go/storage/head/shard/wal/writer/segment_test.go b/pp/go/storage/head/shard/wal/writer/segment_test.go new file mode 100644 index 0000000000..54c889b7b4 --- /dev/null +++ b/pp/go/storage/head/shard/wal/writer/segment_test.go @@ -0,0 +1,46 @@ +package writer_test + +import ( + "bytes" + "encoding/binary" + "io" + "testing" + + "github.com/go-faker/faker/v4" + "github.com/stretchr/testify/require" + + "github.com/prometheus/prometheus/pp/go/storage/head/shard/wal/writer" +) + +func TestWriteSegment(t *testing.T) { + data := []byte(faker.Paragraph()) + segmentCrc32 := uint32(0) + segmentSamples := uint32(42) + + segment := &EncodedSegmentMock{ + CRC32Func: func() uint32 { + return segmentCrc32 + }, + SamplesFunc: func() uint32 { + return segmentSamples + }, + SizeFunc: func() int64 { + return int64(len(data)) + }, + WriteToFunc: func(w io.Writer) (int64, error) { + n, err := w.Write(data) + return int64(n), err + }, + } + + buf := &bytes.Buffer{} + expected := []byte{} + expected = append(expected, binary.AppendUvarint(nil, uint64(len(data)))...) + expected = append(expected, byte(segmentCrc32), byte(segmentSamples)) + expected = append(expected, data...) + + _, err := writer.WriteSegment(buf, segment) + require.NoError(t, err) + + require.Equal(t, expected, buf.Bytes()) +} diff --git a/pp/go/storage/head/task/task.go b/pp/go/storage/head/task/task.go new file mode 100644 index 0000000000..b69c44a8b1 --- /dev/null +++ b/pp/go/storage/head/task/task.go @@ -0,0 +1,98 @@ +package task + +import ( + "errors" + "sync" + "sync/atomic" + "time" + + "github.com/prometheus/client_golang/prometheus" +) + +// +// Shard +// + +// Shard the minimum required head Shard implementation. +type Shard interface { + ShardID() uint16 +} + +// +// GenericTask +// + +// Generic generic task, will be executed on each shard. +type Generic[TShard Shard] struct { + errs []error + shardFn func(shard TShard) error + wg sync.WaitGroup + createdTS int64 + executeTS int64 + created prometheus.Counter + done prometheus.Counter + live prometheus.Counter + execute prometheus.Counter +} + +// NewGeneric init new [Generic]. +func NewGeneric[TShard Shard]( + shardFn func(shard TShard) error, + created, done, live, execute prometheus.Counter, +) *Generic[TShard] { + t := &Generic[TShard]{ + shardFn: shardFn, + wg: sync.WaitGroup{}, + createdTS: time.Now().UnixMicro(), + created: created, + done: done, + live: live, + execute: execute, + } + t.created.Inc() + + return t +} + +// NewReadOnlyGeneric init new GenericTask for read only head. +func NewReadOnlyGeneric[TShard Shard](shardFn func(shard TShard) error) *Generic[TShard] { + t := &Generic[TShard]{ + shardFn: shardFn, + wg: sync.WaitGroup{}, + } + + return t +} + +// SetShardsNumber set shards number +func (t *Generic[TShard]) SetShardsNumber(number uint16) { + t.errs = make([]error, number) + t.wg.Add(int(number)) +} + +// ExecuteOnShard execute task on shard. +func (t *Generic[TShard]) ExecuteOnShard(shard TShard) { + atomic.CompareAndSwapInt64(&t.executeTS, 0, time.Now().UnixMicro()) + if len(t.errs) == 1 { + t.errs[0] = t.shardFn(shard) + } else { + t.errs[shard.ShardID()] = t.shardFn(shard) + } + + t.wg.Done() +} + +// Wait for the task to complete on all shards. +func (t *Generic[TShard]) Wait() error { + t.wg.Wait() + if t.done == nil { + return errors.Join(t.errs...) + } + + now := time.Now().UnixMicro() + t.done.Inc() + t.execute.Add(float64(now - t.executeTS)) + t.live.Add(float64(now - t.createdTS)) + + return errors.Join(t.errs...) +} diff --git a/pp/go/storage/head/task/waiter.go b/pp/go/storage/head/task/waiter.go new file mode 100644 index 0000000000..bf13ee0904 --- /dev/null +++ b/pp/go/storage/head/task/waiter.go @@ -0,0 +1,43 @@ +package task + +import "errors" + +// +// Task +// + +// Task the minimum required Task implementation. +type Task interface { + Wait() error +} + +// +// TaskWaiter +// + +// Waiter aggregates the wait for tasks to be completed. +type Waiter[TTask Task] struct { + tasks []TTask +} + +// NewTaskWaiter init new TaskWaiter for n task. +func NewTaskWaiter[TTask Task](n int) Waiter[TTask] { + return Waiter[TTask]{ + tasks: make([]TTask, 0, n), + } +} + +// Add task to waiter. +func (tw *Waiter[TTask]) Add(t TTask) { + tw.tasks = append(tw.tasks, t) +} + +// Wait for tasks to be completed. +func (tw *Waiter[TTask]) Wait() error { + var err error + for _, t := range tw.tasks { + err = errors.Join(err, t.Wait()) + } + + return err +} diff --git a/pp/go/storage/head/task/waiter_test.go b/pp/go/storage/head/task/waiter_test.go new file mode 100644 index 0000000000..d352f6377e --- /dev/null +++ b/pp/go/storage/head/task/waiter_test.go @@ -0,0 +1,22 @@ +package task_test + +import ( + "testing" + + "github.com/prometheus/prometheus/pp/go/storage/head/task" + "github.com/stretchr/testify/require" +) + +func TestTaskWaiter(t *testing.T) { + tw := task.NewTaskWaiter[*task.Generic[*testShard]](5) + err := tw.Wait() + require.NoError(t, err) +} + +// testHead implementation [Shard]. +type testShard struct{} + +// ShardID implementation [Shard]. +func (*testShard) ShardID() uint16 { + return 0 +} diff --git a/pp/go/storage/loader.go b/pp/go/storage/loader.go new file mode 100644 index 0000000000..244b9d34fd --- /dev/null +++ b/pp/go/storage/loader.go @@ -0,0 +1,417 @@ +package storage + +import ( + "bufio" + "fmt" + "io" + "math" + "os" + "path/filepath" + "sync" + "time" + + "github.com/prometheus/client_golang/prometheus" + + "github.com/prometheus/prometheus/pp/go/cppbridge" + "github.com/prometheus/prometheus/pp/go/logger" + "github.com/prometheus/prometheus/pp/go/storage/catalog" + "github.com/prometheus/prometheus/pp/go/storage/head/head" + "github.com/prometheus/prometheus/pp/go/storage/head/services" + "github.com/prometheus/prometheus/pp/go/storage/head/shard" + "github.com/prometheus/prometheus/pp/go/storage/head/shard/wal" + "github.com/prometheus/prometheus/pp/go/storage/head/shard/wal/reader" + "github.com/prometheus/prometheus/pp/go/storage/head/shard/wal/writer" + "github.com/prometheus/prometheus/pp/go/util" + "github.com/prometheus/prometheus/pp/go/util/optional" +) + +// Loader loads [Head] or [shard.Shard] from [Wal]. +type Loader struct { + dataDir string + maxSegmentSize uint32 + registerer prometheus.Registerer + unloadDataStorageInterval time.Duration +} + +// NewLoader init new [Loader]. +func NewLoader( + dataDir string, + maxSegmentSize uint32, + registerer prometheus.Registerer, + unloadDataStorageInterval time.Duration, +) *Loader { + return &Loader{ + dataDir: dataDir, + maxSegmentSize: maxSegmentSize, + registerer: registerer, + unloadDataStorageInterval: unloadDataStorageInterval, + } +} + +// Load [Head] from [Wal] by head ID. +// +//revive:disable-next-line:cognitive-complexity // function is not complicated +//revive:disable-next-line:function-length // long but readable. +//revive:disable-next-line:cyclomatic // but readable +func (l *Loader) Load( + headRecord *catalog.Record, + generation uint64, +) (_ *Head, corrupted bool) { + headID := headRecord.ID() + headDir := filepath.Join(l.dataDir, headID) + numberOfShards := headRecord.NumberOfShards() + shardLoadResults := make([]ShardLoadResult, numberOfShards) + + wg := &sync.WaitGroup{} + swn := writer.NewSegmentWriteNotifier(numberOfShards, headRecord.SetLastAppendedSegmentID) + for shardID := range numberOfShards { + wg.Add(1) + go func(shardID uint16) { + defer wg.Done() + shardLoadResults[shardID] = l.loadShard( + shardID, + headDir, + l.maxSegmentSize, + swn, + l.registerer, + l.unloadDataStorageInterval, + ) + }(shardID) + } + wg.Wait() + + shards := make([]*shard.Shard, numberOfShards) + numberOfSegmentsRead := optional.Optional[uint32]{} + for shardID, res := range shardLoadResults { + shards[shardID] = res.shard + if res.corrupted { + corrupted = true + } + + if numberOfSegmentsRead.IsNil() { + numberOfSegmentsRead.Set(res.numberOfSegments) + } else if numberOfSegmentsRead.Value() != res.numberOfSegments { + corrupted = true + // calculating maximum number of segments (critical for remote write). + if numberOfSegmentsRead.Value() < res.numberOfSegments { + numberOfSegmentsRead.Set(res.numberOfSegments) + } + } + } + + switch { + case headRecord.Status() == catalog.StatusActive: + // numberOfSegments here is actual number of segments. + if numberOfSegmentsRead.Value() > 0 { + headRecord.SetLastAppendedSegmentID(numberOfSegmentsRead.Value() - 1) + } + case isNumberOfSegmentsMismatched(headRecord, numberOfSegmentsRead.Value()): + corrupted = true + // numberOfSegments here is actual number of segments. + if numberOfSegmentsRead.Value() > 0 { + headRecord.SetLastAppendedSegmentID(numberOfSegmentsRead.Value() - 1) + } + + lastAppendedSegmentID := uint32(0) + if headRecord.LastAppendedSegmentID() != nil { + lastAppendedSegmentID = *headRecord.LastAppendedSegmentID() + } + + logger.Errorf( + "head: %s number of segments mismatched: last appended=%d, number of segments read=%d", + headRecord.ID(), + lastAppendedSegmentID, + numberOfSegmentsRead.Value(), + ) + } + + h := head.NewHead( + headID, + shards, + shard.NewPerGoroutineShard[*Wal], + headRecord.Acquire(), + generation, + l.registerer, + ) + + if err := services.MergeOutOfOrderChunksWithHead(h); err != nil { + corrupted = true + } + + logger.Debugf("[Loader] loaded head: %s, corrupted: %t", headRecord.ID(), corrupted) + + return h, corrupted +} + +func (*Loader) loadShard( + shardID uint16, + dir string, + maxSegmentSize uint32, + notifier *writer.SegmentWriteNotifier, + registerer prometheus.Registerer, + unloadDataStorageInterval time.Duration, +) ShardLoadResult { + shardDataLoader := NewShardDataLoader(shardID, dir, maxSegmentSize, notifier, registerer, unloadDataStorageInterval) + err := shardDataLoader.Load() + return ShardLoadResult{ + corrupted: err != nil, + numberOfSegments: shardDataLoader.shardData.numberOfSegments, + shard: shard.NewShard( + shardDataLoader.shardData.lss, + shardDataLoader.shardData.dataStorage, + shardDataLoader.shardData.unloadedDataStorage, + shardDataLoader.shardData.queriedSeriesStorage, + shardDataLoader.shardData.wal, + shardID, + ), + } +} + +// ShardLoadResult the result of loading a shard from a wal file. +type ShardLoadResult struct { + shard *shard.Shard + numberOfSegments uint32 + corrupted bool +} + +// ShardData data for creating a shard. +type ShardData struct { + lss *shard.LSS + dataStorage *shard.DataStorage + wal *Wal + unloadedDataStorage *shard.UnloadedDataStorage + queriedSeriesStorage *shard.QueriedSeriesStorage + numberOfSegments uint32 +} + +// ShardDataLoader loads shard data from a file and creates a shard. +type ShardDataLoader struct { + shardID uint16 + dir string + maxSegmentSize uint32 + shardData ShardData + notifier *writer.SegmentWriteNotifier + registerer prometheus.Registerer + unloadDataStorageInterval time.Duration +} + +// NewShardDataLoader init new [ShardDataLoader]. +func NewShardDataLoader( + shardID uint16, + dir string, + maxSegmentSize uint32, + notifier *writer.SegmentWriteNotifier, + registerer prometheus.Registerer, + unloadDataStorageInterval time.Duration, +) ShardDataLoader { + return ShardDataLoader{ + shardID: shardID, + dir: dir, + maxSegmentSize: maxSegmentSize, + notifier: notifier, + registerer: registerer, + unloadDataStorageInterval: unloadDataStorageInterval, + } +} + +// Load loads shard data from a file and creates a shard. +func (l *ShardDataLoader) Load() error { + l.shardData = ShardData{ + lss: shard.NewLSS(), + dataStorage: shard.NewDataStorage(), + wal: wal.NewCorruptedWal[ + *cppbridge.HeadEncodedSegment, + *writer.Buffered[*cppbridge.HeadEncodedSegment], + ](), + } + + shardWalFile, err := os.OpenFile( //nolint:gosec // need this permissions + GetShardWalFilename(l.dir, l.shardID), + os.O_RDONLY, + 0o666, //revive:disable-line:add-constant // file permissions simple readable as octa-number + ) + if err != nil { + return err + } + + queriedSeriesStorageIsEmpty := true + if l.unloadDataStorageInterval > 0 { + l.shardData.unloadedDataStorage = shard.NewUnloadedDataStorage( + shard.NewAppendFileStorage(GetUnloadedDataStorageFilename(l.dir, l.shardID)), + ) + queriedSeriesStorageIsEmpty, _ = l.loadQueriedSeries() + } + + shardWalFileName := shardWalFile.Name() + decoder, err := l.loadWalFile(bufio.NewReaderSize(shardWalFile, 1024*1024*10), queriedSeriesStorageIsEmpty) + _ = shardWalFile.Close() + if err != nil { + return err + } + + return l.createShardWal(shardWalFileName, decoder) +} + +// loadWalFile loads and decode wal file. +// +//revive:disable-next-line:flag-parameter this is a flag, but it's more convenient this way +func (l *ShardDataLoader) loadWalFile( + rd io.Reader, + queriedSeriesStorageIsEmpty bool, +) (*cppbridge.HeadWalDecoder, error) { + _, encoderVersion, _, err := reader.ReadHeader(rd) + if err != nil { + return nil, fmt.Errorf("failed to read wal header: %w", err) + } + + var unloader *dataUnloader + if !queriedSeriesStorageIsEmpty { + unloader = &dataUnloader{ + unloadedDataStorage: l.shardData.unloadedDataStorage, + unloadedIntervalIndex: math.MinInt64, + unloadInterval: l.unloadDataStorageInterval, + unloader: l.shardData.dataStorage.CreateUnusedSeriesDataUnloader(), + } + } + + decoder := cppbridge.NewHeadWalDecoder(l.shardData.lss.Target(), encoderVersion) + l.shardData.numberOfSegments, err = l.loadSegments( + rd, + decoder, + l.shardData.dataStorage, + unloader, + ) + + return decoder, err +} + +// createShardWal creates a wal for a shard. +func (l *ShardDataLoader) createShardWal(fileName string, walDecoder *cppbridge.HeadWalDecoder) error { + //revive:disable-next-line:add-constant // file permissions simple readable as octa-number + shardWalFile, err := util.OpenFileAppender(fileName, 0o666) + if err != nil { + return err + } + + sw, err := writer.NewBuffered( + l.shardID, + shardWalFile, + writer.WriteSegment[*cppbridge.HeadEncodedSegment], + l.notifier, + ) + if err != nil { + _ = shardWalFile.Close() + return err + } + + l.notifier.Set(l.shardID, l.shardData.numberOfSegments) + l.shardData.wal = wal.NewWal(walDecoder.CreateEncoder(), sw, l.maxSegmentSize, l.shardID, l.registerer) + + return nil +} + +type dataUnloader struct { + unloader *cppbridge.UnusedSeriesDataUnloader + unloadedDataStorage *shard.UnloadedDataStorage + unloadedIntervalIndex int64 + unloadInterval time.Duration +} + +func (d *dataUnloader) Unload(createTs, encodeTs time.Duration) error { + intervalIndex := int64(createTs / d.unloadInterval) + + if d.unloadedIntervalIndex == math.MinInt64 { + d.unloadedIntervalIndex = intervalIndex + + createTs = encodeTs + intervalIndex = int64(createTs / d.unloadInterval) + } + + if intervalIndex > d.unloadedIntervalIndex { + header, err := d.unloadedDataStorage.WriteSnapshot(d.unloader.CreateSnapshot()) + if err != nil { + return fmt.Errorf("failed to write unloaded data: %w", err) + } + + d.unloadedDataStorage.WriteIndex(header) + d.unloader.Unload() + d.unloadedIntervalIndex = intervalIndex + } + + return nil +} + +// loadSegments loads and decode segments from wal file. +func (*ShardDataLoader) loadSegments( + rd io.Reader, + walDecoder *cppbridge.HeadWalDecoder, + dataStorage *shard.DataStorage, + unloader *dataUnloader, +) (uint32, error) { + numberOfSegments := uint32(0) + + if err := wal.NewSegmentWalReader(rd, reader.NewSegment).ForEachSegment(func(segment *reader.Segment) error { + createTs, encodeTs, decodeErr := dataStorage.DecodeSegment(walDecoder, segment.Bytes()) + if decodeErr != nil { + return fmt.Errorf("failed to decode segment: %w", decodeErr) + } + + numberOfSegments++ + + if createTs != 0 && unloader != nil { + if err := unloader.Unload(time.Duration(createTs), time.Duration(encodeTs)); err != nil { + return fmt.Errorf("failed to unload data: %w", err) + } + } + + return nil + }); err != nil { + logger.Debugf(err.Error()) + return 0, err + } + + return numberOfSegments, nil +} + +func (l *ShardDataLoader) loadQueriedSeries() (bool, error) { + file1 := shard.NewFileStorage(GetQueriedSeriesStorageFilename(l.dir, l.shardID, 0)) + file2 := shard.NewFileStorage(GetQueriedSeriesStorageFilename(l.dir, l.shardID, 1)) + + l.shardData.queriedSeriesStorage = shard.NewQueriedSeriesStorage(file1, file2) + + if queriedSeries, err := l.shardData.queriedSeriesStorage.Read(); err != nil { + if file1.IsEmpty() && file2.IsEmpty() { + return true, nil + } + + logger.Warnf("error loading queried series: %v", err) + } else if !l.shardData.dataStorage.SetQueriedSeriesBitset(queriedSeries) { + logger.Warnf("error set queried series in storage: %v", err) + } + + return false, nil +} + +// GetShardWalFilename returns shard's Wal file name. +func GetShardWalFilename(dir string, shardID uint16) string { + return filepath.Join(dir, fmt.Sprintf("shard_%d.wal", shardID)) +} + +// GetUnloadedDataStorageFilename returns unloaded DataStorage file name. +func GetUnloadedDataStorageFilename(dir string, shardID uint16) string { + return filepath.Join(dir, fmt.Sprintf("unloaded_%d.ds", shardID)) +} + +// GetQueriedSeriesStorageFilename returns queried series storage file name. +func GetQueriedSeriesStorageFilename(dir string, shardID uint16, index uint8) string { + return filepath.Join(dir, fmt.Sprintf("queried_series_%d_%d.ds", shardID, index)) +} + +// isNumberOfSegmentsMismatched check number of segments loaded and last appended to record. +func isNumberOfSegmentsMismatched(record *catalog.Record, loadedSegments uint32) bool { + if record.LastAppendedSegmentID() == nil { + return loadedSegments != 0 + } + + return *record.LastAppendedSegmentID()+1 != loadedSegments +} diff --git a/pp/go/storage/loader_test.go b/pp/go/storage/loader_test.go new file mode 100644 index 0000000000..26ef9a0811 --- /dev/null +++ b/pp/go/storage/loader_test.go @@ -0,0 +1,395 @@ +package storage_test + +import ( + "os" + "path/filepath" + "testing" + "time" + + "github.com/google/uuid" + "github.com/jonboulle/clockwork" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/prometheus/model/labels" + "github.com/prometheus/prometheus/pp/go/cppbridge" + "github.com/prometheus/prometheus/pp/go/storage" + "github.com/prometheus/prometheus/pp/go/storage/catalog" + "github.com/prometheus/prometheus/pp/go/storage/head/services" + "github.com/prometheus/prometheus/pp/go/storage/head/shard" + "github.com/prometheus/prometheus/pp/go/storage/storagetest" + "github.com/stretchr/testify/suite" +) + +const ( + numberOfShards uint16 = 2 + + maxSegmentSize uint32 = 1024 + + unloadDataStorageInterval time.Duration = 100 +) + +type idGeneratorStub struct { + uuid uuid.UUID +} + +func newIdGeneratorStub() *idGeneratorStub { + return &idGeneratorStub{ + uuid: uuid.New(), + } +} + +func (g *idGeneratorStub) Generate() uuid.UUID { + return g.uuid +} + +func (g *idGeneratorStub) last() string { + return g.uuid.String() +} + +type HeadLoadSuite struct { + suite.Suite + dataDir string + clock clockwork.Clock + headIdGenerator *idGeneratorStub + catalog *catalog.Catalog +} + +func TestHeadLoadSuite(t *testing.T) { + suite.Run(t, new(HeadLoadSuite)) +} + +func (s *HeadLoadSuite) SetupTest() { + s.dataDir = s.createDataDirectory() + + s.clock = clockwork.NewFakeClockAt(time.Now()) + s.headIdGenerator = newIdGeneratorStub() + s.createCatalog() +} + +func (s *HeadLoadSuite) createDataDirectory() string { + dataDir := filepath.Join(s.T().TempDir(), "data") + s.Require().NoError(os.MkdirAll(dataDir, os.ModeDir)) + return dataDir +} + +func (s *HeadLoadSuite) createCatalog() { + l, err := catalog.NewFileLogV2(filepath.Join(s.dataDir, "catalog.log")) + s.Require().NoError(err) + + s.catalog, err = catalog.New( + s.clock, + l, + s.headIdGenerator, + catalog.DefaultMaxLogFileSize, + nil, + ) + s.Require().NoError(err) +} + +func (s *HeadLoadSuite) headDir() string { + return filepath.Join(s.dataDir, s.headIdGenerator.last()) +} + +func (s *HeadLoadSuite) createHead(unloadDataStorageInterval time.Duration) (*storage.Head, error) { + return storage.NewBuilder( + s.catalog, + s.dataDir, + maxSegmentSize, + prometheus.DefaultRegisterer, + unloadDataStorageInterval, + ).Build(0, numberOfShards) +} + +func (s *HeadLoadSuite) mustCreateHead(unloadDataStorageInterval time.Duration) *storage.Head { + h, err := s.createHead(unloadDataStorageInterval) + s.Require().NoError(err) + + s.catalog.SetStatus(h.ID(), catalog.StatusActive) + + return h +} + +func (s *HeadLoadSuite) loadHead(unloadDataStorageInterval time.Duration) (*storage.Head, bool) { + record, err := s.catalog.Get(s.headIdGenerator.last()) + s.Require().NoError(err) + + return storage.NewLoader(s.dataDir, maxSegmentSize, prometheus.DefaultRegisterer, unloadDataStorageInterval).Load(record, 0) +} + +func (s *HeadLoadSuite) mustLoadHead(unloadDataStorageInterval time.Duration) *storage.Head { + loadedHead, corrupted := s.loadHead(unloadDataStorageInterval) + s.False(corrupted) + + return loadedHead +} + +func (s *HeadLoadSuite) lockFileForCreation(fileName string) { + s.Require().NoError(os.RemoveAll(fileName)) + s.Require().NoError(os.Mkdir(fileName, os.ModeDir)) +} + +func (s *HeadLoadSuite) appendTimeSeries(head *storage.Head, timeSeries []storagetest.TimeSeries) { + storagetest.MustAppendTimeSeries(&s.Suite, head, timeSeries) +} + +func (*HeadLoadSuite) shards(head *storage.Head) (result []*shard.Shard) { + for sd := range head.RangeShards() { + result = append(result, sd) + } + + return result +} + +func (s *HeadLoadSuite) TestErrorCreateShardFileInOneShard() { + // Arrange + s.Require().NoError(os.Mkdir(s.headDir(), 0), os.ModeDir) + s.lockFileForCreation(storage.GetShardWalFilename(s.headDir(), 0)) + + // Act + head, err := s.createHead(0) + + // Assert + s.Require().Error(err) + s.Nil(head) +} + +func (s *HeadLoadSuite) TestErrorOpenShardFileInOneShard() { + // Arrange + sourceHead := s.mustCreateHead(0) + s.NoError(sourceHead.Close()) + + s.Require().NoError(os.Remove(storage.GetShardWalFilename(s.headDir(), 0))) + + // Act + head, corrupted := s.loadHead(0) + + // Assert + s.True(corrupted) + s.Nil(s.shards(head)[0].UnloadedDataStorage()) + s.Require().NoError(head.Close()) +} + +func (s *HeadLoadSuite) TestErrorOpenShardFileInAllShards() { + // Arrange + sourceHead := s.mustCreateHead(0) + s.NoError(sourceHead.Close()) + + s.Require().NoError(os.Remove(storage.GetShardWalFilename(s.headDir(), 0))) + s.Require().NoError(os.Remove(storage.GetShardWalFilename(s.headDir(), 1))) + + // Act + head, corrupted := s.loadHead(0) + + // Assert + s.True(corrupted) + s.Nil(s.shards(head)[0].UnloadedDataStorage()) + s.Nil(s.shards(head)[1].UnloadedDataStorage()) + s.Require().NoError(head.Close()) +} + +func (s *HeadLoadSuite) TestLoadWithDisabledDataUnloading() { + // Arrange + sourceHead := s.mustCreateHead(0) + s.appendTimeSeries(sourceHead, []storagetest.TimeSeries{ + { + Labels: labels.FromStrings("__name__", "wal_metric"), + Samples: []cppbridge.Sample{ + {Timestamp: 0, Value: 1}, + {Timestamp: 1, Value: 2}, + {Timestamp: 2, Value: 3}, + }, + }, + }) + s.Require().NoError(sourceHead.Close()) + + // Act + loadedHead := s.mustLoadHead(0) + + chunks, queryResult := s.shards(loadedHead)[0].DataStorage().Query(cppbridge.HeadDataStorageQuery{ + StartTimestampMs: 0, + EndTimestampMs: 2, + LabelSetIDs: []uint32{0}, + }) + err := loadedHead.Close() + + // Assert + s.Require().NoError(err) + s.Nil(s.shards(loadedHead)[0].UnloadedDataStorage()) + s.Nil(s.shards(loadedHead)[0].QueriedSeriesStorage()) + s.Nil(s.shards(loadedHead)[1].UnloadedDataStorage()) + s.Nil(s.shards(loadedHead)[1].QueriedSeriesStorage()) + s.Equal(cppbridge.DataStorageQueryStatusSuccess, queryResult.Status) + s.Equal(storagetest.SamplesMap{ + 0: []cppbridge.Sample{ + {Timestamp: 0, Value: 1}, + {Timestamp: 1, Value: 2}, + {Timestamp: 2, Value: 3}, + }, + }, storagetest.GetSamplesFromSerializedChunks(chunks)) + s.Equal([]cppbridge.Labels{ + {{Name: "__name__", Value: "wal_metric"}}, + }, s.shards(loadedHead)[0].LSS().Target().GetLabelSets([]uint32{0}).LabelsSets()) +} + +func (s *HeadLoadSuite) TestAppendAfterLoad() { + // Arrange + sourceHead := s.mustCreateHead(0) + s.appendTimeSeries(sourceHead, []storagetest.TimeSeries{ + { + Labels: labels.FromStrings("__name__", "wal_metric"), + Samples: []cppbridge.Sample{ + {Timestamp: 0, Value: 1}, + {Timestamp: 1, Value: 2}, + {Timestamp: 2, Value: 3}, + }, + }, + }) + s.Require().NoError(sourceHead.Close()) + + // Act + loadedHead := s.mustLoadHead(0) + s.appendTimeSeries(loadedHead, []storagetest.TimeSeries{ + { + Labels: labels.FromStrings("__name__", "wal_metric"), + Samples: []cppbridge.Sample{ + {Timestamp: 3, Value: 4}, + }, + }, + }) + + chunks, queryResult := s.shards(loadedHead)[0].DataStorage().Query(cppbridge.HeadDataStorageQuery{ + StartTimestampMs: 0, + EndTimestampMs: 4, + LabelSetIDs: []uint32{0}, + }) + + err := loadedHead.Close() + + // Assert + s.Require().NoError(err) + s.Equal(cppbridge.DataStorageQueryStatusSuccess, queryResult.Status) + s.Equal(storagetest.SamplesMap{ + 0: []cppbridge.Sample{ + {Timestamp: 0, Value: 1}, + {Timestamp: 1, Value: 2}, + {Timestamp: 2, Value: 3}, + {Timestamp: 3, Value: 4}, + }, + }, storagetest.GetSamplesFromSerializedChunks(chunks)) + s.Equal([]cppbridge.Labels{ + {{Name: "__name__", Value: "wal_metric"}}, + }, s.shards(loadedHead)[0].LSS().Target().GetLabelSets([]uint32{0}).LabelsSets()) +} + +func (s *HeadLoadSuite) TestLoadWithEnabledDataUnloading() { + // Arrange + sourceHead := s.mustCreateHead(0) + s.appendTimeSeries(sourceHead, []storagetest.TimeSeries{ + { + Labels: labels.FromStrings("__name__", "wal_metric"), + Samples: []cppbridge.Sample{ + {Timestamp: 0, Value: 1}, + {Timestamp: 1, Value: 2}, + {Timestamp: 2, Value: 3}, + }, + }, + }) + s.appendTimeSeries(sourceHead, []storagetest.TimeSeries{ + { + Labels: labels.FromStrings("__name__", "wal_metric"), + Samples: []cppbridge.Sample{ + {Timestamp: 100, Value: 1}, + {Timestamp: 101, Value: 2}, + {Timestamp: 102, Value: 3}, + }, + }, + }) + + s.Require().NoError(sourceHead.Close()) + + // Act + loadedHead := s.mustLoadHead(unloadDataStorageInterval) + + // Assert + s.Require().NotNil(s.shards(loadedHead)[0].UnloadedDataStorage()) + s.Require().NotNil(s.shards(loadedHead)[1].UnloadedDataStorage()) + s.True(s.shards(loadedHead)[0].UnloadedDataStorage().IsEmpty()) + s.True(s.shards(loadedHead)[1].UnloadedDataStorage().IsEmpty()) + s.Require().NoError(loadedHead.Close()) +} + +func (s *HeadLoadSuite) TestLoadWithDataUnloading() { + // Arrange + sourceHead := s.mustCreateHead(unloadDataStorageInterval) + s.appendTimeSeries(sourceHead, []storagetest.TimeSeries{ + { + Labels: labels.FromStrings("__name__", "wal_metric"), + Samples: []cppbridge.Sample{ + {Timestamp: 0, Value: 1}, + {Timestamp: 1, Value: 2}, + {Timestamp: 2, Value: 3}, + }, + }, + }) + s.appendTimeSeries(sourceHead, []storagetest.TimeSeries{ + { + Labels: labels.FromStrings("__name__", "wal_metric"), + Samples: []cppbridge.Sample{ + {Timestamp: 100, Value: 1}, + {Timestamp: 101, Value: 2}, + {Timestamp: 102, Value: 3}, + }, + }, + }) + + s.Require().NoError(services.UnloadUnusedSeriesDataWithHead(sourceHead)) + s.Require().NoError(sourceHead.Close()) + + // Act + loadedHead := s.mustLoadHead(unloadDataStorageInterval) + + // Assert + s.NotNil(s.shards(loadedHead)[0].UnloadedDataStorage()) + s.NotNil(s.shards(loadedHead)[1].UnloadedDataStorage()) + s.False(s.shards(loadedHead)[0].UnloadedDataStorage().IsEmpty()) + s.True(s.shards(loadedHead)[1].UnloadedDataStorage().IsEmpty()) + s.Require().NoError(loadedHead.Close()) +} + +func (s *HeadLoadSuite) TestErrorDataUnloading() { + // Arrange + sourceHead := s.mustCreateHead(unloadDataStorageInterval) + s.appendTimeSeries(sourceHead, []storagetest.TimeSeries{ + { + Labels: labels.FromStrings("__name__", "wal_metric"), + Samples: []cppbridge.Sample{ + {Timestamp: 0, Value: 1}, + {Timestamp: 1, Value: 2}, + {Timestamp: 2, Value: 3}, + }, + }, + }) + s.appendTimeSeries(sourceHead, []storagetest.TimeSeries{ + { + Labels: labels.FromStrings("__name__", "wal_metric"), + Samples: []cppbridge.Sample{ + {Timestamp: 100, Value: 1}, + {Timestamp: 101, Value: 2}, + {Timestamp: 102, Value: 3}, + }, + }, + }) + + s.Require().NoError(services.UnloadUnusedSeriesDataWithHead(sourceHead)) + s.Require().NoError(sourceHead.Close()) + + // Act + s.lockFileForCreation(storage.GetUnloadedDataStorageFilename(s.headDir(), 0)) + s.lockFileForCreation(storage.GetUnloadedDataStorageFilename(s.headDir(), 1)) + loadedHead, corrupted := s.loadHead(unloadDataStorageInterval) + + // Assert + s.True(corrupted) + s.NotNil(s.shards(loadedHead)[0].UnloadedDataStorage()) + s.NotNil(s.shards(loadedHead)[1].UnloadedDataStorage()) + s.Require().NoError(loadedHead.Close()) +} diff --git a/pp/go/storage/manager.go b/pp/go/storage/manager.go new file mode 100644 index 0000000000..8a1fbea650 --- /dev/null +++ b/pp/go/storage/manager.go @@ -0,0 +1,541 @@ +package storage + +import ( + "context" + "errors" + "fmt" + "math" + "os" + "path/filepath" + "sync/atomic" + "time" + + "github.com/jonboulle/clockwork" + "github.com/oklog/run" + "github.com/prometheus/client_golang/prometheus" + + "github.com/prometheus/prometheus/config" + + "github.com/prometheus/prometheus/pp/go/cppbridge" + "github.com/prometheus/prometheus/pp/go/logger" + "github.com/prometheus/prometheus/pp/go/storage/block" + "github.com/prometheus/prometheus/pp/go/storage/catalog" + "github.com/prometheus/prometheus/pp/go/storage/head/container" + "github.com/prometheus/prometheus/pp/go/storage/head/head" + "github.com/prometheus/prometheus/pp/go/storage/head/keeper" + "github.com/prometheus/prometheus/pp/go/storage/head/services" + "github.com/prometheus/prometheus/pp/go/storage/head/shard" + "github.com/prometheus/prometheus/pp/go/storage/mediator" + "github.com/prometheus/prometheus/pp/go/storage/querier" + "github.com/prometheus/prometheus/pp/go/storage/ready" + "github.com/prometheus/prometheus/pp/go/util" +) + +const ( + // DefaultRotateDuration default block duration. + DefaultRotateDuration = 2 * time.Hour + + // DefaultMergeDuration the default interval for the merge out of order chunks. + DefaultMergeDuration = 5 * time.Minute + + // DefaultMetricWriteInterval default metric scrape interval. + DefaultMetricWriteInterval = 15 * time.Second + + // defaultStartMetricWriteInterval the default interval for start [MetricsUpdater] timer. + defaultStartMetricWriteInterval = 5 * time.Second + + // DefaultPersistDuration the default interval for persisting [Head]. + DefaultPersistDuration = 2 * time.Minute + + // DefaultUnloadDataStorageInterval the default interval for unloading [DataStorage]. + DefaultUnloadDataStorageInterval = 5 * time.Minute + + // defaultStartPersistnerInterval the default interval for start [Persistener] timer. + defaultStartPersistnerInterval = 15 * time.Second +) + +var ( + // UnloadDataStorage flags for unloading [DataStorage]. + UnloadDataStorage = false + + // DefaultNumberOfShards default number of shards. + DefaultNumberOfShards uint16 = 2 +) + +// +// Options +// + +// Options manager launch options. +type Options struct { + Seed uint64 + BlockDuration time.Duration + CommitInterval time.Duration + MaxRetentionPeriod time.Duration + HeadRetentionPeriod time.Duration + KeeperCapacity int + DataDir string + MaxSegmentSize uint32 + NumberOfShards uint16 +} + +// +// Config +// + +// Config config for [Manager]. +type Config struct { + numberOfShards uint32 +} + +// NewConfig init new [Config]. +func NewConfig(numberOfShards uint16) *Config { + if numberOfShards == 0 { + numberOfShards = DefaultNumberOfShards + } + + return &Config{ + numberOfShards: uint32(numberOfShards), + } +} + +// NumberOfShards returns current number of shards. +func (c *Config) NumberOfShards() uint16 { + return uint16(atomic.LoadUint32(&c.numberOfShards)) // #nosec G115 // no overflow +} + +// SetNumberOfShards set new number of shards. +func (c *Config) SetNumberOfShards(numberOfShards uint16) bool { + if numberOfShards == 0 { + numberOfShards = DefaultNumberOfShards + } + + if c.NumberOfShards() == numberOfShards { + return false + } + + atomic.StoreUint32(&c.numberOfShards, uint32(numberOfShards)) + + return true +} + +// +// Manager +// + +// Manager manages services for the work of the heads. +type Manager struct { + g run.Group + closer *util.Closer + proxy *Proxy + cgogc *cppbridge.CGOGC + cfg *Config + rotatorMediator *mediator.Mediator + mergerMediator *mediator.Mediator + isRunning bool +} + +// NewManager init new [Manager]. +// +//revive:disable-next-line:function-length // this is contructor. +func NewManager( + o *Options, + clock clockwork.Clock, + hcatalog *catalog.Catalog, + reloadBlocksNotifier *TriggerNotifier, + removedHeadNotifier *TriggerNotifier, + readyNotifier ready.Notifier, + r prometheus.Registerer, +) (*Manager, error) { + if o == nil { + return nil, errors.New("manager options is nil") + } + + dataDir, err := filepath.Abs(o.DataDir) + if err != nil { + return nil, err + } + o.DataDir = dataDir + + dirStat, err := os.Stat(o.DataDir) + if err != nil { + return nil, fmt.Errorf("failed to stat dir: %w", err) + } + + if !dirStat.IsDir() { + return nil, fmt.Errorf("%s is not directory", o.DataDir) + } + + var unloadDataStorageInterval time.Duration + if UnloadDataStorage { + unloadDataStorageInterval = DefaultUnloadDataStorageInterval + } + + builder := NewBuilder(hcatalog, o.DataDir, o.MaxSegmentSize, r, unloadDataStorageInterval) + loader := NewLoader(o.DataDir, o.MaxSegmentSize, r, unloadDataStorageInterval) + cfg := NewConfig(o.NumberOfShards) + h, err := uploadOrBuildHead(clock, hcatalog, builder, loader, o.BlockDuration, cfg.NumberOfShards()) + if err != nil { + return nil, err + } + + if _, err = hcatalog.SetStatus(h.ID(), catalog.StatusActive); err != nil { + return nil, errors.Join(fmt.Errorf("failed to set active status: %w", err), h.Close()) + } + + hKeeper := keeper.NewKeeper[Head]( + o.KeeperCapacity, + removedHeadNotifier, + ) + + m := &Manager{ + g: run.Group{}, + closer: util.NewCloser(), + proxy: NewProxy(container.NewWeighted(h), hKeeper, services.CFSViaRange), + cgogc: cppbridge.NewCGOGC(r), + cfg: cfg, + rotatorMediator: mediator.NewMediator( + mediator.NewRotateTimerWithSeed(clock, o.BlockDuration, o.Seed), + ), + mergerMediator: mediator.NewMediator( + mediator.NewConstantIntervalTimer(clock, DefaultMergeDuration, DefaultMergeDuration), + ), + } + + m.initServices(o, hcatalog, builder, loader, reloadBlocksNotifier, readyNotifier, clock, r) + logger.Infof("[Head Manager] created") + + return m, nil +} + +// ApplyConfig update config. +func (m *Manager) ApplyConfig(cfg *config.Config) error { + logger.Infof("reconfiguration start") + defer logger.Infof("reconfiguration completed") + + if m.proxy.Get().NumberOfShards() == cfg.PPNumberOfShards() { + return nil + } + + if m.cfg.SetNumberOfShards(cfg.PPNumberOfShards()) { + m.rotatorMediator.Trigger() + } + + return nil +} + +// MergeOutOfOrderChunks send signal to merge chunks with out of order data chunks. +func (m *Manager) MergeOutOfOrderChunks() { + m.mergerMediator.Trigger() +} + +// Proxy returns proxy to the active [Head] and the keeper of old [Head]s. +func (m *Manager) Proxy() *Proxy { + return m.proxy +} + +// Run launches the [Manager]'s services. +func (m *Manager) Run() error { + defer m.closer.Done() + + return m.g.Run() +} + +// Shutdown safe shutdown [Manager]: stop services and close [Head]'s. +func (m *Manager) Shutdown(ctx context.Context) error { + m.close() + + return errors.Join(m.proxy.Close(), m.cgogc.Shutdown(ctx)) +} + +// initServices initializes services for startup. +// +//revive:disable-next-line:function-length // init contructor. +func (m *Manager) initServices( + o *Options, + hcatalog *catalog.Catalog, + builder *Builder, + loader *Loader, + reloadBlocksTriggerNotifier *TriggerNotifier, + readyNotifier ready.Notifier, + clock clockwork.Clock, + r prometheus.Registerer, +) { + baseCtx := context.Background() + + // Termination handler. + m.g.Add( + func() error { + readyNotifier.NotifyReady() + m.isRunning = true + <-m.closer.Signal() + + return nil + }, + func(error) { + m.close() + }, + ) + + // Persistener + persistenerMediator := mediator.NewMediator( + mediator.NewConstantIntervalTimer(clock, defaultStartPersistnerInterval, DefaultPersistDuration), + ) + m.g.Add( + func() error { + services.NewPersistenerService( + m.proxy, + loader, + hcatalog, + block.NewWriter[*shard.Shard]( + o.DataDir, + block.DefaultChunkSegmentSize, + o.BlockDuration, + r, + ), + reloadBlocksTriggerNotifier, + clock, + persistenerMediator, + o.MaxRetentionPeriod, + o.HeadRetentionPeriod, + r, + ).Execute() + + return nil + }, + func(error) { + persistenerMediator.Close() + }, + ) + + // Rotator + rotatorCtx, rotatorCancel := context.WithCancel(baseCtx) + m.g.Add( + func() error { + return services.NewRotator( + m.proxy, + builder, + m.rotatorMediator, + m.cfg, + &headInformer{catalog: hcatalog}, + head.CopyAddedSeries[*shard.Shard, *shard.PerGoroutineShard](shard.CopyAddedSeries), + persistenerMediator.TriggerWithResetTimer, + r, + ).Execute(rotatorCtx) + }, + func(error) { + m.rotatorMediator.Close() + rotatorCancel() + }, + ) + + // Committer + committerMediator := mediator.NewMediator( + mediator.NewConstantIntervalTimer(clock, o.CommitInterval, o.CommitInterval), + ) + committerCtx, committerCancel := context.WithCancel(baseCtx) + m.g.Add( + func() error { + return services.NewCommitter( + m.proxy, + committerMediator, + isNewHead(clock, hcatalog, o.CommitInterval), + ).Execute(committerCtx) + }, + func(error) { + committerMediator.Close() + committerCancel() + }, + ) + + // Merger + mergerCtx, mergerCancel := context.WithCancel(baseCtx) + m.g.Add( + func() error { + return services.NewMerger( + m.proxy, + m.mergerMediator, + isNewHead(clock, hcatalog, DefaultMergeDuration), + ).Execute(mergerCtx) + }, + func(error) { + m.mergerMediator.Close() + mergerCancel() + }, + ) + + // MetricsUpdater + metricsUpdaterMediator := mediator.NewMediator( + mediator.NewConstantIntervalTimer(clock, defaultStartMetricWriteInterval, DefaultMetricWriteInterval), + ) + metricsUpdaterCtx, metricsUpdaterCancel := context.WithCancel(baseCtx) + m.g.Add( + func() error { + return services.NewMetricsUpdater( + m.proxy, + metricsUpdaterMediator, + querier.QueryHeadStatus, + r, + ).Execute(metricsUpdaterCtx) + }, + func(error) { + metricsUpdaterMediator.Close() + metricsUpdaterCancel() + }, + ) +} + +func (m *Manager) close() { + if !m.isRunning { + m.closer.Done() + } + + select { + case <-m.closer.Signal(): + default: + _ = m.closer.Close() + } +} + +// +// headInformer +// + +// headInformer wrapper over [catalog.Catalog] for set statuses and get info. +type headInformer struct { + catalog *catalog.Catalog +} + +// CreatedAt returns the timestamp when the [Record]([Head]) was created. +func (hi *headInformer) CreatedAt(headID string) time.Duration { + record, err := hi.catalog.Get(headID) + if err != nil { + return time.Duration(math.MaxInt64) + } + + return time.Duration(record.CreatedAt()) * time.Millisecond +} + +// SetActiveStatus sets the [catalog.StatusActive] status by headID. +func (hi *headInformer) SetActiveStatus(headID string) error { + _, err := hi.catalog.SetStatus(headID, catalog.StatusActive) + return err +} + +// SetRotatedStatus sets the [catalog.StatusRotated] status by headID. +func (hi *headInformer) SetRotatedStatus(headID string) error { + _, err := hi.catalog.SetStatus(headID, catalog.StatusRotated) + return err +} + +// +// isNewHead +// + +// isNewHead builds a checker that checks if the head is new. +func isNewHead(clock clockwork.Clock, hcatalog *catalog.Catalog, interval time.Duration) func(headID string) bool { + return func(headID string) bool { + rec, err := hcatalog.Get(headID) + if err != nil { + return true + } + + return clock.Now().Add(-interval).UnixMilli() < rec.CreatedAt() + } +} + +// +// TriggerNotifier +// + +// TriggerNotifier to receive notifications about new events. +type TriggerNotifier struct { + c chan struct{} +} + +// NewTriggerNotifier init new [TriggerNotifier]. +func NewTriggerNotifier() *TriggerNotifier { + return &TriggerNotifier{c: make(chan struct{}, 1)} +} + +// Chan returns channel with notifications. +func (tn *TriggerNotifier) Chan() <-chan struct{} { + return tn.c +} + +// Notify sends a notify that the writing is completed. +func (tn *TriggerNotifier) Notify() { + select { + case tn.c <- struct{}{}: + default: + } +} + +// +// uploadOrBuildHead +// + +// uploadOrBuildHead uploads or builds a new head. +// +//revive:disable-next-line:function-length // long but readable. +//revive:disable-next-line:cyclomatic // long but readable. +func uploadOrBuildHead( + clock clockwork.Clock, + hcatalog *catalog.Catalog, + builder *Builder, + loader *Loader, + blockDuration time.Duration, + numberOfShards uint16, +) (*Head, error) { + headRecords := hcatalog.List( + func(record *catalog.Record) bool { + statusIsAppropriate := record.Status() == catalog.StatusNew || + record.Status() == catalog.StatusActive + + isInBlockTimeRange := clock.Now().Sub( + time.UnixMilli(record.CreatedAt()), + ).Milliseconds() < blockDuration.Milliseconds() + + return record.DeletedAt() == 0 && statusIsAppropriate && isInBlockTimeRange + }, + func(lhs, rhs *catalog.Record) bool { + return lhs.CreatedAt() > rhs.CreatedAt() + }, + ) + + if numberOfShards == 0 { + numberOfShards = DefaultNumberOfShards + } + + var generation uint64 + if len(headRecords) == 0 { + logger.Debugf("[Head Manager] no suitable heads were found, building new") + return builder.Build(generation, numberOfShards) + } + + h, corrupted := loader.Load(headRecords[0], generation) + if corrupted { + if !headRecords[0].Corrupted() { + if _, setCorruptedErr := hcatalog.SetCorrupted(headRecords[0].ID()); setCorruptedErr != nil { + logger.Errorf("failed to set corrupted state, head {%s}: %v", headRecords[0].ID(), setCorruptedErr) + } + } + logger.Warnf("[Head Manager] upload corrupted head {%s}, building new...", headRecords[0].ID()) + + if _, err := hcatalog.SetStatus(headRecords[0].ID(), catalog.StatusRotated); err != nil { + logger.Warnf("failed to set rotated status for head {%s}: %s", headRecords[0].ID(), err) + } + + _ = h.Close() + + return builder.Build(generation, numberOfShards) + } + + if _, err := hcatalog.SetStatus(headRecords[0].ID(), catalog.StatusActive); err != nil { + logger.Warnf("failed to set active status for head {%s}: %s", headRecords[0].ID(), err) + + return builder.Build(generation, numberOfShards) + } + + return h, nil +} diff --git a/pp/go/storage/mediator/mediator.go b/pp/go/storage/mediator/mediator.go new file mode 100644 index 0000000000..b3bcfef5b8 --- /dev/null +++ b/pp/go/storage/mediator/mediator.go @@ -0,0 +1,100 @@ +package mediator + +import ( + "sync" + "time" + + "github.com/prometheus/prometheus/pp/go/util" +) + +//go:generate -command moq go run github.com/matryer/moq --rm --skip-ensure --pkg mediator_test --out + +// +// Timer +// + +// Timer implementation timer. +// +//go:generate moq mediator_moq_test.go . Timer +type Timer interface { + // Chan returns chan with ticker time. + Chan() <-chan time.Time + + // Reset changes the timer to expire after duration Block and clearing channels. + Reset() + + // Stop prevents the Timer from firing. + Stop() +} + +// +// Mediator +// + +// Mediator notifies about events via the channel. +type Mediator struct { + timer Timer + c chan struct{} + closeOnce sync.Once + closer *util.Closer +} + +// NewMediator init new Mediator. +func NewMediator(timer Timer) *Mediator { + m := &Mediator{ + timer: timer, + c: make(chan struct{}), + closeOnce: sync.Once{}, + closer: util.NewCloser(), + } + + go m.loop() + + return m +} + +// C returns channel with events. +func (m *Mediator) C() <-chan struct{} { + return m.c +} + +// Close stops the internal timer and clears the channel. +func (m *Mediator) Close() { + _ = m.closer.Close() + m.timer.Stop() + m.closeOnce.Do(func() { + close(m.c) + }) +} + +// Trigger send notify to channel. +func (m *Mediator) Trigger() { + select { + case m.c <- struct{}{}: + default: + } +} + +// TriggerWithResetTimer send notify to channel and reset timer. +func (m *Mediator) TriggerWithResetTimer() { + select { + case m.c <- struct{}{}: + m.timer.Reset() + default: + } +} + +// loop by timer. +func (m *Mediator) loop() { + defer m.closer.Done() + + for { + select { + case <-m.timer.Chan(): + m.Trigger() + m.timer.Reset() + case <-m.closer.Signal(): + return + } + } +} diff --git a/pp/go/storage/mediator/mediator_moq_test.go b/pp/go/storage/mediator/mediator_moq_test.go new file mode 100644 index 0000000000..0d5a5ce0b7 --- /dev/null +++ b/pp/go/storage/mediator/mediator_moq_test.go @@ -0,0 +1,138 @@ +// Code generated by moq; DO NOT EDIT. +// github.com/matryer/moq + +package mediator_test + +import ( + "sync" + "time" +) + +// TimerMock is a mock implementation of mediator.Timer. +// +// func TestSomethingThatUsesTimer(t *testing.T) { +// +// // make and configure a mocked mediator.Timer +// mockedTimer := &TimerMock{ +// ChanFunc: func() <-chan time.Time { +// panic("mock out the Chan method") +// }, +// ResetFunc: func() { +// panic("mock out the Reset method") +// }, +// StopFunc: func() { +// panic("mock out the Stop method") +// }, +// } +// +// // use mockedTimer in code that requires mediator.Timer +// // and then make assertions. +// +// } +type TimerMock struct { + // ChanFunc mocks the Chan method. + ChanFunc func() <-chan time.Time + + // ResetFunc mocks the Reset method. + ResetFunc func() + + // StopFunc mocks the Stop method. + StopFunc func() + + // calls tracks calls to the methods. + calls struct { + // Chan holds details about calls to the Chan method. + Chan []struct { + } + // Reset holds details about calls to the Reset method. + Reset []struct { + } + // Stop holds details about calls to the Stop method. + Stop []struct { + } + } + lockChan sync.RWMutex + lockReset sync.RWMutex + lockStop sync.RWMutex +} + +// Chan calls ChanFunc. +func (mock *TimerMock) Chan() <-chan time.Time { + if mock.ChanFunc == nil { + panic("TimerMock.ChanFunc: method is nil but Timer.Chan was just called") + } + callInfo := struct { + }{} + mock.lockChan.Lock() + mock.calls.Chan = append(mock.calls.Chan, callInfo) + mock.lockChan.Unlock() + return mock.ChanFunc() +} + +// ChanCalls gets all the calls that were made to Chan. +// Check the length with: +// +// len(mockedTimer.ChanCalls()) +func (mock *TimerMock) ChanCalls() []struct { +} { + var calls []struct { + } + mock.lockChan.RLock() + calls = mock.calls.Chan + mock.lockChan.RUnlock() + return calls +} + +// Reset calls ResetFunc. +func (mock *TimerMock) Reset() { + if mock.ResetFunc == nil { + panic("TimerMock.ResetFunc: method is nil but Timer.Reset was just called") + } + callInfo := struct { + }{} + mock.lockReset.Lock() + mock.calls.Reset = append(mock.calls.Reset, callInfo) + mock.lockReset.Unlock() + mock.ResetFunc() +} + +// ResetCalls gets all the calls that were made to Reset. +// Check the length with: +// +// len(mockedTimer.ResetCalls()) +func (mock *TimerMock) ResetCalls() []struct { +} { + var calls []struct { + } + mock.lockReset.RLock() + calls = mock.calls.Reset + mock.lockReset.RUnlock() + return calls +} + +// Stop calls StopFunc. +func (mock *TimerMock) Stop() { + if mock.StopFunc == nil { + panic("TimerMock.StopFunc: method is nil but Timer.Stop was just called") + } + callInfo := struct { + }{} + mock.lockStop.Lock() + mock.calls.Stop = append(mock.calls.Stop, callInfo) + mock.lockStop.Unlock() + mock.StopFunc() +} + +// StopCalls gets all the calls that were made to Stop. +// Check the length with: +// +// len(mockedTimer.StopCalls()) +func (mock *TimerMock) StopCalls() []struct { +} { + var calls []struct { + } + mock.lockStop.RLock() + calls = mock.calls.Stop + mock.lockStop.RUnlock() + return calls +} diff --git a/pp/go/storage/mediator/mediator_test.go b/pp/go/storage/mediator/mediator_test.go new file mode 100644 index 0000000000..1595c9a6a6 --- /dev/null +++ b/pp/go/storage/mediator/mediator_test.go @@ -0,0 +1,235 @@ +package mediator_test + +import ( + "context" + "sync" + "testing" + "time" + + "github.com/prometheus/prometheus/pp/go/storage/mediator" + "github.com/stretchr/testify/suite" +) + +type MediatorSuite struct { + suite.Suite +} + +func TestMediatorSuite(t *testing.T) { + suite.Run(t, new(MediatorSuite)) +} + +func (s *MediatorSuite) TestC() { + chTimer := make(chan time.Time, 1) + + timer := &TimerMock{ + ChanFunc: func() <-chan time.Time { + return chTimer + }, + ResetFunc: func() {}, + StopFunc: func() {}, + } + + m := mediator.NewMediator(timer) + + counter := 0 + done := make(chan struct{}) + start := sync.WaitGroup{} + start.Add(1) + + s.T().Run("service_run", func(t *testing.T) { + t.Parallel() + start.Done() + <-m.C() + counter++ + close(done) + }) + + s.T().Run("timer_tick", func(t *testing.T) { + t.Parallel() + start.Wait() + + s.T().Log("timer tick") + chTimer <- time.Time{} + chTimer <- time.Time{} + + <-done + + s.Equal(1, counter) + m.Close() + }) +} + +func (s *MediatorSuite) TestClose() { + chTimer := make(chan time.Time, 1) + stopCounter := 0 + + timer := &TimerMock{ + ChanFunc: func() <-chan time.Time { + return chTimer + }, + ResetFunc: func() {}, + StopFunc: func() { stopCounter++ }, + } + + m := mediator.NewMediator(timer) + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + counter := 0 + done := make(chan struct{}) + start := sync.WaitGroup{} + start.Add(1) + go func() { + start.Done() + + for range m.C() { + counter++ + break + } + + close(done) + }() + + start.Wait() + s.T().Log("mediator close") + m.Close() + + select { + case <-done: + case <-ctx.Done(): + m.Trigger() + } + cancel() + + <-done + + s.Equal(0, counter) + s.Equal(1, stopCounter) +} + +func (s *MediatorSuite) TestTrigger() { + chTimer := make(chan time.Time, 1) + + timer := &TimerMock{ + ChanFunc: func() <-chan time.Time { + return chTimer + }, + ResetFunc: func() {}, + StopFunc: func() {}, + } + + m := mediator.NewMediator(timer) + defer m.Close() + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + counter := 0 + done := make(chan struct{}) + start := sync.WaitGroup{} + start.Add(1) + go func() { + start.Done() + select { + case <-m.C(): + counter++ + close(done) + case <-ctx.Done(): + } + }() + + start.Wait() + s.T().Log("trigger") + m.Trigger() + + select { + case <-done: + case <-ctx.Done(): + } + cancel() + + s.Equal(1, counter) + s.Empty(timer.ResetCalls()) +} + +func (s *MediatorSuite) TestTriggerWithResetTimer() { + chTimer := make(chan time.Time, 1) + + timer := &TimerMock{ + ChanFunc: func() <-chan time.Time { + return chTimer + }, + ResetFunc: func() {}, + StopFunc: func() {}, + } + + m := mediator.NewMediator(timer) + defer m.Close() + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + counter := 0 + done := make(chan struct{}) + start := sync.WaitGroup{} + start.Add(1) + go func() { + start.Done() + select { + case <-m.C(): + counter++ + close(done) + case <-ctx.Done(): + } + }() + + start.Wait() + s.T().Log("trigger with reset timer") + m.TriggerWithResetTimer() + + select { + case <-done: + case <-ctx.Done(): + } + cancel() + + s.Equal(1, counter) + s.Len(timer.ResetCalls(), 1) +} + +func (s *MediatorSuite) TestTriggerSkip() { + chTimer := make(chan time.Time, 1) + + timer := &TimerMock{ + ChanFunc: func() <-chan time.Time { + return chTimer + }, + ResetFunc: func() {}, + StopFunc: func() {}, + } + + m := mediator.NewMediator(timer) + defer m.Close() + + ctx, cancel := context.WithTimeout(context.Background(), 1*time.Second) + counter := 0 + done := make(chan struct{}) + start := sync.WaitGroup{} + start.Add(1) + go func() { + start.Wait() + select { + case <-m.C(): + counter++ + close(done) + case <-ctx.Done(): + } + }() + + s.T().Log("trigger") + m.Trigger() + start.Done() + + select { + case <-done: + case <-ctx.Done(): + } + cancel() + + s.Equal(0, counter) +} diff --git a/pp/go/storage/mediator/timer.go b/pp/go/storage/mediator/timer.go new file mode 100644 index 0000000000..1d7d7303b4 --- /dev/null +++ b/pp/go/storage/mediator/timer.go @@ -0,0 +1,140 @@ +package mediator + +import ( + "math" + "math/rand" + "sync" + "time" + + "github.com/jonboulle/clockwork" +) + +// +// RotateTimer +// + +// RotateTimer - custom timer with reset the timer for the delay time. +type RotateTimer struct { + clock clockwork.Clock + timer clockwork.Timer + rotateAt time.Time + mx *sync.Mutex + durationBlock int64 + rndDurationBlock int64 +} + +// NewRotateTimer init new [RotateTimer]. The duration durationBlock and delayAfterNotify must be greater than zero; +// if not, Ticker will panic. Stop the ticker to release associated resources. +func NewRotateTimer(clock clockwork.Clock, desiredBlockFormationDuration time.Duration) *RotateTimer { + return NewRotateTimerWithSeed( + clock, + desiredBlockFormationDuration, + uint64(clock.Now().UnixNano()), // #nosec G115 // no overflow + ) +} + +// NewRotateTimerWithSeed init new [RotateTimer]. The duration durationBlock and delayAfterNotify +// must be greater than zero; if not, Ticker will panic. Stop the ticker to release associated resources. +func NewRotateTimerWithSeed( + clock clockwork.Clock, + desiredBlockFormationDuration time.Duration, + seed uint64, +) *RotateTimer { + bd := desiredBlockFormationDuration.Milliseconds() + //nolint:gosec // there is no need for cryptographic strength here + rnd := rand.New(rand.NewSource(int64(seed))) + rt := &RotateTimer{ + clock: clock, + durationBlock: bd, + rndDurationBlock: rnd.Int63n(bd), + mx: new(sync.Mutex), + } + + rt.rotateAt = rt.RotateAtNext() + rt.timer = clock.NewTimer(rt.rotateAt.Sub(rt.clock.Now())) + + return rt +} + +// Chan - return chan with ticker time. +func (rt *RotateTimer) Chan() <-chan time.Time { + return rt.timer.Chan() +} + +// Reset - changes the timer to expire after duration Block and clearing channels. +func (rt *RotateTimer) Reset() { + rt.mx.Lock() + rt.rotateAt = rt.RotateAtNext() + if !rt.timer.Stop() { + select { + case <-rt.timer.Chan(): + default: + } + } + rt.timer.Reset(rt.rotateAt.Sub(rt.clock.Now())) + rt.mx.Unlock() +} + +// RotateAtNext - calculated next rotate time. +func (rt *RotateTimer) RotateAtNext() time.Time { + now := rt.clock.Now().UnixMilli() + k := now % rt.durationBlock + startBlock := math.Floor(float64(now)/float64(rt.durationBlock)) * float64(rt.durationBlock) + + if rt.rndDurationBlock > k { + return time.UnixMilli(int64(startBlock) + rt.rndDurationBlock) + } + + return time.UnixMilli(int64(startBlock) + rt.durationBlock + rt.rndDurationBlock) +} + +// Stop - prevents the Timer from firing. +// Stop does not close the channel, to prevent a read from the channel succeeding incorrectly. +func (rt *RotateTimer) Stop() { + // drain timer + if !rt.timer.Stop() { + select { + case <-rt.timer.Chan(): + default: + } + } +} + +// +// ConstantIntervalTimer +// + +// ConstantIntervalTimer timer with contatnt interval duration. +type ConstantIntervalTimer struct { + timer clockwork.Timer + interval time.Duration +} + +// NewConstantIntervalTimer init new [ConstantIntervalTimer]. +func NewConstantIntervalTimer(clock clockwork.Clock, start, interval time.Duration) *ConstantIntervalTimer { + return &ConstantIntervalTimer{ + timer: clock.NewTimer(start), + interval: interval, + } +} + +// Chan returns chan with tick time. +func (t *ConstantIntervalTimer) Chan() <-chan time.Time { + return t.timer.Chan() +} + +// Reset to a constant interval duration. +func (t *ConstantIntervalTimer) Reset() { + t.timer.Reset(t.interval) +} + +// Stop timer. +func (t *ConstantIntervalTimer) Stop() { + // drain timer + if !t.timer.Stop() { + select { + case <-t.timer.Chan(): + default: + } + } +} diff --git a/pp/go/storage/models.go b/pp/go/storage/models.go new file mode 100644 index 0000000000..b2261fdc4d --- /dev/null +++ b/pp/go/storage/models.go @@ -0,0 +1,62 @@ +package storage + +// // +// // MetricData +// // + +// // MetricData is an universal interface for blob protobuf data or batch [model.TimeSeries]. +// type MetricData interface { +// // Destroy incoming data. +// Destroy() +// } + +// // +// // IncomingData +// // + +// // IncomingData implements. +// type IncomingData struct { +// Hashdex cppbridge.ShardedData +// Data MetricData +// } + +// // Destroy IncomingData. +// func (i *IncomingData) Destroy() { +// i.Hashdex = nil +// if i.Data != nil { +// i.Data.Destroy() +// } +// } + +// // ShardedData return hashdex. +// func (i *IncomingData) ShardedData() cppbridge.ShardedData { +// return i.Hashdex +// } + +// // +// // HeadStatus +// // + +// // HeadStatus holds information about all shards. +// type HeadStatus struct { +// HeadStats HeadStats `json:"headStats"` +// SeriesCountByMetricName []HeadStat `json:"seriesCountByMetricName"` +// LabelValueCountByLabelName []HeadStat `json:"labelValueCountByLabelName"` +// MemoryInBytesByLabelName []HeadStat `json:"memoryInBytesByLabelName"` +// SeriesCountByLabelValuePair []HeadStat `json:"seriesCountByLabelValuePair"` +// } + +// // HeadStat holds the information about individual cardinality. +// type HeadStat struct { +// Name string `json:"name"` +// Value uint64 `json:"value"` +// } + +// // HeadStats has information about the head. +// type HeadStats struct { +// NumSeries uint64 `json:"numSeries"` +// NumLabelPairs int `json:"numLabelPairs"` +// ChunkCount int64 `json:"chunkCount"` +// MinTime int64 `json:"minTime"` +// MaxTime int64 `json:"maxTime"` +// } diff --git a/pp/go/storage/proxy.go b/pp/go/storage/proxy.go new file mode 100644 index 0000000000..5e265b4fc0 --- /dev/null +++ b/pp/go/storage/proxy.go @@ -0,0 +1,87 @@ +package storage + +import ( + "context" + "errors" + "time" + + "github.com/prometheus/prometheus/pp/go/storage/head/container" + "github.com/prometheus/prometheus/pp/go/storage/head/keeper" +) + +// +// Proxy +// + +// Proxy it proxies requests to the active [Head] and the keeper of old [Head]s. +type Proxy struct { + activeHeadContainer *container.Weighted[Head, *Head] + keeper *keeper.Keeper[Head, *Head] + onClose func(h *Head) error +} + +// NewProxy init new [Proxy]. +func NewProxy( + activeHeadContainer *container.Weighted[Head, *Head], + hKeeper *keeper.Keeper[Head, *Head], + onClose func(h *Head) error, +) *Proxy { + return &Proxy{ + activeHeadContainer: activeHeadContainer, + keeper: hKeeper, + onClose: onClose, + } +} + +// Add the [Head] to the [Keeper] if there is a free slot. +func (p *Proxy) Add(head *Head, createdAt time.Duration) error { + return p.keeper.Add(head, createdAt) +} + +// AddWithReplace the [Head] to the [Keeper] with replace if the createdAt is earlier. +func (p *Proxy) AddWithReplace(head *Head, createdAt time.Duration) error { + return p.keeper.AddWithReplace(head, createdAt) +} + +// Close closes [ActiveHeadContainer] and [Keeper] for the inability work with [Head]. +func (p *Proxy) Close() error { + ahErr := p.activeHeadContainer.Close() + + h := p.activeHeadContainer.Get() + onCloseErr := p.onClose(h) + headCloseErr := h.Close() + + keeperErr := p.keeper.Close() + + return errors.Join(ahErr, onCloseErr, headCloseErr, keeperErr) +} + +// Get the active [Head]. +func (p *Proxy) Get() *Head { + return p.activeHeadContainer.Get() +} + +// HasSlot returns the tru if there is a slot in the [Keeper]. +func (p *Proxy) HasSlot() bool { + return p.keeper.HasSlot() +} + +// Heads returns a slice of the [Head]s stored in the [Keeper]. +func (p *Proxy) Heads() []*Head { + return p.keeper.Heads() +} + +// Remove removes [Head]s from the [Keeper]. +func (p *Proxy) Remove(headsForRemove []*Head) { + p.keeper.Remove(headsForRemove) +} + +// Replace the active [Head] with a new [Head]. +func (p *Proxy) Replace(ctx context.Context, newHead *Head) error { + return p.activeHeadContainer.Replace(ctx, newHead) +} + +// With calls fn(h Head) on active [Head]. +func (p *Proxy) With(ctx context.Context, fn func(h *Head) error) error { + return p.activeHeadContainer.With(ctx, fn) +} diff --git a/pp/go/storage/querier/chunk_querier.go b/pp/go/storage/querier/chunk_querier.go new file mode 100644 index 0000000000..5a4b3b9049 --- /dev/null +++ b/pp/go/storage/querier/chunk_querier.go @@ -0,0 +1,162 @@ +package querier + +import ( + "context" + "errors" + + "github.com/prometheus/prometheus/model/labels" + "github.com/prometheus/prometheus/storage" + "github.com/prometheus/prometheus/util/annotations" + + "github.com/prometheus/prometheus/pp/go/cppbridge" + "github.com/prometheus/prometheus/pp/go/logger" + "github.com/prometheus/prometheus/pp/go/util/locker" +) + +const ( + // lssQueryChunkQuerySelector name of task. + lssQueryChunkQuerySelector = "lss_query_chunk_query_selector" + // lssLabelValuesChunkQuerier name of task. + lssLabelValuesChunkQuerier = "lss_label_values_chunk_querier" + // lssLabelNamesChunkQuerier name of task. + lssLabelNamesChunkQuerier = "lss_label_names_chunk_querier" + + // dsQueryChunkQuerier name of task. + dsQueryChunkQuerier = "data_storage_query_chunk_querier" +) + +// ChunkQuerier provides querying access over time series data of a fixed time range. +type ChunkQuerier[ + TTask Task, + TDataStorage DataStorage, + TLSS LSS, + TShard Shard[TDataStorage, TLSS], + THead Head[TTask, TDataStorage, TLSS, TShard], +] struct { + head THead + deduplicatorCtor deduplicatorCtor + mint int64 + maxt int64 + closer func() error +} + +// NewChunkQuerier init new [ChunkQuerier]. +func NewChunkQuerier[ + TTask Task, + TDataStorage DataStorage, + TLSS LSS, + TShard Shard[TDataStorage, TLSS], + THead Head[TTask, TDataStorage, TLSS, TShard], +]( + head THead, + deduplicatorCtor deduplicatorCtor, + mint, maxt int64, + closer func() error, +) *ChunkQuerier[TTask, TDataStorage, TLSS, TShard, THead] { + return &ChunkQuerier[TTask, TDataStorage, TLSS, TShard, THead]{ + head: head, + deduplicatorCtor: deduplicatorCtor, + mint: mint, + maxt: maxt, + closer: closer, + } +} + +// Close [ChunkQuerier] if need. +// +//revive:disable-next-line:confusing-naming // other type of querier. +func (q *ChunkQuerier[TTask, TDataStorage, TLSS, TShard, THead]) Close() error { + if q.closer != nil { + err := q.closer() + q.closer = nil + return err + } + + return nil +} + +// LabelNames returns label values present in the head for the specific label name. +// +//revive:disable-next-line:confusing-naming // other type of querier. +func (q *ChunkQuerier[TTask, TDataStorage, TLSS, TShard, THead]) LabelNames( + ctx context.Context, + hints *storage.LabelHints, + matchers ...*labels.Matcher, +) ([]string, annotations.Annotations, error) { + return queryLabelNames( + ctx, + q.head, + q.deduplicatorCtor, + nil, + lssLabelNamesChunkQuerier, + hints, + matchers..., + ) +} + +// LabelValues returns label values present in the head for the specific label name +// that are within the time range mint to maxt. If matchers are specified the returned +// result set is reduced to label values of metrics matching the matchers. +// +//revive:disable:confusing-naming // other type of querier. +func (q *ChunkQuerier[TTask, TDataStorage, TLSS, TShard, THead]) LabelValues( + ctx context.Context, + name string, + hints *storage.LabelHints, + matchers ...*labels.Matcher, +) ([]string, annotations.Annotations, error) { + return queryLabelValues( + ctx, + name, + q.head, + q.deduplicatorCtor, + nil, + lssLabelValuesChunkQuerier, + hints, + matchers..., + ) +} + +// Select returns a chunk set of series that matches the given label matchers. +// +//revive:disable-next-line:confusing-naming // other type of querier. +func (q *ChunkQuerier[TTask, TDataStorage, TLSS, TShard, THead]) Select( + ctx context.Context, + _ bool, + _ *storage.SelectHints, + matchers ...*labels.Matcher, +) storage.ChunkSeriesSet { + release, err := q.head.AcquireQuery(ctx) + if err != nil { + if errors.Is(err, locker.ErrSemaphoreClosed) { + return &EmptyChunkSeriesSet{} + } + + logger.Warnf("[ChunkQuerier]: Select failed: %s", err) + return storage.ErrChunkSeriesSet(err) + } + defer release() + + lssQueryResults, snapshots, err := queryLss(lssQueryChunkQuerySelector, q.head, matchers) + if err != nil { + logger.Warnf("[ChunkQuerier]: failed: %s", err) + return storage.ErrChunkSeriesSet(err) + } + + serializedChunksShards := queryDataStorage(dsQueryChunkQuerier, q.head, lssQueryResults, q.mint, q.maxt) + chunkSeriesSets := make([]storage.ChunkSeriesSet, q.head.NumberOfShards()) + for shardID, serializedChunks := range serializedChunksShards { + if serializedChunks == nil { + chunkSeriesSets[shardID] = &EmptyChunkSeriesSet{} + continue + } + + chunkSeriesSets[shardID] = NewChunkSeriesSet( + lssQueryResults[shardID], + snapshots[shardID], + cppbridge.NewSerializedChunkRecoder(serializedChunks, cppbridge.TimeInterval{MinT: q.mint, MaxT: q.maxt}), + ) + } + + return storage.NewMergeChunkSeriesSet(chunkSeriesSets, storage.NewConcatenatingChunkSeriesMerger()) +} diff --git a/pp/go/storage/querier/chunk_series.go b/pp/go/storage/querier/chunk_series.go new file mode 100644 index 0000000000..231da34ec9 --- /dev/null +++ b/pp/go/storage/querier/chunk_series.go @@ -0,0 +1,225 @@ +package querier + +import ( + "github.com/prometheus/prometheus/model/labels" + "github.com/prometheus/prometheus/pp/go/cppbridge" + "github.com/prometheus/prometheus/storage" + "github.com/prometheus/prometheus/tsdb/chunkenc" + "github.com/prometheus/prometheus/tsdb/chunks" + "github.com/prometheus/prometheus/util/annotations" +) + +// +// ChunkSeriesSet +// + +// ChunkSeriesSet contains a set of chunked series. +type ChunkSeriesSet struct { + lssQueryResult *cppbridge.LSSQueryResult + labelSetSnapshot *cppbridge.LabelSetSnapshot + chunkRecoder *cppbridge.ChunkRecoder + + index int + lastRecodedChunk *cppbridge.RecodedChunk + chunkSeries *ChunkSeries + + recoderIsExhausted bool +} + +// NewChunkSeriesSet init new [ChunkSeriesSet]. +func NewChunkSeriesSet( + lssQueryResult *cppbridge.LSSQueryResult, + labelSetSnapshot *cppbridge.LabelSetSnapshot, + chunkRecoder *cppbridge.ChunkRecoder, +) *ChunkSeriesSet { + return &ChunkSeriesSet{ + lssQueryResult: lssQueryResult, + labelSetSnapshot: labelSetSnapshot, + chunkRecoder: chunkRecoder, + } +} + +// At returns full chunk series. Returned series should be iterable even after Next is called. +func (css *ChunkSeriesSet) At() storage.ChunkSeries { + return css.chunkSeries +} + +// Err returns the current error - always nil. +func (*ChunkSeriesSet) Err() error { + return nil +} + +// Next advances the iterator by one and returns false if there are no more values. +func (css *ChunkSeriesSet) Next() bool { + if css.lastRecodedChunk == nil && !css.nextChunk() { + return false + } + + seriesID := css.lastRecodedChunk.SeriesId + recodedChunks := make([]cppbridge.RecodedChunk, 1) + recodedChunks[0] = *css.lastRecodedChunk + + nextSeriesIDFound := false + for css.nextChunk() { + if css.lastRecodedChunk.SeriesId != seriesID { + nextSeriesIDFound = true + break + } + recodedChunks = append(recodedChunks, *css.lastRecodedChunk) + } + + if !nextSeriesIDFound && css.recoderIsExhausted { + css.lastRecodedChunk = nil + } + + var ( + lsID uint32 + lsLength uint16 + ) + + for { + if css.index >= css.lssQueryResult.Len() { + return false + } + + lsID, lsLength = css.lssQueryResult.GetByIndex(css.index) + + if lsID == seriesID { + break + } + + css.index++ + } + + css.chunkSeries = &ChunkSeries{ + labelSet: labels.NewLabelsWithLSS(css.labelSetSnapshot, lsID, lsLength), + recodedChunks: recodedChunks, + } + + return true +} + +// Warnings a collection of warnings for the whole set - always nil. +func (*ChunkSeriesSet) Warnings() annotations.Annotations { + return nil +} + +// nextChunk advances the iterator by one and returns false if there are no more values. +func (css *ChunkSeriesSet) nextChunk() bool { + if css.recoderIsExhausted { + return false + } + + lastRecodedChunk := css.chunkRecoder.RecodeNextChunk() + css.recoderIsExhausted = !lastRecodedChunk.HasMoreData + chunkData := make([]byte, len(lastRecodedChunk.ChunkData)) + copy(chunkData, lastRecodedChunk.ChunkData) + lastRecodedChunk.ChunkData = chunkData + css.lastRecodedChunk = &lastRecodedChunk + + return true +} + +// +// ChunkSeries +// + +// ChunkSeries exposes a single time series and allows iterating over chunks. +type ChunkSeries struct { + labelSet labels.Labels + recodedChunks []cppbridge.RecodedChunk +} + +// Iterator returns an iterator that iterates over potentially overlapping +// chunks of the series, sorted by min time. +func (cs *ChunkSeries) Iterator(iterator chunks.Iterator) chunks.Iterator { + if ci, ok := iterator.(*ChunkSeriesChunksIterator); ok { + ci.ResetTo(cs.recodedChunks) + return ci + } + + return NewChunkSeriesChunksIterator(cs.recodedChunks) +} + +// Labels returns the complete set of labels. For series it means all labels identifying the series. +func (cs *ChunkSeries) Labels() labels.Labels { + return cs.labelSet +} + +// +// ChunkSeriesChunksIterator +// + +// ChunkSeriesChunksIterator iterator that iterates over chunks of the series, sorted by min time. +type ChunkSeriesChunksIterator struct { + idx int + recodedChunks []cppbridge.RecodedChunk + xorChunk *chunkenc.XORChunk + meta chunks.Meta +} + +// NewChunkSeriesChunksIterator init new [ChunkSeriesChunksIterator]. +func NewChunkSeriesChunksIterator(recodedChunks []cppbridge.RecodedChunk) *ChunkSeriesChunksIterator { + return &ChunkSeriesChunksIterator{ + recodedChunks: recodedChunks, + xorChunk: chunkenc.NewXORChunk(), + } +} + +// At returns the current meta. +func (ci *ChunkSeriesChunksIterator) At() chunks.Meta { + return ci.meta +} + +// Err returns the current error - always nil. +func (*ChunkSeriesChunksIterator) Err() error { + return nil +} + +// Next advances the iterator by one. +func (ci *ChunkSeriesChunksIterator) Next() bool { + if ci.idx >= len(ci.recodedChunks) { + return false + } + + ci.meta.MinTime = ci.recodedChunks[ci.idx].MinT + ci.meta.MaxTime = ci.recodedChunks[ci.idx].MaxT + ci.xorChunk.Reset(ci.recodedChunks[ci.idx].ChunkData) + ci.meta.Chunk = ci.xorChunk + ci.idx++ + + return true +} + +// ResetTo reset [ChunkSeriesChunksIterator] to recodedChunks. +func (ci *ChunkSeriesChunksIterator) ResetTo(recodedChunks []cppbridge.RecodedChunk) { + ci.idx = 0 + ci.recodedChunks = recodedChunks +} + +// +// EmptyChunkSeriesSet +// + +// EmptyChunkSeriesSet implementation [ChunkSeriesSet], do nothing. +type EmptyChunkSeriesSet struct{} + +// At implementation [ChunkSeriesSet], do nothing. +func (EmptyChunkSeriesSet) At() storage.ChunkSeries { + return nil +} + +// Err implementation [ChunkSeriesSet], do nothing. +func (EmptyChunkSeriesSet) Err() error { + return nil +} + +// Next implementation [ChunkSeriesSet], do nothing. +func (EmptyChunkSeriesSet) Next() bool { + return false +} + +// Warnings implementation [ChunkSeriesSet], do nothing. +func (EmptyChunkSeriesSet) Warnings() annotations.Annotations { + return nil +} diff --git a/pp/go/storage/querier/dedup.go b/pp/go/storage/querier/dedup.go new file mode 100644 index 0000000000..3ba2714864 --- /dev/null +++ b/pp/go/storage/querier/dedup.go @@ -0,0 +1,45 @@ +package querier + +import ( + "runtime" + "strings" + "sync/atomic" + + "github.com/prometheus/prometheus/pp/go/cppbridge" +) + +// NoOpShardedDeduplicator container for accumulating values without deduplication. +type NoOpShardedDeduplicator struct { + shardedValues [][]string + snapshots []*cppbridge.LabelSetSnapshot + count uint64 +} + +// NewNoOpShardedDeduplicator int new [NoOpShardedDeduplicator]. +func NewNoOpShardedDeduplicator(numberOfShards uint16) Deduplicator { + return &NoOpShardedDeduplicator{ + shardedValues: make([][]string, numberOfShards), + snapshots: make([]*cppbridge.LabelSetSnapshot, numberOfShards), + } +} + +// Add values to deduplicator by shard ID. +func (d *NoOpShardedDeduplicator) Add(shardID uint16, snapshot *cppbridge.LabelSetSnapshot, values []string) { + d.shardedValues[shardID] = make([]string, len(values)) + n := copy(d.shardedValues[shardID], values) + atomic.AddUint64(&d.count, uint64(n)) // #nosec G115 // no overflow + d.snapshots[shardID] = snapshot +} + +// Values returns collected values. +func (d *NoOpShardedDeduplicator) Values() []string { + values := make([]string, 0, d.count) + for _, shardedValues := range d.shardedValues { + for _, v := range shardedValues { + values = append(values, strings.Clone(v)) + } + } + runtime.KeepAlive(d.snapshots) + + return values +} diff --git a/pp/go/storage/querier/interface.go b/pp/go/storage/querier/interface.go new file mode 100644 index 0000000000..5cb1fcf6d0 --- /dev/null +++ b/pp/go/storage/querier/interface.go @@ -0,0 +1,137 @@ +package querier + +import ( + "context" + + "github.com/prometheus/prometheus/pp/go/cppbridge" + "github.com/prometheus/prometheus/pp/go/model" + "github.com/prometheus/prometheus/pp/go/storage/head/shard" +) + +// +// Deduplicator +// + +// Deduplicator accumulates and deduplicates incoming values. +type Deduplicator interface { + // Add values to deduplicator by shard ID. + Add(shardID uint16, snapshot *cppbridge.LabelSetSnapshot, values []string) + + // Values returns collected values. + Values() []string +} + +// deduplicatorCtor constructor [Deduplicator]. +type deduplicatorCtor func(numberOfShards uint16) Deduplicator + +// +// GenericTask +// + +// Task the minimum required task [Generic] implementation. +type Task interface { + // Wait for the task to complete on all shards. + Wait() error +} + +// +// DataStorage +// + +// DataStorage the minimum required [DataStorage] implementation. +type DataStorage interface { + // InstantQuery returns samples for instant query from data storage. + InstantQuery( + maxt, valueNotFoundTimestampValue int64, + ids []uint32, + ) ([]cppbridge.Sample, cppbridge.DataStorageQueryResult) + + // QueryDataStorage returns serialized chunks from data storage. + Query( + query cppbridge.HeadDataStorageQuery, + ) (*cppbridge.HeadDataStorageSerializedChunks, cppbridge.DataStorageQueryResult) + + // WithRLock calls fn on raw [cppbridge.HeadDataStorage] with read lock. + WithRLock(fn func(ds *cppbridge.HeadDataStorage) error) error +} + +// +// LSS +// + +// LSS the minimum required [LSS] implementation. +type LSS interface { + // QueryLabelNames returns all the unique label names present in lss in sorted order. + QueryLabelNames( + shardID uint16, + matchers []model.LabelMatcher, + dedupAdd func(shardID uint16, snapshot *cppbridge.LabelSetSnapshot, values []string), + ) error + + // QueryLabelValues query labels values to lss and add values to + // the dedup-container that matches the given label matchers. + QueryLabelValues( + shardID uint16, + name string, + matchers []model.LabelMatcher, + dedupAdd func(shardID uint16, snapshot *cppbridge.LabelSetSnapshot, values []string), + ) error + + // QuerySelector returns a created selector that matches the given label matchers. + QuerySelector(shardID uint16, matchers []model.LabelMatcher) (uintptr, *cppbridge.LabelSetSnapshot, error) + + // WithRLock calls fn on raws [cppbridge.LabelSetStorage] with read lock. + WithRLock(fn func(target, input *cppbridge.LabelSetStorage) error) error +} + +// +// Shard +// + +// Shard the minimum required head [Shard] implementation. +type Shard[TDataStorage DataStorage, TLSS LSS] interface { + // DataStorage returns shard [DataStorage]. + DataStorage() TDataStorage + + // LSS returns shard labelset storage [LSS]. + LSS() TLSS + + // ShardID returns the shard ID. + ShardID() uint16 + + LoadAndQuerySeriesData() error + + LoadAndQuerySeriesDataTask() *shard.LoadAndQuerySeriesDataTask +} + +// +// Head +// + +// Head the minimum required [Head] implementation. +type Head[ + TGenericTask Task, + TDataStorage DataStorage, + TLSS LSS, + TShard Shard[TDataStorage, TLSS], +] interface { + // AcquireQuery acquires the [Head] semaphore with a weight of 1, + // blocking until resources are available or ctx is done. + // On success, returns nil. On failure, returns ctx.Err() and leaves the semaphore unchanged. + AcquireQuery(ctx context.Context) (release func(), err error) + + // CreateTask create a task for operations on the [Head] shards. + CreateTask(taskName string, shardFn func(s TShard) error) TGenericTask + + // Enqueue the task to be executed on shards [Head]. + Enqueue(t TGenericTask) + + // EnqueueOnShard the task to be executed on head on specific shard. + EnqueueOnShard(t TGenericTask, shardID uint16) + + // NumberOfShards returns current number of shards in to [Head]. + NumberOfShards() uint16 + + // IsReadOnly returns true if the [Head] has switched to read-only. + IsReadOnly() bool +} diff --git a/pp/go/storage/querier/load_and_query_waiter.go b/pp/go/storage/querier/load_and_query_waiter.go new file mode 100644 index 0000000000..a11c54a366 --- /dev/null +++ b/pp/go/storage/querier/load_and_query_waiter.go @@ -0,0 +1,52 @@ +package querier + +import ( + "github.com/prometheus/prometheus/pp/go/storage/head/shard" + "github.com/prometheus/prometheus/pp/go/storage/head/task" +) + +const ( + // dsLoadAndQuerySeriesData + dsLoadAndQuerySeriesData = "data_storage_load_and_query_series_data" +) + +// LoadAndQueryWaiter is a waiter for the load and query series data task. +type LoadAndQueryWaiter[ + TTask Task, + TDataStorage DataStorage, + TLSS LSS, + TShard Shard[TDataStorage, TLSS], + THead Head[TTask, TDataStorage, TLSS, TShard], +] struct { + waiter task.Waiter[shard.Task] + head THead +} + +// NewLoadAndQueryWaiter creates a new [LoadAndQueryWaiter]. +func NewLoadAndQueryWaiter[ + TTask Task, + TDataStorage DataStorage, + TLSS LSS, + TShard Shard[TDataStorage, TLSS], + THead Head[TTask, TDataStorage, TLSS, TShard], +](head THead) LoadAndQueryWaiter[TTask, TDataStorage, TLSS, TShard, THead] { + return LoadAndQueryWaiter[TTask, TDataStorage, TLSS, TShard, THead]{ + head: head, + } +} + +// Add adds a querier to the load and query series data task. +func (l *LoadAndQueryWaiter[TTask, TDataStorage, TLSS, TShard, THead]) Add(s TShard, querier uintptr) { + l.waiter.Add(s.LoadAndQuerySeriesDataTask().Add(querier, func() shard.Task { + t := l.head.CreateTask(dsLoadAndQuerySeriesData, func(s TShard) error { + return s.LoadAndQuerySeriesData() + }) + l.head.EnqueueOnShard(t, s.ShardID()) + return t + })) +} + +// Wait waits for the load and query series data task to complete. +func (l *LoadAndQueryWaiter[TTask, TDataStorage, TLSS, TShard, THead]) Wait() error { + return l.waiter.Wait() +} diff --git a/pp/go/storage/querier/metrics.go b/pp/go/storage/querier/metrics.go new file mode 100644 index 0000000000..71e968890a --- /dev/null +++ b/pp/go/storage/querier/metrics.go @@ -0,0 +1,67 @@ +package querier + +import ( + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/prometheus/pp/go/util" +) + +const ( + // QueryableAppenderSource metrics source for Appender. + QueryableAppenderSource = "queryable_appender" + // QueryableStorageSource metrics source for Storage. + QueryableStorageSource = "queryable_storage" +) + +// Metrics for [Querier]. +type Metrics struct { + LabelNamesDuration prometheus.Histogram + LabelValuesDuration prometheus.Histogram + SelectDuration *prometheus.HistogramVec +} + +// NewMetrics init new [Metrics]. +func NewMetrics(registerer prometheus.Registerer, source string) *Metrics { + factory := util.NewUnconflictRegisterer(registerer) + return &Metrics{ + LabelNamesDuration: factory.NewHistogram( + prometheus.HistogramOpts{ + Name: "prompp_querier_query_label_names_duration", + Help: "Label names query from head duration in microseconds", + Buckets: []float64{ + 50, 100, 250, 500, 750, + 1000, 2500, 5000, 7500, + 10000, 25000, 50000, 75000, + 100000, 500000, + }, + ConstLabels: prometheus.Labels{"source": source}, + }, + ), + LabelValuesDuration: factory.NewHistogram( + prometheus.HistogramOpts{ + Name: "prompp_querier_query_label_values_duration", + Help: "Label values query from head duration in microseconds", + Buckets: []float64{ + 50, 100, 250, 500, 750, + 1000, 2500, 5000, 7500, + 10000, 25000, 50000, 75000, + 100000, 500000, + }, + ConstLabels: prometheus.Labels{"source": source}, + }, + ), + SelectDuration: factory.NewHistogramVec( + prometheus.HistogramOpts{ + Name: "prompp_querier_query_select_duration", + Help: "Select query from head duration in microseconds", + Buckets: []float64{ + 50, 100, 250, 500, 750, + 1000, 2500, 5000, 7500, + 10000, 25000, 50000, 75000, + 100000, 500000, + }, + ConstLabels: prometheus.Labels{"source": source}, + }, + []string{"query_type"}, + ), + } +} diff --git a/pp/go/storage/querier/multi.go b/pp/go/storage/querier/multi.go new file mode 100644 index 0000000000..db879598b9 --- /dev/null +++ b/pp/go/storage/querier/multi.go @@ -0,0 +1,170 @@ +package querier + +import ( + "context" + "errors" + "sort" + "sync" + + "github.com/prometheus/prometheus/model/labels" + "github.com/prometheus/prometheus/storage" + "github.com/prometheus/prometheus/util/annotations" +) + +// +// MultiQuerier +// + +// MultiQuerier querier which makes requests to all queriers from the created list and merges the received results. +type MultiQuerier struct { + queriers []storage.Querier + closer func() error +} + +// NewMultiQuerier init new [MultiQuerier]. +func NewMultiQuerier(queriers []storage.Querier, closer func() error) *MultiQuerier { + qs := make([]storage.Querier, 0, len(queriers)) + for _, q := range queriers { + if rawQ, ok := q.(*MultiQuerier); ok { + qs = append(qs, rawQ.queriers...) + continue + } + + qs = append(qs, q) + } + + return &MultiQuerier{ + queriers: qs, + closer: closer, + } +} + +// Close closes all [storage.Querier]s if need. +func (q *MultiQuerier) Close() (err error) { + for _, querier := range q.queriers { + err = errors.Join(err, querier.Close()) + } + + if q.closer != nil { + err = errors.Join(err, q.closer()) + } + + return err +} + +// LabelNames returns label values present in the head for the specific label name from all [storage.Querier]s. +func (q *MultiQuerier) LabelNames( + ctx context.Context, + hints *storage.LabelHints, + matchers ...*labels.Matcher, +) ([]string, annotations.Annotations, error) { + if len(q.queriers) == 1 { + return q.queriers[0].LabelNames(ctx, hints, matchers...) + } + + labelNamesResults := make([][]string, len(q.queriers)) + annotationResults := make([]annotations.Annotations, len(q.queriers)) + errs := make([]error, len(q.queriers)) + + wg := &sync.WaitGroup{} + for index, querier := range q.queriers { + wg.Add(1) + go func(index int, querier storage.Querier) { + defer wg.Done() + labelNamesResults[index], annotationResults[index], errs[index] = querier.LabelNames( + ctx, + hints, + matchers..., + ) + }(index, querier) + } + + wg.Wait() + + labelNames := DeduplicateAndSortStringSlices(labelNamesResults...) + + return labelNames, nil, errors.Join(errs...) +} + +// LabelValues returns label values present in the head for the specific label name +// that are within the time range mint to maxt from all [storage.Querier]s. If matchers are specified the returned +// result set is reduced to label values of metrics matching the matchers. +func (q *MultiQuerier) LabelValues( + ctx context.Context, + name string, + hints *storage.LabelHints, + matchers ...*labels.Matcher, +) ([]string, annotations.Annotations, error) { + if len(q.queriers) == 1 { + return q.queriers[0].LabelValues(ctx, name, hints, matchers...) + } + + labelValuesResults := make([][]string, len(q.queriers)) + annotationResults := make([]annotations.Annotations, len(q.queriers)) + errs := make([]error, len(q.queriers)) + + wg := &sync.WaitGroup{} + for index, querier := range q.queriers { + wg.Add(1) + go func(index int, querier storage.Querier) { + defer wg.Done() + labelValuesResults[index], annotationResults[index], errs[index] = querier.LabelValues( + ctx, + name, + hints, + matchers..., + ) + }(index, querier) + } + + wg.Wait() + + labelValues := DeduplicateAndSortStringSlices(labelValuesResults...) + return labelValues, nil, errors.Join(errs...) +} + +// Select returns a set of series that matches the given label matchers from all [storage.Querier]s. +func (q *MultiQuerier) Select( + ctx context.Context, + sortSeries bool, + hints *storage.SelectHints, + matchers ...*labels.Matcher, +) storage.SeriesSet { + if len(q.queriers) == 1 { + return q.queriers[0].Select(ctx, sortSeries, hints, matchers...) + } + + seriesSets := make([]storage.SeriesSet, len(q.queriers)) + wg := &sync.WaitGroup{} + + for index, querier := range q.queriers { + wg.Add(1) + go func(index int, querier storage.Querier) { + defer wg.Done() + seriesSets[index] = querier.Select(ctx, sortSeries, hints, matchers...) + }(index, querier) + } + + wg.Wait() + + return storage.NewMergeSeriesSet(seriesSets, storage.ChainedSeriesMerge) +} + +// DeduplicateAndSortStringSlices merge, deduplicate, and sort rows in slices +// and return a single sorted slice of unique rows. +func DeduplicateAndSortStringSlices(stringSlices ...[]string) []string { + dedup := make(map[string]struct{}) + for _, stringSlice := range stringSlices { + for _, value := range stringSlice { + dedup[value] = struct{}{} + } + } + + result := make([]string, 0, len(dedup)) + for value := range dedup { + result = append(result, value) + } + + sort.Strings(result) + return result +} diff --git a/pp/go/storage/querier/querier.go b/pp/go/storage/querier/querier.go new file mode 100644 index 0000000000..8b83a910ec --- /dev/null +++ b/pp/go/storage/querier/querier.go @@ -0,0 +1,573 @@ +package querier + +import ( + "context" + "errors" + "fmt" + "sort" + "time" + + "github.com/prometheus/client_golang/prometheus" + + "github.com/prometheus/prometheus/model/labels" + "github.com/prometheus/prometheus/storage" + "github.com/prometheus/prometheus/util/annotations" + + "github.com/prometheus/prometheus/pp/go/cppbridge" + "github.com/prometheus/prometheus/pp/go/logger" + "github.com/prometheus/prometheus/pp/go/model" + "github.com/prometheus/prometheus/pp/go/util/locker" +) + +const ( + // lssQueryInstantQuerySelector name of task. + lssQueryInstantQuerySelector = "lss_query_instant_query_selector" + // lssQueryRangeQuerySelector name of task. + lssQueryRangeQuerySelector = "lss_query_range_query_selector" + // lssLabelValuesQuerier name of task. + lssLabelValuesQuerier = "lss_label_values_querier" + // lssLabelNamesQuerier name of task. + lssLabelNamesQuerier = "lss_label_names_querier" + + // dsQueryInstantQuerier name of task. + dsQueryInstantQuerier = "data_storage_query_instant_querier" + // dsQueryRangeQuerier name of task. + dsQueryRangeQuerier = "data_storage_query_range_querier" + + // DefaultInstantQueryValueNotFoundTimestampValue default value for not found timestamp value. + DefaultInstantQueryValueNotFoundTimestampValue int64 = 0 +) + +// +// Querier +// + +// Querier provides querying access over time series data of a fixed time range. +type Querier[ + TTask Task, + TDataStorage DataStorage, + TLSS LSS, + TShard Shard[TDataStorage, TLSS], + THead Head[TTask, TDataStorage, TLSS, TShard], +] struct { + mint int64 + maxt int64 + head THead + deduplicatorCtor deduplicatorCtor + closer func() error + metrics *Metrics +} + +// NewQuerier init new [Querier]. +func NewQuerier[ + TTask Task, + TDataStorage DataStorage, + TLSS LSS, + TShard Shard[TDataStorage, TLSS], + THead Head[TTask, TDataStorage, TLSS, TShard], +]( + head THead, + deduplicatorCtor deduplicatorCtor, + mint, maxt int64, + closer func() error, + metrics *Metrics, +) *Querier[TTask, TDataStorage, TLSS, TShard, THead] { + return &Querier[TTask, TDataStorage, TLSS, TShard, THead]{ + mint: mint, + maxt: maxt, + head: head, + deduplicatorCtor: deduplicatorCtor, + closer: closer, + metrics: metrics, + } +} + +// Close [Querier] if need. +// +//revive:disable-next-line:confusing-naming // other type of querier. +func (q *Querier[TTask, TDataStorage, TLSS, TShard, THead]) Close() error { + if q.closer != nil { + return q.closer() + } + + return nil +} + +// LabelNames returns label values present in the head for the specific label name. +// +//revive:disable-next-line:confusing-naming // other type of querier. +func (q *Querier[TTask, TDataStorage, TLSS, TShard, THead]) LabelNames( + ctx context.Context, + hints *storage.LabelHints, + matchers ...*labels.Matcher, +) ([]string, annotations.Annotations, error) { + return queryLabelNames( + ctx, + q.head, + q.deduplicatorCtor, + q.metrics, + lssLabelNamesQuerier, + hints, + matchers..., + ) +} + +// LabelValues returns label values present in the head for the specific label name +// that are within the time range mint to maxt. If matchers are specified the returned +// result set is reduced to label values of metrics matching the matchers. +// +//revive:disable-next-line:confusing-naming // other type of querier. +func (q *Querier[TTask, TDataStorage, TLSS, TShard, THead]) LabelValues( + ctx context.Context, + name string, + hints *storage.LabelHints, + matchers ...*labels.Matcher, +) ([]string, annotations.Annotations, error) { + return queryLabelValues( + ctx, + name, + q.head, + q.deduplicatorCtor, + q.metrics, + lssLabelValuesQuerier, + hints, + matchers..., + ) +} + +// Select returns a set of series that matches the given label matchers. +// +//revive:disable-next-line:confusing-naming // other type of querier. +func (q *Querier[TTask, TDataStorage, TLSS, TShard, THead]) Select( + ctx context.Context, + sortSeries bool, + hints *storage.SelectHints, + matchers ...*labels.Matcher, +) storage.SeriesSet { + if q.mint == q.maxt { + return q.selectInstant(ctx, sortSeries, hints, matchers...) + } + return q.selectRange(ctx, sortSeries, hints, matchers...) +} + +// selectInstant returns a instant set of series that matches the given label matchers. +// +//revive:disable-next-line:function-length long but readable. +func (q *Querier[TTask, TDataStorage, TLSS, TShard, THead]) selectInstant( + ctx context.Context, + _ bool, + _ *storage.SelectHints, + matchers ...*labels.Matcher, +) storage.SeriesSet { + start := time.Now() + + release, err := q.head.AcquireQuery(ctx) + if err != nil { + if errors.Is(err, locker.ErrSemaphoreClosed) { + return &SeriesSet{} + } + + logger.Warnf("[QUERIER]: select instant failed on the capture of the read lock query: %s", err) + return storage.ErrSeriesSet(err) + } + defer release() + + defer func() { + if q.metrics != nil { + q.metrics.SelectDuration.With( + prometheus.Labels{"query_type": "instant"}, + ).Observe(float64(time.Since(start).Microseconds())) + } + }() + + lssQueryResults, snapshots, err := queryLss(lssQueryInstantQuerySelector, q.head, matchers) + if err != nil { + logger.Warnf("[QUERIER]: failed to instant: %s", err) + return storage.ErrSeriesSet(err) + } + + valueNotFoundTimestampValue := DefaultInstantQueryValueNotFoundTimestampValue + if q.mint <= valueNotFoundTimestampValue { + valueNotFoundTimestampValue = q.mint - 1 + } + + numberOfShards := q.head.NumberOfShards() + seriesSets := make([]storage.SeriesSet, numberOfShards) + loadAndQueryWaiter := NewLoadAndQueryWaiter[TTask, TDataStorage, TLSS, TShard, THead](q.head) + tDataStorageQuery := q.head.CreateTask( + dsQueryInstantQuerier, + func(s TShard) error { + shardID := s.ShardID() + lssQueryResult := lssQueryResults[shardID] + if lssQueryResult == nil { + seriesSets[shardID] = &SeriesSet{} + return nil + } + + samples, result := s.DataStorage().InstantQuery(q.maxt, valueNotFoundTimestampValue, lssQueryResult.IDs()) + if result.Status == cppbridge.DataStorageQueryStatusNeedDataLoad { + loadAndQueryWaiter.Add(s, result.Querier) + } + + seriesSets[shardID] = NewInstantSeriesSet( + lssQueryResult, + snapshots[shardID], + valueNotFoundTimestampValue, + samples, + ) + + return nil + }, + ) + q.head.Enqueue(tDataStorageQuery) + _ = tDataStorageQuery.Wait() + + if err := loadAndQueryWaiter.Wait(); err != nil { + SendUnrecoverableError(err) + return storage.ErrSeriesSet(err) + } + + return storage.NewMergeSeriesSet(seriesSets, storage.ChainedSeriesMerge) +} + +// selectRange returns a range set of series that matches the given label matchers. +func (q *Querier[TTask, TDataStorage, TLSS, TShard, THead]) selectRange( + ctx context.Context, + _ bool, + _ *storage.SelectHints, + matchers ...*labels.Matcher, +) storage.SeriesSet { + start := time.Now() + + release, err := q.head.AcquireQuery(ctx) + if err != nil { + if errors.Is(err, locker.ErrSemaphoreClosed) { + return &SeriesSet{} + } + + logger.Warnf("[QUERIER]: select range failed on the capture of the read lock query: %s", err) + return storage.ErrSeriesSet(err) + } + defer release() + + defer func() { + if q.metrics != nil { + q.metrics.SelectDuration.With( + prometheus.Labels{"query_type": "range"}, + ).Observe(float64(time.Since(start).Microseconds())) + } + }() + + lssQueryResults, snapshots, err := queryLss(lssQueryRangeQuerySelector, q.head, matchers) + if err != nil { + logger.Warnf("[QUERIER]: failed to range: %s", err) + return storage.ErrSeriesSet(err) + } + + serializedChunksShards := queryDataStorage(dsQueryRangeQuerier, q.head, lssQueryResults, q.mint, q.maxt) + seriesSets := make([]storage.SeriesSet, q.head.NumberOfShards()) + for shardID, serializedChunksShard := range serializedChunksShards { + if serializedChunksShard == nil { + seriesSets[shardID] = &SeriesSet{} + continue + } + + seriesSets[shardID] = &SeriesSet{ + mint: q.mint, + maxt: q.maxt, + deserializer: cppbridge.NewHeadDataStorageDeserializer(serializedChunksShard), + chunksIndex: serializedChunksShard.MakeIndex(), + serializedChunks: serializedChunksShard, + lssQueryResult: lssQueryResults[shardID], + labelSetSnapshot: snapshots[shardID], + } + } + + return storage.NewMergeSeriesSet(seriesSets, storage.ChainedSeriesMerge) +} + +// convertPrometheusMatchersToPPMatchers converts prometheus matchers to pp matchers. +func convertPrometheusMatchersToPPMatchers(matchers ...*labels.Matcher) []model.LabelMatcher { + promppMatchers := make([]model.LabelMatcher, 0, len(matchers)) + for _, matcher := range matchers { + promppMatchers = append(promppMatchers, model.LabelMatcher{ + Name: matcher.Name, + Value: matcher.Value, + MatcherType: uint8(matcher.Type), // #nosec G115 // no overflow + }) + } + + return promppMatchers +} + +// queryDataStorage returns serialized chunks from data storage for each shard. +func queryDataStorage[ + TTask Task, + TDataStorage DataStorage, + TLSS LSS, + TShard Shard[TDataStorage, TLSS], + THead Head[TTask, TDataStorage, TLSS, TShard], +]( + taskName string, + head THead, + lssQueryResults []*cppbridge.LSSQueryResult, + mint, maxt int64, +) []*cppbridge.HeadDataStorageSerializedChunks { + serializedChunksShards := make([]*cppbridge.HeadDataStorageSerializedChunks, head.NumberOfShards()) + loadAndQueryWaiter := NewLoadAndQueryWaiter[TTask, TDataStorage, TLSS, TShard, THead](head) + tDataStorageQuery := head.CreateTask( + taskName, + func(s TShard) error { + shardID := s.ShardID() + lssQueryResult := lssQueryResults[shardID] + if lssQueryResult == nil { + return nil + } + + var result cppbridge.DataStorageQueryResult + serializedChunksShards[shardID], result = s.DataStorage().Query(cppbridge.HeadDataStorageQuery{ + StartTimestampMs: mint, + EndTimestampMs: maxt, + LabelSetIDs: lssQueryResult.IDs(), + }) + if result.Status == cppbridge.DataStorageQueryStatusNeedDataLoad { + loadAndQueryWaiter.Add(s, result.Querier) + } + + return nil + }, + ) + head.Enqueue(tDataStorageQuery) + _ = tDataStorageQuery.Wait() + + if err := loadAndQueryWaiter.Wait(); err != nil { + SendUnrecoverableError(err) + return nil + } + + return serializedChunksShards +} + +// queryLabelValues returns label values present in the head for the specific label name. +func queryLabelNames[ + TTask Task, + TDataStorage DataStorage, + TLSS LSS, + TShard Shard[TDataStorage, TLSS], + THead Head[TTask, TDataStorage, TLSS, TShard], +]( + ctx context.Context, + head THead, + deduplicatorCtor deduplicatorCtor, + metrics *Metrics, + taskName string, + hints *storage.LabelHints, + matchers ...*labels.Matcher, +) ([]string, annotations.Annotations, error) { + start := time.Now() + + anns := *annotations.New() + release, err := head.AcquireQuery(ctx) + if err != nil { + if errors.Is(err, locker.ErrSemaphoreClosed) { + return nil, anns, nil + } + + logger.Warnf("[QUERIER]: label names failed on the capture of the read lock query: %s", err) + return nil, anns, err + } + defer release() + + defer func() { + if metrics != nil { + metrics.LabelNamesDuration.Observe(float64(time.Since(start).Microseconds())) + } + }() + + dedup := deduplicatorCtor(head.NumberOfShards()) + convertedMatchers := convertPrometheusMatchersToPPMatchers(matchers...) + + t := head.CreateTask( + taskName, + func(shard TShard) error { + return shard.LSS().QueryLabelNames(shard.ShardID(), convertedMatchers, dedup.Add) + }, + ) + head.Enqueue(t) + + if err := t.Wait(); err != nil { + anns.Add(err) + } + + select { + case <-ctx.Done(): + return nil, anns, context.Cause(ctx) + default: + } + + lns := dedup.Values() + sort.Strings(lns) + + if hints.Limit > 0 && hints.Limit < len(lns) { + return lns[:hints.Limit], anns, nil + } + return lns, anns, nil +} + +// queryLabelValues returns label values present in the head for the specific label name. +func queryLabelValues[ + TTask Task, + TDataStorage DataStorage, + TLSS LSS, + TShard Shard[TDataStorage, TLSS], + THead Head[TTask, TDataStorage, TLSS, TShard], +]( + ctx context.Context, + name string, + head THead, + deduplicatorCtor deduplicatorCtor, + metrics *Metrics, + taskName string, + _ *storage.LabelHints, + matchers ...*labels.Matcher, +) ([]string, annotations.Annotations, error) { + start := time.Now() + + anns := *annotations.New() + release, err := head.AcquireQuery(ctx) + if err != nil { + if errors.Is(err, locker.ErrSemaphoreClosed) { + return nil, anns, nil + } + + logger.Warnf("[QUERIER]: label values failed on the capture of the read lock query: %s", err) + return nil, anns, err + } + defer release() + + defer func() { + if metrics != nil { + metrics.LabelValuesDuration.Observe(float64(time.Since(start).Microseconds())) + } + }() + + dedup := deduplicatorCtor(head.NumberOfShards()) + convertedMatchers := convertPrometheusMatchersToPPMatchers(matchers...) + + t := head.CreateTask( + taskName, + func(shard TShard) error { + return shard.LSS().QueryLabelValues(shard.ShardID(), name, convertedMatchers, dedup.Add) + }, + ) + head.Enqueue(t) + + if err := t.Wait(); err != nil { + anns.Add(err) + } + + select { + case <-ctx.Done(): + return nil, anns, context.Cause(ctx) + default: + } + + lvs := dedup.Values() + sort.Strings(lvs) + + return lvs, anns, nil +} + +// lssQuery returns query results and snapshots. +// +//revive:disable-next-line:cyclomatic but readable. +//revive:disable-next-line:function-length long but readable. +func queryLss[ + TTask Task, + TDataStorage DataStorage, + TLSS LSS, + TShard Shard[TDataStorage, TLSS], + THead Head[TTask, TDataStorage, TLSS, TShard], +]( + taskName string, + head THead, + matchers []*labels.Matcher, +) ( + []*cppbridge.LSSQueryResult, + []*cppbridge.LabelSetSnapshot, + error, +) { + numberOfShards := head.NumberOfShards() + selectors := make([]uintptr, numberOfShards) + snapshots := make([]*cppbridge.LabelSetSnapshot, numberOfShards) + convertedMatchers := convertPrometheusMatchersToPPMatchers(matchers...) + + tLSSQuerySelector := head.CreateTask( + taskName, + func(shard TShard) (err error) { + shardID := shard.ShardID() + selectors[shardID], snapshots[shardID], err = shard.LSS().QuerySelector(shardID, convertedMatchers) + + return err + }, + ) + head.Enqueue(tLSSQuerySelector) + if err := tLSSQuerySelector.Wait(); err != nil { + return nil, nil, err + } + + lssQueryResults := make([]*cppbridge.LSSQueryResult, numberOfShards) + errs := make([]error, numberOfShards) + for shardID, selector := range selectors { + if selector == 0 { + continue + } + + lssQueryResult := snapshots[shardID].Query(selector) + switch lssQueryResult.Status() { + case cppbridge.LSSQueryStatusMatch: + lssQueryResults[shardID] = lssQueryResult + case cppbridge.LSSQueryStatusNoMatch: + default: + errs[shardID] = fmt.Errorf( + "failed to query from shard: %d, query status: %d", shardID, lssQueryResult.Status(), + ) + } + } + if err := errors.Join(errs...); err != nil { + return nil, nil, err + } + + return lssQueryResults, snapshots, nil +} + +// UnrecoverableErrorChan channel singal for [UnrecoverableError]. +var UnrecoverableErrorChan = make(chan error) + +// SendUnrecoverableError send to terminate on [UnrecoverableError]. +func SendUnrecoverableError(err error) { + if err != nil { + logger.Warnf("Unrecoverable error: %v", err) + } + + select { + case UnrecoverableErrorChan <- UnrecoverableError{err}: + default: + } +} + +// UnrecoverableError error if Head get unrecoverable error. +type UnrecoverableError struct { + err error +} + +// Error implements error. +func (err UnrecoverableError) Error() string { + return fmt.Sprintf("Unrecoverable error: %v", err.err) +} + +// Is implements errors.Is interface. +func (UnrecoverableError) Is(target error) bool { + _, ok := target.(UnrecoverableError) + return ok +} diff --git a/pp/go/storage/querier/querier_test.go b/pp/go/storage/querier/querier_test.go new file mode 100644 index 0000000000..991c103cc8 --- /dev/null +++ b/pp/go/storage/querier/querier_test.go @@ -0,0 +1,514 @@ +package querier_test + +import ( + "context" + "os" + "path/filepath" + "testing" + "time" + + "github.com/jonboulle/clockwork" + "github.com/prometheus/client_golang/prometheus" + "github.com/stretchr/testify/suite" + + "github.com/prometheus/prometheus/model/labels" + prom_storage "github.com/prometheus/prometheus/storage" + + "github.com/prometheus/prometheus/pp/go/cppbridge" + "github.com/prometheus/prometheus/pp/go/storage" + "github.com/prometheus/prometheus/pp/go/storage/catalog" + "github.com/prometheus/prometheus/pp/go/storage/head/services" + "github.com/prometheus/prometheus/pp/go/storage/head/shard" + "github.com/prometheus/prometheus/pp/go/storage/head/task" + "github.com/prometheus/prometheus/pp/go/storage/querier" + "github.com/prometheus/prometheus/pp/go/storage/storagetest" +) + +const ( + numberOfShards uint16 = 2 + + maxSegmentSize uint32 = 1024 +) + +type Querier = querier.Querier[ + *task.Generic[*shard.PerGoroutineShard], + *shard.DataStorage, + *shard.LSS, + *shard.PerGoroutineShard, + *storage.Head, +] + +type QuerierSuite struct { + suite.Suite + dataDir string + context context.Context + head *storage.Head +} + +func TestQuerierSuite(t *testing.T) { + suite.Run(t, new(QuerierSuite)) +} + +func (s *QuerierSuite) SetupTest() { + s.dataDir = s.createDataDirectory() + s.context = context.Background() + + s.head = s.mustCreateHead(1) +} + +func (s *QuerierSuite) createDataDirectory() string { + dataDir := filepath.Join(s.T().TempDir(), "data") + s.Require().NoError(os.MkdirAll(dataDir, os.ModeDir)) + return dataDir +} + +func (s *QuerierSuite) mustCreateCatalog() *catalog.Catalog { + l, err := catalog.NewFileLogV2(filepath.Join(s.dataDir, "catalog.log")) + s.Require().NoError(err) + + c, err := catalog.New( + clockwork.NewFakeClock(), + l, + &catalog.DefaultIDGenerator{}, + catalog.DefaultMaxLogFileSize, + nil, + ) + s.Require().NoError(err) + + return c +} + +func (s *QuerierSuite) mustCreateHead(unloadDataStorageInterval time.Duration) *storage.Head { + h, err := storage.NewBuilder( + s.mustCreateCatalog(), + s.dataDir, + maxSegmentSize, + prometheus.DefaultRegisterer, + unloadDataStorageInterval, + ).Build(0, numberOfShards) + s.Require().NoError(err) + return h +} + +func (s *QuerierSuite) appendTimeSeries(timeSeries []storagetest.TimeSeries) { + storagetest.MustAppendTimeSeries(&s.Suite, s.head, timeSeries) +} + +func (s *QuerierSuite) TestRangeQuery() { + // Arrange + timeSeries := []storagetest.TimeSeries{ + { + Labels: labels.FromStrings("__name__", "metric", "job", "test"), + Samples: []cppbridge.Sample{ + {Timestamp: 0, Value: 1}, + }, + }, + { + Labels: labels.FromStrings("__name__", "metric", "job", "test2"), + Samples: []cppbridge.Sample{ + {Timestamp: 0, Value: 10}, + }, + }, + } + s.appendTimeSeries(timeSeries) + + q := querier.NewQuerier[*task.Generic[*shard.PerGoroutineShard], + *shard.DataStorage, + *shard.LSS, + *shard.PerGoroutineShard, + *storage.Head, + ](s.head, querier.NewNoOpShardedDeduplicator, 0, 2, nil, nil) + defer func() { _ = q.Close() }() + matcher, _ := labels.NewMatcher(labels.MatchEqual, "__name__", "metric") + + // Act + seriesSet := q.Select(s.context, false, nil, matcher) + + // Assert + s.Equal(timeSeries, storagetest.TimeSeriesFromSeriesSet(seriesSet)) +} + +func (s *QuerierSuite) TestRangeQueryWithoutMatching() { + // Arrange + timeSeries := []storagetest.TimeSeries{ + { + Labels: labels.FromStrings("__name__", "metric", "job", "test"), + Samples: []cppbridge.Sample{ + {Timestamp: 0, Value: 1}, + }, + }, + } + s.appendTimeSeries(timeSeries) + + q := querier.NewQuerier[*task.Generic[*shard.PerGoroutineShard], + *shard.DataStorage, + *shard.LSS, + *shard.PerGoroutineShard, + *storage.Head, + ](s.head, querier.NewNoOpShardedDeduplicator, 0, 2, nil, nil) + defer func() { _ = q.Close() }() + matcher, _ := labels.NewMatcher(labels.MatchEqual, "__name__", "unknown_metric") + + // Act + seriesSet := q.Select(s.context, false, nil, matcher) + + // Assert + s.Equal([]storagetest.TimeSeries(nil), storagetest.TimeSeriesFromSeriesSet(seriesSet)) +} + +func (s *QuerierSuite) TestRangeQueryWithDataStorageLoading() { + // Arrange + timeSeries := []storagetest.TimeSeries{ + { + Labels: labels.FromStrings("__name__", "metric", "job", "test"), + Samples: []cppbridge.Sample{ + {Timestamp: 0, Value: 0}, + {Timestamp: 1, Value: 1}, + {Timestamp: 2, Value: 2}, + }, + }, + { + Labels: labels.FromStrings("__name__", "metric", "job", "test2"), + Samples: []cppbridge.Sample{ + {Timestamp: 0, Value: 10}, + {Timestamp: 1, Value: 11}, + {Timestamp: 2, Value: 12}, + }, + }, + } + s.appendTimeSeries(timeSeries) + + timeSeriesAfterUnload := []storagetest.TimeSeries{ + { + Labels: timeSeries[0].Labels, + Samples: []cppbridge.Sample{ + {Timestamp: 3, Value: 3}, + }, + }, + { + Labels: timeSeries[1].Labels, + Samples: []cppbridge.Sample{ + {Timestamp: 3, Value: 13}, + }, + }, + } + + q := querier.NewQuerier[*task.Generic[*shard.PerGoroutineShard], + *shard.DataStorage, + *shard.LSS, + *shard.PerGoroutineShard, + *storage.Head, + ](s.head, querier.NewNoOpShardedDeduplicator, 0, 3, nil, nil) + defer func() { _ = q.Close() }() + matcher, _ := labels.NewMatcher(labels.MatchEqual, "__name__", "metric") + + // Act + s.Require().NoError(services.UnloadUnusedSeriesDataWithHead(s.head)) + s.appendTimeSeries(timeSeriesAfterUnload) + seriesSet := q.Select(s.context, false, nil, matcher) + + // Assert + timeSeries[0].AppendSamples(timeSeriesAfterUnload[0].Samples...) + timeSeries[1].AppendSamples(timeSeriesAfterUnload[1].Samples...) + s.Equal(timeSeries, storagetest.TimeSeriesFromSeriesSet(seriesSet)) +} + +func (s *QuerierSuite) TestInstantQuery() { + // Arrange + timeSeries := []storagetest.TimeSeries{ + { + Labels: labels.FromStrings("__name__", "metric", "job", "test"), + Samples: []cppbridge.Sample{ + {Timestamp: 0, Value: 1}, + }, + }, + { + Labels: labels.FromStrings("__name__", "metric", "job", "test2"), + Samples: []cppbridge.Sample{ + {Timestamp: 0, Value: 10}, + }, + }, + } + s.appendTimeSeries(timeSeries) + + q := querier.NewQuerier[*task.Generic[*shard.PerGoroutineShard], + *shard.DataStorage, + *shard.LSS, + *shard.PerGoroutineShard, + *storage.Head, + ](s.head, querier.NewNoOpShardedDeduplicator, 0, 0, nil, nil) + defer func() { _ = q.Close() }() + matcher, _ := labels.NewMatcher(labels.MatchEqual, "__name__", "metric") + + // Act + seriesSet := q.Select(s.context, false, nil, matcher) + + // Assert + s.Equal(timeSeries, storagetest.TimeSeriesFromSeriesSet(seriesSet)) +} + +func (s *QuerierSuite) TestInstantQueryWithDataStorageLoading() { + // Arrange + timeSeries := []storagetest.TimeSeries{ + { + Labels: labels.FromStrings("__name__", "metric", "job", "test"), + Samples: []cppbridge.Sample{ + {Timestamp: 0, Value: 0}, + {Timestamp: 1, Value: 1}, + {Timestamp: 2, Value: 2}, + }, + }, + { + Labels: labels.FromStrings("__name__", "metric", "job", "test2"), + Samples: []cppbridge.Sample{ + {Timestamp: 0, Value: 10}, + {Timestamp: 1, Value: 11}, + {Timestamp: 2, Value: 12}, + }, + }, + } + s.appendTimeSeries(timeSeries) + + timeSeriesAfterUnload := []storagetest.TimeSeries{ + { + Labels: timeSeries[0].Labels, + Samples: []cppbridge.Sample{ + {Timestamp: 3, Value: 3}, + }, + }, + { + Labels: timeSeries[1].Labels, + Samples: []cppbridge.Sample{ + {Timestamp: 3, Value: 13}, + }, + }, + } + + q := querier.NewQuerier[*task.Generic[*shard.PerGoroutineShard], + *shard.DataStorage, + *shard.LSS, + *shard.PerGoroutineShard, + *storage.Head, + ](s.head, querier.NewNoOpShardedDeduplicator, 0, 0, nil, nil) + defer func() { _ = q.Close() }() + matcher, _ := labels.NewMatcher(labels.MatchEqual, "__name__", "metric") + + // Act + s.Require().NoError(services.UnloadUnusedSeriesDataWithHead(s.head)) + s.appendTimeSeries(timeSeriesAfterUnload) + seriesSet := q.Select(s.context, false, nil, matcher) + + // Assert + s.Equal([]storagetest.TimeSeries{ + { + Labels: timeSeries[0].Labels, + Samples: []cppbridge.Sample{ + {Timestamp: 0, Value: 0}, + }, + }, + { + Labels: timeSeries[1].Labels, + Samples: []cppbridge.Sample{ + {Timestamp: 0, Value: 10}, + }, + }, + }, storagetest.TimeSeriesFromSeriesSet(seriesSet)) +} + +func (s *QuerierSuite) TestLabelNames() { + // Arrange + timeSeries := []storagetest.TimeSeries{ + { + Labels: labels.FromStrings("__name__", "metric0", "job0", "test0"), + Samples: []cppbridge.Sample{ + {Timestamp: 0, Value: 1}, + }, + }, + { + Labels: labels.FromStrings("__name__", "metric1", "job1", "test1"), + Samples: []cppbridge.Sample{ + {Timestamp: 0, Value: 10}, + }, + }, + } + s.appendTimeSeries(timeSeries) + + q := querier.NewQuerier(s.head, querier.NewNoOpShardedDeduplicator, 0, 2, nil, nil) + defer func() { _ = q.Close() }() + matcher, err := labels.NewMatcher(labels.MatchEqual, "__name__", "metric0") + s.Require().NoError(err) + hints := &prom_storage.LabelHints{Limit: 10} + + // Act + names, anns, err := q.LabelNames(s.context, hints, matcher) + s.Require().NoError(err) + + // Assert + s.Equal([]string{"__name__", "job0"}, names) + s.Len(anns.AsErrors(), 1) +} + +func (s *QuerierSuite) TestLabelNamesWithLimit() { + // Arrange + timeSeries := []storagetest.TimeSeries{ + { + Labels: labels.FromStrings("__name__", "metric0", "job0", "test0"), + Samples: []cppbridge.Sample{ + {Timestamp: 0, Value: 1}, + }, + }, + { + Labels: labels.FromStrings("__name__", "metric1", "job1", "test1"), + Samples: []cppbridge.Sample{ + {Timestamp: 0, Value: 10}, + }, + }, + } + s.appendTimeSeries(timeSeries) + + q := querier.NewQuerier(s.head, querier.NewNoOpShardedDeduplicator, 0, 2, nil, nil) + defer func() { _ = q.Close() }() + matcher, err := labels.NewMatcher(labels.MatchEqual, "__name__", "metric0") + s.Require().NoError(err) + hints := &prom_storage.LabelHints{Limit: 1} + + // Act + names, anns, err := q.LabelNames(s.context, hints, matcher) + s.Require().NoError(err) + + // Assert + s.Equal([]string{"__name__"}, names) + s.Len(anns.AsErrors(), 1) +} + +func (s *QuerierSuite) TestLabelNamesNoMatches() { + // Arrange + timeSeries := []storagetest.TimeSeries{ + { + Labels: labels.FromStrings("__name__", "metric0", "job0", "test0"), + Samples: []cppbridge.Sample{ + {Timestamp: 0, Value: 1}, + }, + }, + { + Labels: labels.FromStrings("__name__", "metric1", "job1", "test1"), + Samples: []cppbridge.Sample{ + {Timestamp: 0, Value: 10}, + }, + }, + } + s.appendTimeSeries(timeSeries) + + q := querier.NewQuerier(s.head, querier.NewNoOpShardedDeduplicator, 0, 2, nil, nil) + defer func() { _ = q.Close() }() + matcher, err := labels.NewMatcher(labels.MatchEqual, "__name__", "metric3") + s.Require().NoError(err) + hints := &prom_storage.LabelHints{Limit: 1} + + // Act + names, anns, err := q.LabelNames(s.context, hints, matcher) + s.Require().NoError(err) + + // Assert + s.Equal([]string{}, names) + s.Len(anns.AsErrors(), 1) +} + +func (s *QuerierSuite) TestLabelValues() { + // Arrange + timeSeries := []storagetest.TimeSeries{ + { + Labels: labels.FromStrings("__name__", "metric0", "job0", "test0"), + Samples: []cppbridge.Sample{ + {Timestamp: 0, Value: 1}, + }, + }, + { + Labels: labels.FromStrings("__name__", "metric1", "job1", "test1"), + Samples: []cppbridge.Sample{ + {Timestamp: 0, Value: 10}, + }, + }, + } + s.appendTimeSeries(timeSeries) + + q := querier.NewQuerier(s.head, querier.NewNoOpShardedDeduplicator, 0, 2, nil, nil) + defer func() { _ = q.Close() }() + matcher, err := labels.NewMatcher(labels.MatchRegexp, "__name__", "metric.*") + s.Require().NoError(err) + hints := &prom_storage.LabelHints{Limit: 10} + + // Act + names, anns, err := q.LabelValues(s.context, "__name__", hints, matcher) + s.Require().NoError(err) + + // Assert + s.Equal([]string{"metric0", "metric1"}, names) + s.Len(anns.AsErrors(), 1) +} + +func (s *QuerierSuite) TestLabelValuesNoMatches() { + // Arrange + timeSeries := []storagetest.TimeSeries{ + { + Labels: labels.FromStrings("__name__", "metric0", "job0", "test0"), + Samples: []cppbridge.Sample{ + {Timestamp: 0, Value: 1}, + }, + }, + { + Labels: labels.FromStrings("__name__", "metric1", "job1", "test1"), + Samples: []cppbridge.Sample{ + {Timestamp: 0, Value: 10}, + }, + }, + } + s.appendTimeSeries(timeSeries) + + q := querier.NewQuerier(s.head, querier.NewNoOpShardedDeduplicator, 0, 2, nil, nil) + defer func() { _ = q.Close() }() + matcher, err := labels.NewMatcher(labels.MatchEqual, "__name__", "metric2") + s.Require().NoError(err) + hints := &prom_storage.LabelHints{Limit: 10} + + // Act + names, anns, err := q.LabelValues(s.context, "__name__", hints, matcher) + s.Require().NoError(err) + + // Assert + s.Equal([]string{}, names) + s.Len(anns.AsErrors(), 1) +} + +func (s *QuerierSuite) TestLabelValuesNoMatchesOnName() { + // Arrange + timeSeries := []storagetest.TimeSeries{ + { + Labels: labels.FromStrings("__name__", "metric0", "job0", "test0"), + Samples: []cppbridge.Sample{ + {Timestamp: 0, Value: 1}, + }, + }, + { + Labels: labels.FromStrings("__name__", "metric1", "job1", "test1"), + Samples: []cppbridge.Sample{ + {Timestamp: 0, Value: 10}, + }, + }, + } + s.appendTimeSeries(timeSeries) + + q := querier.NewQuerier(s.head, querier.NewNoOpShardedDeduplicator, 0, 2, nil, nil) + defer func() { _ = q.Close() }() + matcher, err := labels.NewMatcher(labels.MatchRegexp, "__name__", "metric.*") + s.Require().NoError(err) + hints := &prom_storage.LabelHints{Limit: 10} + + // Act + names, anns, err := q.LabelValues(s.context, "instance", hints, matcher) + s.Require().NoError(err) + + // Assert + s.Equal([]string{}, names) + s.Len(anns.AsErrors(), 1) +} diff --git a/pp/go/storage/querier/series.go b/pp/go/storage/querier/series.go new file mode 100644 index 0000000000..3956ce84d8 --- /dev/null +++ b/pp/go/storage/querier/series.go @@ -0,0 +1,483 @@ +package querier + +import ( + "github.com/prometheus/prometheus/model/histogram" + "github.com/prometheus/prometheus/model/labels" + "github.com/prometheus/prometheus/pp/go/cppbridge" + "github.com/prometheus/prometheus/storage" + "github.com/prometheus/prometheus/tsdb/chunkenc" + "github.com/prometheus/prometheus/util/annotations" +) + +// SampleProvider create chunk iterator over samples. +type SampleProvider interface { + Samples(minT, maxtT int64) chunkenc.Iterator +} + +// +// SeriesSet +// + +// SeriesSet contains a set of series, allows to iterate over sorted, populated series. +type SeriesSet struct { + mint int64 + maxt int64 + deserializer *cppbridge.HeadDataStorageDeserializer + chunksIndex cppbridge.HeadDataStorageSerializedChunkIndex + serializedChunks *cppbridge.HeadDataStorageSerializedChunks + lssQueryResult *cppbridge.LSSQueryResult + labelSetSnapshot *cppbridge.LabelSetSnapshot + + index int + currentSeries *Series +} + +// At returns full series. Returned series should be iterable even after Next is called. +func (ss *SeriesSet) At() storage.Series { + return ss.currentSeries +} + +// Err the error that iteration as failed with. +func (*SeriesSet) Err() error { + return nil +} + +// Next return true if exist there is a next series and false otherwise. +func (ss *SeriesSet) Next() bool { + if ss.lssQueryResult == nil { + return false + } + + var ( + lsID uint32 + lsLength uint16 + chunksMetadata []cppbridge.HeadDataStorageSerializedChunkMetadata + ) + + for { + if ss.index >= ss.lssQueryResult.Len() { + return false + } + + lsID, lsLength = ss.lssQueryResult.GetByIndex(ss.index) + + chunksMetadata = ss.chunksIndex.Chunks(ss.serializedChunks, lsID) + ss.index++ + if len(chunksMetadata) != 0 { + break + } + } + + ss.currentSeries = &Series{ + mint: ss.mint, + maxt: ss.maxt, + labelSet: labels.NewLabelsWithLSS(ss.labelSetSnapshot, lsID, lsLength), + sampleProvider: &DefaultSampleProvider{ + deserializer: ss.deserializer, + chunksMetadata: chunksMetadata, + }, + } + + return true +} + +// Warnings a collection of warnings for the whole set. +func (*SeriesSet) Warnings() annotations.Annotations { + return nil +} + +// +// Series +// + +// Series is a stream of data points belonging to a metric. +type Series struct { + mint, maxt int64 + labelSet labels.Labels + sampleProvider SampleProvider +} + +// Iterator returns an iterator of the data of the series. +func (s *Series) Iterator(_ chunkenc.Iterator) chunkenc.Iterator { + return s.sampleProvider.Samples(s.mint, s.maxt) +} + +// Labels returns the complete set of labels. +func (s *Series) Labels() labels.Labels { + return s.labelSet +} + +// +// DefaultSampleProvider +// + +// DefaultSampleProvider create default chunk iterator over samples. +type DefaultSampleProvider struct { + deserializer *cppbridge.HeadDataStorageDeserializer + chunksMetadata []cppbridge.HeadDataStorageSerializedChunkMetadata +} + +// Samples reurns chunk iterator over samples. +func (sp *DefaultSampleProvider) Samples(mint, maxt int64) chunkenc.Iterator { + return NewLimitedChunkIterator( + NewChunkIterator(sp.deserializer, sp.chunksMetadata), + mint, + maxt, + ) +} + +// +// ChunkIterator +// + +// ChunkIterator iterates over the samples of a time series, that can only get the next value. +type ChunkIterator struct { + deserializer *cppbridge.HeadDataStorageDeserializer + chunksMetadata []cppbridge.HeadDataStorageSerializedChunkMetadata + decodeIterator *cppbridge.HeadDataStorageDecodeIterator + ts int64 + v float64 +} + +// NewChunkIterator init new [ChunkIterator]. +func NewChunkIterator( + deserializer *cppbridge.HeadDataStorageDeserializer, + chunksMetadata []cppbridge.HeadDataStorageSerializedChunkMetadata, +) *ChunkIterator { + return &ChunkIterator{ + deserializer: deserializer, + chunksMetadata: chunksMetadata, + } +} + +// At returns the current timestamp/value pair if the value is a float. +// +//nolint:gocritic // unnamedResult not need +func (i *ChunkIterator) At() (int64, float64) { + return i.ts, i.v +} + +// AtFloatHistogram returns the current timestamp/value pair if the value is a histogram with floating-point counts. +func (*ChunkIterator) AtFloatHistogram(_ *histogram.FloatHistogram) (int64, *histogram.FloatHistogram) { + return 0, nil +} + +// AtHistogram returns the current timestamp/value pair if the value is a histogram with integer counts. +func (*ChunkIterator) AtHistogram(*histogram.Histogram) (int64, *histogram.Histogram) { + return 0, nil +} + +// AtT returns the current timestamp. +func (i *ChunkIterator) AtT() int64 { + return i.ts +} + +// Err returns the current error. +func (*ChunkIterator) Err() error { + return nil +} + +// Next advances the iterator by one and returns the type of the value. +func (i *ChunkIterator) Next() chunkenc.ValueType { + if i.decodeIterator == nil { + if len(i.chunksMetadata) == 0 { + return chunkenc.ValNone + } + + i.decodeIterator = i.deserializer.CreateDecodeIterator(i.chunksMetadata[0]) + i.chunksMetadata = i.chunksMetadata[1:] + } + + if !i.decodeIterator.Next() { + i.decodeIterator = nil + return i.Next() + } + + i.ts, i.v = i.decodeIterator.Sample() + return chunkenc.ValFloat +} + +// Seek advances the iterator forward to the first sample with a timestamp equal or greater than t. +func (i *ChunkIterator) Seek(t int64) chunkenc.ValueType { + if i.decodeIterator == nil { + if i.Next() == chunkenc.ValNone { + return chunkenc.ValNone + } + } + + for i.ts < t { + if i.Next() == chunkenc.ValNone { + return chunkenc.ValNone + } + } + + return chunkenc.ValFloat +} + +// +// LimitedChunkIterator +// + +// LimitedChunkIterator iterates over the samples of a time series, that can only get the next value with limit. +type LimitedChunkIterator struct { + chunkIterator chunkenc.Iterator + mint int64 + maxt int64 +} + +// NewLimitedChunkIterator init new [LimitedChunkIterator]. +func NewLimitedChunkIterator(iterator chunkenc.Iterator, mint, maxt int64) *LimitedChunkIterator { + return &LimitedChunkIterator{ + chunkIterator: iterator, + mint: mint, + maxt: maxt, + } +} + +// At returns the current timestamp/value pair if the value is a float. +// +//nolint:gocritic // unnamedResult not need +func (i *LimitedChunkIterator) At() (int64, float64) { + return i.chunkIterator.At() +} + +// AtFloatHistogram returns the current timestamp/value pair if the value is a histogram with floating-point counts. +func (i *LimitedChunkIterator) AtFloatHistogram(h *histogram.FloatHistogram) (int64, *histogram.FloatHistogram) { + return i.chunkIterator.AtFloatHistogram(h) +} + +// AtHistogram returns the current timestamp/value pair if the value is a histogram with integer counts. +func (i *LimitedChunkIterator) AtHistogram(h *histogram.Histogram) (int64, *histogram.Histogram) { + return i.chunkIterator.AtHistogram(h) +} + +// AtT returns the current timestamp. +func (i *LimitedChunkIterator) AtT() int64 { + return i.chunkIterator.AtT() +} + +// Err returns the current error. +func (i *LimitedChunkIterator) Err() error { + return i.chunkIterator.Err() +} + +// Next advances the iterator by one and returns the type of the value. +func (i *LimitedChunkIterator) Next() chunkenc.ValueType { + if i.chunkIterator.Next() == chunkenc.ValNone { + return chunkenc.ValNone + } + + if i.Seek(i.mint) == chunkenc.ValNone { + return chunkenc.ValNone + } + + if i.chunkIterator.AtT() > i.maxt { + return chunkenc.ValNone + } + + return chunkenc.ValFloat +} + +// Seek advances the iterator forward to the first sample with a timestamp equal or greater than t. +func (i *LimitedChunkIterator) Seek(t int64) chunkenc.ValueType { + if t < i.mint { + t = i.mint + } + + if t > i.maxt { + t = i.maxt + } + + if i.chunkIterator.Seek(t) == chunkenc.ValNone { + return chunkenc.ValNone + } + + if i.chunkIterator.AtT() > i.maxt { + return chunkenc.ValNone + } + + return chunkenc.ValFloat +} + +// +// InstantSeriesSet +// + +// InstantSeriesSet contains a instatnt set of series, allows to iterate over sorted, populated series. +type InstantSeriesSet struct { + lssQueryResult *cppbridge.LSSQueryResult + labelSetSnapshot *cppbridge.LabelSetSnapshot + valueNotFoundTimestampValue int64 + samples []cppbridge.Sample + + index int + currentSeries *InstantSeries +} + +// NewInstantSeriesSet init new [InstantSeriesSet]. +func NewInstantSeriesSet( + lssQueryResult *cppbridge.LSSQueryResult, + labelSetSnapshot *cppbridge.LabelSetSnapshot, + valueNotFoundTimestampValue int64, + samples []cppbridge.Sample, +) *InstantSeriesSet { + return &InstantSeriesSet{ + lssQueryResult: lssQueryResult, + labelSetSnapshot: labelSetSnapshot, + valueNotFoundTimestampValue: valueNotFoundTimestampValue, + samples: samples, + index: -1, + } +} + +// At returns full series. Returned series should be iterable even after Next is called. +func (ss *InstantSeriesSet) At() storage.Series { + return ss.currentSeries +} + +// Err the error that iteration as failed with. +func (*InstantSeriesSet) Err() error { + return nil +} + +// Next return true if exist there is a next series and false otherwise. +func (ss *InstantSeriesSet) Next() bool { + for { + if ss.index+1 >= ss.lssQueryResult.Len() { + return false + } + + ss.index++ + if ss.samples[ss.index].Timestamp != ss.valueNotFoundTimestampValue { + break + } + } + + lsID, lsLength := ss.lssQueryResult.GetByIndex(ss.index) + ss.currentSeries = &InstantSeries{ + labelSet: labels.NewLabelsWithLSS( + ss.labelSetSnapshot, + lsID, + lsLength, + ), + sample: ss.samples[ss.index], + } + + return true +} + +// Warnings a collection of warnings for the whole set. +func (*InstantSeriesSet) Warnings() annotations.Annotations { + return nil +} + +// +// InstantSeries +// + +// InstantSeries is a instant stream of data points belonging to a metric. +type InstantSeries struct { + labelSet labels.Labels + sample cppbridge.Sample +} + +// Iterator is storage.Series interface implementation. +func (s *InstantSeries) Iterator(iterator chunkenc.Iterator) chunkenc.Iterator { + if i, ok := iterator.(*InstantSeriesChunkIterator); ok { + i.ResetTo(s.sample.Timestamp, s.sample.Value) + return i + } + return NewInstantSeriesChunkIterator(s.sample.Timestamp, s.sample.Value) +} + +// Labels is storage.Series interface implementation. +func (s *InstantSeries) Labels() labels.Labels { + return s.labelSet +} + +// +// InstantSeriesChunkIterator +// + +// InstantSeriesChunkIterator iterates over the samples of a instant time series, that can only get the next value. +type InstantSeriesChunkIterator struct { + i int + t int64 + v float64 +} + +// NewInstantSeriesChunkIterator init new [InstantSeriesChunkIterator]. +func NewInstantSeriesChunkIterator(t int64, v float64) *InstantSeriesChunkIterator { + return &InstantSeriesChunkIterator{ + i: -1, + t: t, + v: v, + } +} + +// At returns the current timestamp/value pair if the value is a float. +// +//nolint:gocritic // unnamedResult not need +func (i *InstantSeriesChunkIterator) At() (int64, float64) { + return i.t, i.v +} + +// AtFloatHistogram returns the current timestamp/value pair if the value is a histogram with floating-point counts. +func (*InstantSeriesChunkIterator) AtFloatHistogram(_ *histogram.FloatHistogram) (int64, *histogram.FloatHistogram) { + return 0, nil +} + +// AtHistogram returns the current timestamp/value pair if the value is a histogram with integer counts. +func (*InstantSeriesChunkIterator) AtHistogram(_ *histogram.Histogram) (int64, *histogram.Histogram) { + return 0, nil +} + +// AtT returns the current timestamp. +func (i *InstantSeriesChunkIterator) AtT() int64 { + return i.t +} + +// Err returns the current error. +func (*InstantSeriesChunkIterator) Err() error { + return nil +} + +// Next advances the iterator by one and returns the type of the value. +func (i *InstantSeriesChunkIterator) Next() chunkenc.ValueType { + if i.i < 1 { + i.i++ + } + return i.valueType() +} + +// ResetTo reset state to timestamp and value. +func (i *InstantSeriesChunkIterator) ResetTo(t int64, v float64) { + i.i = -1 + i.t = t + i.v = v +} + +// Seek advances the iterator forward to the first sample with a timestamp equal or greater than t. +func (i *InstantSeriesChunkIterator) Seek(t int64) chunkenc.ValueType { + if i.valueType() == chunkenc.ValFloat && i.t >= t { + return chunkenc.ValFloat + } + + for { + if i.Next() == chunkenc.ValNone { + return chunkenc.ValNone + } + + if i.t >= t { + return chunkenc.ValFloat + } + } +} + +func (i *InstantSeriesChunkIterator) valueType() chunkenc.ValueType { + if i.i == 0 { + return chunkenc.ValFloat + } + + return chunkenc.ValNone +} diff --git a/pp/go/storage/querier/status_querier.go b/pp/go/storage/querier/status_querier.go new file mode 100644 index 0000000000..893ddd0e7c --- /dev/null +++ b/pp/go/storage/querier/status_querier.go @@ -0,0 +1,188 @@ +package querier + +import ( + "context" + "errors" + "math" + "sort" + + "github.com/prometheus/prometheus/pp/go/cppbridge" + "github.com/prometheus/prometheus/pp/go/logger" + "github.com/prometheus/prometheus/pp/go/storage/head/task" + "github.com/prometheus/prometheus/pp/go/util/locker" +) + +const ( + // dsHeadStatus name of task. + dsHeadStatus = "data_storage_head_status" + + // lssHeadStatus name of task. + lssHeadStatus = "lss_head_status" +) + +// QueryHeadStatus returns [HeadStatus] holds information about all shards from [Head]. +func QueryHeadStatus[ + TTask Task, + TDataStorage DataStorage, + TLSS LSS, + TShard Shard[TDataStorage, TLSS], + THead Head[TTask, TDataStorage, TLSS, TShard], +]( + ctx context.Context, + head THead, + limit int, +) (*HeadStatus, error) { + shardStatuses := make([]*cppbridge.HeadStatus, head.NumberOfShards()) + for i := range shardStatuses { + shardStatuses[i] = cppbridge.NewHeadStatus() + } + + tw := task.NewTaskWaiter[TTask](2) //revive:disable-line:add-constant // 2 task for wait + + release, err := head.AcquireQuery(ctx) + if err != nil { + if !errors.Is(err, locker.ErrSemaphoreClosed) { + logger.Warnf("[HeadStatusQuerier]: query status failed on the capture of the read lock query: %s", err) + } + + return nil, err + } + defer release() + + tLSSHeadStatus := head.CreateTask( + lssHeadStatus, + func(shard TShard) error { + return shard.LSS().WithRLock(func(target, _ *cppbridge.LabelSetStorage) error { + shardStatuses[shard.ShardID()].FromLSS(target, limit) + + return nil + }) + }, + ) + head.Enqueue(tLSSHeadStatus) + + if limit != 0 { + tDataStorageHeadStatus := head.CreateTask( + dsHeadStatus, + func(shard TShard) error { + return shard.DataStorage().WithRLock(func(ds *cppbridge.HeadDataStorage) error { + shardStatuses[shard.ShardID()].FromDataStorage(ds) + + return nil + }) + }, + ) + head.Enqueue(tDataStorageHeadStatus) + tw.Add(tDataStorageHeadStatus) + } + + tw.Add(tLSSHeadStatus) + _ = tw.Wait() + + return sumStatuses(shardStatuses, limit), nil +} + +// sumStatuses summarize the statuses received from the shards. +func sumStatuses(shardStatuses []*cppbridge.HeadStatus, limit int) *HeadStatus { + seriesStats := make(map[string]uint64) + labelsStats := make(map[string]uint64) + memoryStats := make(map[string]uint64) + countStats := make(map[string]uint64) + + headStatus := &HeadStatus{HeadStats: HeadStats{MinTime: math.MaxInt64, MaxTime: math.MinInt64}} + + for _, shardStatus := range shardStatuses { + headStatus.HeadStats.NumSeries += uint64(shardStatus.NumSeries) + if limit == 0 { + continue + } + + headStatus.HeadStats.ChunkCount += int64(shardStatus.ChunkCount) + if headStatus.HeadStats.MaxTime < shardStatus.TimeInterval.Max { + headStatus.HeadStats.MaxTime = shardStatus.TimeInterval.Max + } + if headStatus.HeadStats.MinTime > shardStatus.TimeInterval.Min { + headStatus.HeadStats.MinTime = shardStatus.TimeInterval.Min + } + + headStatus.HeadStats.NumLabelPairs += int(shardStatus.NumLabelPairs) + + for _, stat := range shardStatus.SeriesCountByMetricName { + seriesStats[stat.Name] += uint64(stat.Count) + } + for _, stat := range shardStatus.LabelValueCountByLabelName { + labelsStats[stat.Name] += uint64(stat.Count) + } + for _, stat := range shardStatus.MemoryInBytesByLabelName { + memoryStats[stat.Name] += uint64(stat.Size) + } + for _, stat := range shardStatus.SeriesCountByLabelValuePair { + countStats[stat.Name+"="+stat.Value] += uint64(stat.Count) + } + } + + if limit == 0 { + return headStatus + } + + headStatus.SeriesCountByMetricName = getSortedStats(seriesStats, limit) + headStatus.LabelValueCountByLabelName = getSortedStats(labelsStats, limit) + headStatus.MemoryInBytesByLabelName = getSortedStats(memoryStats, limit) + headStatus.SeriesCountByLabelValuePair = getSortedStats(countStats, limit) + + return headStatus +} + +// getSortedStats returns sorted statistics for the [Head]. +func getSortedStats(stats map[string]uint64, limit int) []HeadStat { + result := make([]HeadStat, 0, len(stats)) + for k, v := range stats { + result = append(result, HeadStat{ + Name: k, + Value: v, + }) + } + + sort.Slice(result, func(i, j int) bool { + return result[i].Value > result[j].Value + }) + + if len(result) > limit { + return result[:limit] + } + + return result +} + +// +// HeadStatus +// + +// HeadStatus holds information about all shards. +type HeadStatus struct { + HeadStats HeadStats `json:"headStats"` + SeriesCountByMetricName []HeadStat `json:"seriesCountByMetricName"` + LabelValueCountByLabelName []HeadStat `json:"labelValueCountByLabelName"` + MemoryInBytesByLabelName []HeadStat `json:"memoryInBytesByLabelName"` + SeriesCountByLabelValuePair []HeadStat `json:"seriesCountByLabelValuePair"` +} + +// NumSeries returns number of series. +func (hs *HeadStatus) NumSeries() uint64 { + return hs.HeadStats.NumSeries +} + +// HeadStat holds the information about individual cardinality. +type HeadStat struct { + Name string `json:"name"` + Value uint64 `json:"value"` +} + +// HeadStats has information about the head. +type HeadStats struct { + NumSeries uint64 `json:"numSeries"` + NumLabelPairs int `json:"numLabelPairs"` + ChunkCount int64 `json:"chunkCount"` + MinTime int64 `json:"minTime"` + MaxTime int64 `json:"maxTime"` +} diff --git a/pp/go/storage/ready/ready.go b/pp/go/storage/ready/ready.go new file mode 100644 index 0000000000..3dd69ae0db --- /dev/null +++ b/pp/go/storage/ready/ready.go @@ -0,0 +1,146 @@ +package ready + +import ( + "sync" + "sync/atomic" +) + +// +// Notifier +// + +// Notifier the sender notifies about readiness for work. +type Notifier interface { + // NotifyReady the sender notifies about readiness for work. + NotifyReady() +} + +// +// Notifiable +// + +// Notifiable notifies the recipient that it is ready to work. +type Notifiable interface { + // ReadyChan notifies the recipient that it is ready to work. + ReadyChan() <-chan struct{} +} + +// +// Builder +// + +// Builder for creating [MultiNotifiable]. +type Builder struct { + input []Notifiable +} + +// NewMultiNotifiableBuilder init new [Builder]. +func NewMultiNotifiableBuilder() *Builder { + return &Builder{} +} + +// Add [Notifiable] to list. +func (b *Builder) Add(notifiable Notifiable) *Builder { + b.input = append(b.input, notifiable) + return b +} + +// Build creating [MultiNotifiable] from [Notifiable]s. +func (b *Builder) Build() *MultiNotifiable { + mn := &MultiNotifiable{ + ready: make(chan struct{}), + closed: make(chan struct{}), + } + + mn.counter.Add(int64(len(b.input))) + for _, notifiable := range b.input { + go func(notifiable Notifiable) { + select { + case <-notifiable.ReadyChan(): + if mn.counter.Add(-1) == 0 { + mn.setReady() + } + case <-mn.closed: + } + }(notifiable) + } + + return mn +} + +// +// MultiNotifiable +// + +// MultiNotifiable aggregates multiple [Notifiable]s. +type MultiNotifiable struct { + readyOnce sync.Once + ready chan struct{} + closedOnce sync.Once + closed chan struct{} + counter atomic.Int64 +} + +// Close stop [MultiNotifiable]. +func (mn *MultiNotifiable) Close() error { + mn.setClosed() + return nil +} + +// ReadyChan notifies the recipient that it is ready to work. +func (mn *MultiNotifiable) ReadyChan() <-chan struct{} { + return mn.ready +} + +// setClosed set once [MultiNotifiable] is closed. +func (mn *MultiNotifiable) setClosed() { + mn.closedOnce.Do(func() { + close(mn.closed) + }) +} + +// setReady set once [MultiNotifiable] is ready. +func (mn *MultiNotifiable) setReady() { + mn.readyOnce.Do(func() { + close(mn.ready) + }) +} + +// +// NotifiableNotifier +// + +// NotifiableNotifier the sender notifies about readiness for work, notifies the recipient that it is ready to work. +type NotifiableNotifier struct { + once sync.Once + c chan struct{} +} + +// NewNotifiableNotifier init new [NotifiableNotifier]. +func NewNotifiableNotifier() *NotifiableNotifier { + return &NotifiableNotifier{ + c: make(chan struct{}), + } +} + +// NotifyReady the sender notifies about readiness for work. +func (nn *NotifiableNotifier) NotifyReady() { + nn.once.Do(func() { + close(nn.c) + }) +} + +// ReadyChan notifies the recipient that it is ready to work. +func (nn *NotifiableNotifier) ReadyChan() <-chan struct{} { + return nn.c +} + +// +// NoOpNotifier +// + +// NoOpNotifier do nothing notifier. +type NoOpNotifier struct{} + +// NotifyReady implementation [Notifier], do nothing. +func (NoOpNotifier) NotifyReady() {} diff --git a/pp/go/storage/remotewriter/README.md b/pp/go/storage/remotewriter/README.md new file mode 100644 index 0000000000..b20e450f3a --- /dev/null +++ b/pp/go/storage/remotewriter/README.md @@ -0,0 +1,19 @@ +1. Create cursor. +2. Create client. +3. Loop: + 1. create batch. (readTimeout) + 1. read next segment. + 1. if permanent error - batch completed + end of block is reached. + 2. if not permanent error - if batch is fulfilled or deadline reached - batch is completed. + 1. recalculate number of output shards. + 3. if no error, and batch is not full, wait for 5 sec and repeat + 2. try go (write cache). (retry+backoff) + 3. encode protobuf. + 4. send. (retry+backoff) + 1. if outdated + 1. return permanent error + 2. on error -> non permanent error + 3. on success -> nil + 5. try ack. + 6. check end of block + \ No newline at end of file diff --git a/pp/go/storage/remotewriter/cursor.go b/pp/go/storage/remotewriter/cursor.go new file mode 100644 index 0000000000..441504a8b7 --- /dev/null +++ b/pp/go/storage/remotewriter/cursor.go @@ -0,0 +1,159 @@ +package remotewriter + +import ( + "errors" + "fmt" + "os" + "unsafe" + + "github.com/edsrzf/mmap-go" + + "github.com/prometheus/prometheus/tsdb/fileutil" +) + +// +// MMapFile +// + +// MMapFile a wrapper over the mmap file. +type MMapFile struct { + file *os.File + mmap mmap.MMap +} + +// NewMMapFile init new [MMapFile]. +func NewMMapFile(fileName string, flag int, perm os.FileMode, targetSize int64) (*MMapFile, error) { + file, err := os.OpenFile(fileName, flag, perm) // #nosec G304 // it's meant to be that way + if err != nil { + return nil, fmt.Errorf("open file: %w", err) + } + + fileInfo, err := file.Stat() + if err != nil { + return nil, errors.Join(fmt.Errorf("stat file: %w", err), file.Close()) + } + + if fileInfo.Size() < targetSize { + if err = fileutil.Preallocate(file, targetSize, true); err != nil { + return nil, errors.Join(fmt.Errorf("preallocate file: %w", err), file.Close()) + } + if err = file.Sync(); err != nil { + return nil, errors.Join(fmt.Errorf("sync file: %w", err), file.Close()) + } + } + + mapped, err := mmap.Map(file, mmap.RDWR, 0) + if err != nil { + return nil, errors.Join(fmt.Errorf("map file: %w", err), file.Close()) + } + + if err = mapped.Lock(); err != nil { + return nil, errors.Join(fmt.Errorf("lock mapped file: %w", err), mapped.Unmap(), file.Close()) + } + + return &MMapFile{ + file: file, + mmap: mapped, + }, nil +} + +// Bytes returns mapped into memory data. +func (f *MMapFile) Bytes() []byte { + return f.mmap +} + +// Close closes the [os.File], rendering it unusable for I/O. +// Unmap deletes the memory mapped region, flushes any remaining changes, and sets m to nil. +func (f *MMapFile) Close() error { + return errors.Join(f.mmap.Unlock(), f.mmap.Unmap(), f.file.Close()) +} + +// Sync synchronizes the mapping's contents to the file's contents on disk. +func (f *MMapFile) Sync() error { + return f.mmap.Flush() +} + +// +// Cursor +// + +// Cursor to the required ID segment. +type Cursor struct { + targetSegmentID uint32 + configCRC32 uint32 +} + +// CursorReadWriter reader and writer [Cursor]s from mmaped [MMapFile]. +type CursorReadWriter struct { + cursor *Cursor + failedShards []byte + file *MMapFile +} + +// NewCursorReadWriter init new [CursorReadWriter]. +func NewCursorReadWriter(fileName string, numberOfShards uint16) (*CursorReadWriter, error) { + cursorSize := int64(unsafe.Sizeof(Cursor{})) + fileSize := cursorSize + int64(numberOfShards) + //revive:disable-next-line:add-constant file permissions simple readable as octa-number + file, err := NewMMapFile(fileName, os.O_CREATE|os.O_RDWR, 0o600, fileSize) + if err != nil { + return nil, fmt.Errorf("create file: %w", err) + } + + crw := &CursorReadWriter{ + cursor: (*Cursor)(unsafe.Pointer(&file.Bytes()[0])), // #nosec G103 // cast to Cursor + failedShards: unsafe.Slice( // #nosec G103 // it's meant to be that way + unsafe.SliceData(file.Bytes()[cursorSize:]), // #nosec G103 // it's meant to be that way + numberOfShards, + ), + file: file, + } + + return crw, nil +} + +// Close closes the [MMapFile]. +func (crw *CursorReadWriter) Close() error { + if crw.file != nil { + err := crw.file.Close() + if err == nil { + crw.file = nil + } + return err + } + + return nil +} + +// GetConfigCRC32 returns CRC32 for config. +func (crw *CursorReadWriter) GetConfigCRC32() uint32 { + return crw.cursor.configCRC32 +} + +// GetTargetSegmentID returns target segment ID. +func (crw *CursorReadWriter) GetTargetSegmentID() uint32 { + return crw.cursor.targetSegmentID +} + +// SetConfigCRC32 set CRC32 for config. +func (crw *CursorReadWriter) SetConfigCRC32(configCRC32 uint32) error { + crw.cursor.configCRC32 = configCRC32 + return crw.file.Sync() +} + +// SetShardCorrupted adds a flag that is a shard corrupted by shard ID. +func (crw *CursorReadWriter) SetShardCorrupted(shardID uint16) error { + crw.failedShards[shardID] = 1 + return crw.file.Sync() +} + +// SetTargetSegmentID set target segment ID. +func (crw *CursorReadWriter) SetTargetSegmentID(segmentID uint32) error { + crw.cursor.targetSegmentID = segmentID + return crw.file.Sync() +} + +// ShardIsCorrupted returns a flag that is a shard corrupted by shard ID. +func (crw *CursorReadWriter) ShardIsCorrupted(shardID uint16) bool { + return crw.failedShards[shardID] > 0 +} diff --git a/pp/go/storage/remotewriter/cursor_test.go b/pp/go/storage/remotewriter/cursor_test.go new file mode 100644 index 0000000000..d71b01d2be --- /dev/null +++ b/pp/go/storage/remotewriter/cursor_test.go @@ -0,0 +1,160 @@ +package remotewriter + +import ( + "bytes" + "os" + "testing" + "unsafe" + + "github.com/edsrzf/mmap-go" + "github.com/stretchr/testify/require" + + "github.com/prometheus/prometheus/tsdb/fileutil" +) + +func TestNewCursor(t *testing.T) { + tempFile, err := os.CreateTemp("", "mmaptest") + require.NoError(t, err) + tempFileName := tempFile.Name() + defer func() { + _ = os.RemoveAll(tempFileName) + }() + require.NoError(t, tempFile.Close()) + + var numberOfShards uint16 = 2 + func(t *testing.T) { + var crw *CursorReadWriter + crw, err = NewCursorReadWriter(tempFileName, numberOfShards) + require.NoError(t, err) + defer func() { _ = crw.Close() }() + + require.Equal(t, uint32(0), crw.GetTargetSegmentID()) + require.Equal(t, uint32(0), crw.GetConfigCRC32()) + require.False(t, crw.ShardIsCorrupted(0)) + require.False(t, crw.ShardIsCorrupted(1)) + require.NoError(t, crw.SetTargetSegmentID(42)) + require.NoError(t, crw.SetShardCorrupted(1)) + }(t) + + func(t *testing.T) { + var crw *CursorReadWriter + crw, err = NewCursorReadWriter(tempFileName, numberOfShards) + require.NoError(t, err) + defer func() { _ = crw.Close() }() + + require.Equal(t, uint32(42), crw.GetTargetSegmentID()) + require.Equal(t, uint32(0), crw.GetConfigCRC32()) + require.False(t, crw.ShardIsCorrupted(0)) + require.True(t, crw.ShardIsCorrupted(1)) + }(t) +} + +type CursorV2 struct { + TargetSegmentID uint32 + ConfigCRC32 uint32 +} + +func TestMMapFile_Bytes(t *testing.T) { + tempFile, err := os.CreateTemp("", "mmaptest") + require.NoError(t, err) + tempFileName := tempFile.Name() + defer func() { + _ = os.RemoveAll(tempFileName) + }() + require.NoError(t, tempFile.Close()) + + targetSize := unsafe.Sizeof(Cursor{}) + + func(t *testing.T) { + mfile, err := NewMMapFile(tempFileName, os.O_CREATE|os.O_RDWR, 0o600, int64(targetSize)) + require.NoError(t, err) + defer func() { + _ = mfile.Close() + }() + t.Log(mfile.mmap) + c := (*CursorV2)(unsafe.Pointer(&mfile.mmap[0])) + c.TargetSegmentID = 1 + require.NoError(t, mfile.Sync()) + }(t) + + func(t *testing.T) { + mfile, err := NewMMapFile(tempFileName, os.O_CREATE|os.O_RDWR, 0o600, int64(targetSize)) + require.NoError(t, err) + defer func() { + _ = mfile.Close() + }() + c := (*CursorV2)(unsafe.Pointer(&mfile.mmap[0])) + + require.Equal(t, uint32(1), c.TargetSegmentID) + }(t) +} + +func TestMMap(t *testing.T) { + tempFile, err := os.CreateTemp("", "mmaptest") + require.NoError(t, err) + tempFileName := tempFile.Name() + defer func() { + _ = os.RemoveAll(tempFileName) + }() + require.NoError(t, tempFile.Close()) + + testString := "lol + kek = chebureck" + + func(t *testing.T) { + var file *os.File + file, err = os.OpenFile(tempFileName, os.O_CREATE|os.O_RDWR, 0o755) + require.NoError(t, err) + defer func() { + require.NoError(t, file.Close()) + }() + + require.NoError(t, fileutil.Preallocate(file, int64(len(testString)), true)) + require.NoError(t, file.Sync()) + + var mappedFile mmap.MMap + mappedFile, err = mmap.MapRegion(file, len(testString), mmap.RDWR, 0, 0) + require.NoError(t, err) + defer func() { + require.NoError(t, mappedFile.Unmap()) + }() + copy(mappedFile, testString) + require.NoError(t, mappedFile.Flush()) + }(t) + + func(t *testing.T) { + testFile, err := os.OpenFile(tempFileName, os.O_CREATE|os.O_RDWR, 0o755) + require.NoError(t, err) + defer func() { + require.NoError(t, testFile.Close()) + }() + + mappedFile, err := mmap.Map(testFile, mmap.RDWR, 0) + require.NoError(t, err) + defer func() { + require.NoError(t, mappedFile.Unmap()) + }() + require.True(t, bytes.Equal(mappedFile, []byte(testString))) + }(t) + + func(t *testing.T) { + testFile, err := os.OpenFile(tempFileName, os.O_CREATE|os.O_RDWR|os.O_TRUNC, 0o755) + require.NoError(t, err) + defer func() { + require.NoError(t, testFile.Close()) + }() + + _, err = testFile.WriteString(testString) + require.NoError(t, err) + + mappedFile, err := mmap.Map(testFile, mmap.RDWR, 0) + require.NoError(t, err) + defer func() { + require.NoError(t, mappedFile.Unmap()) + }() + + require.True(t, bytes.Equal(mappedFile, []byte("lol + kek = chebureck"))) + copy(mappedFile, "lol + lol") + require.NoError(t, mappedFile.Flush()) + require.True(t, bytes.Equal(mappedFile, []byte("lol + lol = chebureck"))) + }(t) +} diff --git a/pp/go/storage/remotewriter/datasource.go b/pp/go/storage/remotewriter/datasource.go new file mode 100644 index 0000000000..2f71e3b3c9 --- /dev/null +++ b/pp/go/storage/remotewriter/datasource.go @@ -0,0 +1,612 @@ +package remotewriter + +import ( + "bytes" + "context" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "sync" + + "github.com/prometheus/client_golang/prometheus" + + "github.com/prometheus/prometheus/model/labels" + "github.com/prometheus/prometheus/model/relabel" + "github.com/prometheus/prometheus/pp/go/cppbridge" + "github.com/prometheus/prometheus/pp/go/logger" + "github.com/prometheus/prometheus/pp/go/storage/catalog" + "github.com/prometheus/prometheus/pp/go/util/optional" +) + +// +// CorruptMarker +// + +// CorruptMarker mark head as corrupted by ID. +type CorruptMarker interface { + // MarkCorrupted mark head as corrupted by ID. + MarkCorrupted(headID string) error +} + +// +// ShardError +// + +// ShardError error reading the shard. +type ShardError struct { + shardID int + processable bool + err error +} + +// NewShardError init new [ShardError]. +func NewShardError(shardID int, processable bool, err error) ShardError { + return ShardError{ + shardID: shardID, + processable: processable, + err: err, + } +} + +// Error returns error as string, implementation error. +func (e ShardError) Error() string { + return e.err.Error() +} + +// ShardID returns shard ID. +func (e ShardError) ShardID() int { + return e.shardID +} + +// Unwrap retruns source error. +func (e ShardError) Unwrap() error { + return e.err +} + +// +// ShardWalReader +// + +// ShardWalReader a shard wall reader. +type ShardWalReader interface { + // Close wal file. + Close() error + + // Read [Segment] from wal and return. + Read() (segment Segment, err error) +} + +// NoOpShardWalReader a shard wall reader, do nothing. +type NoOpShardWalReader struct{} + +// Close implementation [ShardWalReader], do nothing. +func (NoOpShardWalReader) Close() error { return nil } + +// Read implementation [ShardWalReader], do nothing. +func (NoOpShardWalReader) Read() (segment Segment, err error) { return segment, io.EOF } + +// +// shard +// + +type shard struct { + headID string + shardID uint16 + corrupted bool + lastReadSegmentID optional.Optional[uint32] + walReader ShardWalReader + decoder *Decoder + decoderStateFile io.WriteCloser + unexpectedEOFCount prometheus.Counter + segmentSize prometheus.Histogram +} + +// newShard init new [shard]. +// +//revive:disable-next-line:flag-parameter this is a flag, but it's more convenient this way +func newShard( + headID string, + shardID uint16, + shardFileName, decoderStateFileName string, + resetDecoderState bool, + externalLabels labels.Labels, + relabelConfigs []*cppbridge.RelabelConfig, + unexpectedEOFCount prometheus.Counter, + segmentSize prometheus.Histogram, +) (*shard, error) { + wr, encoderVersion, err := newWalReader(shardFileName) + if err != nil { + return nil, fmt.Errorf("failed to create wal file reader: %w", err) + } + + decoder, err := NewDecoder( + externalLabels, + relabelConfigs, + shardID, + encoderVersion, + ) + if err != nil { + return nil, errors.Join(fmt.Errorf("failed to create decoder: %w", err), wr.Close()) + } + + decoderStateFileFlags := os.O_CREATE | os.O_RDWR + if resetDecoderState { + decoderStateFileFlags |= os.O_TRUNC + } + decoderStateFile, err := os.OpenFile( // #nosec G304 // it's meant to be that way + decoderStateFileName, + decoderStateFileFlags, + 0o600, //revive:disable-line:add-constant // file permissions simple readable as octa-number + ) + if err != nil { + return nil, errors.Join(fmt.Errorf("failed to open decoder state file: %w", err), wr.Close()) + } + + // create new shard + s := &shard{ + headID: headID, + shardID: shardID, + walReader: wr, + decoder: decoder, + decoderStateFile: decoderStateFile, + unexpectedEOFCount: unexpectedEOFCount, + segmentSize: segmentSize, + } + + if !resetDecoderState { + if err = decoder.LoadFrom(decoderStateFile); err != nil { + return nil, errors.Join(fmt.Errorf("failed to restore from cache: %w", err), s.Close()) + } + } else { + if err = decoderStateFile.Truncate(0); err != nil { + return nil, errors.Join(fmt.Errorf("failed to truncate decoder state file: %w", err), s.Close()) + } + } + + return s, nil +} + +func (s *shard) Close() error { + return errors.Join(s.walReader.Close(), s.decoderStateFile.Close()) +} + +func (s *shard) Read(ctx context.Context, targetSegmentID uint32, minTimestamp int64) (*DecodedSegment, error) { + if s.corrupted { + return nil, ErrShardIsCorrupted + } + + if !s.lastReadSegmentID.IsNil() && s.lastReadSegmentID.Value() >= targetSegmentID { + return nil, nil + } + + for { + if ctx.Err() != nil { + return nil, ctx.Err() + } + + segment, err := s.walReader.Read() + if err != nil { + s.corrupted = true + logger.Errorf("remotewritedebug shard %s/%d is corrupted by read: %v", s.headID, s.shardID, err) + return nil, errors.Join(err, ErrShardIsCorrupted) + } + + s.segmentSize.Observe(float64(segment.Length())) + + decodedSegment, err := s.decoder.Decode(segment.Bytes(), minTimestamp) + if err != nil { + s.corrupted = true + logger.Errorf("remotewritedebug shard %s/%d is corrupted by decode: %v", s.headID, s.shardID, err) + return nil, errors.Join(err, ErrShardIsCorrupted) + } + + s.lastReadSegmentID.Set(segment.ID) + + if segment.ID == targetSegmentID { + decodedSegment.ID = segment.ID + return decodedSegment, nil + } + } +} + +func (s *shard) SetCorrupted() { + s.corrupted = true +} + +// +// SegmentReadyChecker +// + +// SegmentReadyChecker is a segment ready checker. +type SegmentReadyChecker interface { + // SegmentIsReady checks if the segment is ready. + SegmentIsReady(segmentID uint32) (ready bool, outOfRange bool) +} + +// +// segmentReadyChecker +// + +type segmentReadyChecker struct { + headRecord *catalog.Record +} + +func newSegmentReadyChecker(headRecord *catalog.Record) *segmentReadyChecker { + return &segmentReadyChecker{headRecord: headRecord} +} + +func (src *segmentReadyChecker) SegmentIsReady(segmentID uint32) (ready, outOfRange bool) { + ready = src.headRecord.LastAppendedSegmentID() != nil && *src.headRecord.LastAppendedSegmentID() >= segmentID + outOfRange = (src.headRecord.Status() != catalog.StatusNew && + src.headRecord.Status() != catalog.StatusActive) && + !ready + return ready, outOfRange +} + +// +// shardCache +// + +type shardCache struct { + shardID uint16 + cache *bytes.Buffer + written bool + writer io.Writer +} + +// +// dataSource +// + +type dataSource struct { + ID string + shards []*shard + segmentReadyChecker SegmentReadyChecker + corruptMarker CorruptMarker + closed bool + completed bool + corrupted bool + headReleaseFunc func() + + lssSlice []*cppbridge.LabelSetStorage + + cacheMtx sync.Mutex + caches []*shardCache + cacheWriteSignal chan struct{} + cacheWriteLoopClosed chan struct{} + + unexpectedEOFCount prometheus.Counter + segmentSize prometheus.Histogram +} + +// newDataSource creates a new [dataSource]. +func newDataSource(dataDir string, + numberOfShards uint16, + config DestinationConfig, //nolint:gocritic // hugeParam // config + discardCache bool, + segmentReadyChecker SegmentReadyChecker, + corruptMarker CorruptMarker, + headRecord *catalog.Record, + unexpectedEOFCount prometheus.Counter, + segmentSize prometheus.Histogram, +) (*dataSource, error) { + var err error + var convertedRelabelConfigs []*cppbridge.RelabelConfig + convertedRelabelConfigs, err = convertRelabelConfigs(config.WriteRelabelConfigs...) + if err != nil { + return nil, fmt.Errorf("failed to convert relabel configs: %w", err) + } + + b := &dataSource{ + corruptMarker: corruptMarker, + segmentReadyChecker: segmentReadyChecker, + headReleaseFunc: headRecord.Acquire(), + unexpectedEOFCount: unexpectedEOFCount, + segmentSize: segmentSize, + cacheWriteSignal: make(chan struct{}), + cacheWriteLoopClosed: make(chan struct{}), + } + + go b.cacheWriteLoop() + + for shardID := range numberOfShards { + shardFileName := filepath.Join(dataDir, fmt.Sprintf("shard_%d.wal", shardID)) + decoderStateFileName := filepath.Join(dataDir, fmt.Sprintf("%s_shard_%d.state", config.Name, shardID)) + var s *shard + s, err = createShard( + headRecord.ID(), + shardID, + shardFileName, + decoderStateFileName, + discardCache, + config.ExternalLabels, + convertedRelabelConfigs, + b.unexpectedEOFCount, + b.segmentSize, + ) + if err != nil { + return nil, errors.Join(fmt.Errorf("failed to create shard: %w", err), b.Close()) + } + b.shards = append(b.shards, s) + b.lssSlice = append(b.lssSlice, s.decoder.lss) + b.caches = append(b.caches, &shardCache{ + shardID: shardID, + cache: bytes.NewBuffer(nil), + written: true, + writer: s.decoderStateFile, + }) + } + + return b, nil +} + +func createShard( + headID string, + shardID uint16, + shardFileName, decoderStateFileName string, + resetDecoderState bool, + externalLabels labels.Labels, + relabelConfigs []*cppbridge.RelabelConfig, + unexpectedEOFCount prometheus.Counter, + segmentSize prometheus.Histogram, +) (*shard, error) { + s, err := newShard( + headID, + shardID, + shardFileName, + decoderStateFileName, + resetDecoderState, + externalLabels, + relabelConfigs, + unexpectedEOFCount, + segmentSize, + ) + if err != nil { + logger.Errorf("failed to create shard: %v", err) + return newShard( + headID, + shardID, + shardFileName, + decoderStateFileName, + true, + externalLabels, + relabelConfigs, + unexpectedEOFCount, + segmentSize, + ) + } + return s, nil +} + +func convertRelabelConfigs(relabelConfigs ...*relabel.Config) ([]*cppbridge.RelabelConfig, error) { + convertedConfigs := make([]*cppbridge.RelabelConfig, 0, len(relabelConfigs)) + for _, relabelConfig := range relabelConfigs { + var sourceLabels []string + for _, label := range relabelConfig.SourceLabels { + sourceLabels = append(sourceLabels, string(label)) + } + + convertedConfig := &cppbridge.RelabelConfig{ + SourceLabels: sourceLabels, + Separator: relabelConfig.Separator, + Regex: relabelConfig.Regex.String(), + Modulus: relabelConfig.Modulus, + TargetLabel: relabelConfig.TargetLabel, + Replacement: relabelConfig.Replacement, + Action: cppbridge.ActionNameToValueMap[string(relabelConfig.Action)], + } + + if err := convertedConfig.Validate(); err != nil { + return nil, fmt.Errorf("failed to validate config: %w", err) + } + + convertedConfigs = append(convertedConfigs, convertedConfig) + } + + return convertedConfigs, nil +} + +func (ds *dataSource) Close() error { + if ds.closed { + return nil + } + ds.closed = true + var err error + // stop cache writing first + close(ds.cacheWriteSignal) + <-ds.cacheWriteLoopClosed + + for _, s := range ds.shards { + err = errors.Join(err, s.Close()) + } + ds.headReleaseFunc() + return err +} + +func (ds *dataSource) IsCompleted() bool { + return ds.completed +} + +type readShardResult struct { + segment *DecodedSegment + err error +} + +func (ds *dataSource) Read(ctx context.Context, segmentID uint32, minTimestamp int64) ([]*DecodedSegment, error) { + if ds.completed { + return nil, ErrEndOfBlock + } + + segmentIsReady, segmentIsOutOfRange := ds.segmentReadyChecker.SegmentIsReady(segmentID) + if !segmentIsReady { + if segmentIsOutOfRange { + return nil, ErrEndOfBlock + } + + return nil, ErrEmptyReadResult + } + + wg := &sync.WaitGroup{} + readShardResults := make([]readShardResult, len(ds.shards)) + for shardID := 0; shardID < len(ds.shards); shardID++ { + if ds.shards[shardID].corrupted { + readShardResults[shardID] = readShardResult{ + segment: nil, + err: NewShardError(shardID, false, ErrShardIsCorrupted), + } + continue + } + wg.Add(1) + go func(shardID int) { + defer wg.Done() + segment, err := ds.shards[shardID].Read(ctx, segmentID, minTimestamp) + if err != nil { + err = NewShardError(shardID, true, err) + } + readShardResults[shardID] = readShardResult{segment: segment, err: err} + }(shardID) + } + wg.Wait() + + segments := make([]*DecodedSegment, 0, len(ds.shards)) + errs := make([]error, 0, len(ds.shards)) + for _, result := range readShardResults { + if result.segment != nil { + segments = append(segments, result.segment) + } + if result.err != nil { + errs = append(errs, result.err) + } + } + + return segments, ds.handleReadErrors(errs) +} + +func (ds *dataSource) handleReadErrors(errs []error) error { + if len(errs) == 0 { + return nil + } + + if len(errs) == len(ds.shards) { + ds.corrupted = true + if ds.corruptMarker != nil { + if err := ds.corruptMarker.MarkCorrupted(ds.ID); err != nil { + return fmt.Errorf("failed to mark head corrupted: %w", err) + } + ds.corruptMarker = nil + } + + return ErrEndOfBlock + } + + ds.corrupted = true + if ds.corruptMarker != nil { + if err := ds.corruptMarker.MarkCorrupted(ds.ID); err != nil { + return fmt.Errorf("failed to mark head corrupted: %w", err) + } + ds.corruptMarker = nil + } + + for _, err := range errs { + var shardErr ShardError + if errors.As(err, &shardErr) { + if shardErr.processable { + logger.Errorf("shard %s/%d is corrupted", ds.ID, shardErr.ShardID()) + } + } + } + + return nil +} + +func (ds *dataSource) LSSes() []*cppbridge.LabelSetStorage { + return ds.lssSlice +} + +// WriteCaches writes caches to the buffer and sends the signal to write the caches. +func (ds *dataSource) WriteCaches() { + ds.cacheMtx.Lock() + for shardID, sc := range ds.caches { + if !sc.written { + continue + } + sc.cache.Reset() + if _, err := ds.shards[shardID].decoder.WriteTo(sc.cache); err != nil { + logger.Errorf("failed to get output decoder cache: %v", err) + continue + } + sc.written = false + } + ds.cacheMtx.Unlock() + + select { + case ds.cacheWriteSignal <- struct{}{}: + default: + } +} + +// cacheWriteLoop loop that writes caches to the buffer and sends the signal to write the caches. +func (ds *dataSource) cacheWriteLoop() { + defer close(ds.cacheWriteLoopClosed) + var closed bool + var writeRequested bool + var writeResultc chan struct{} + + for { + if writeRequested && !closed && writeResultc == nil { + writeResultc = make(chan struct{}) + go func() { + defer close(writeResultc) + ds.writeBufferedCaches() + }() + writeRequested = false + } + + if closed && writeResultc == nil { + return + } + + select { + case _, ok := <-ds.cacheWriteSignal: + if !ok { + return + } + writeRequested = true + case <-writeResultc: + writeResultc = nil + } + } +} + +// writeBufferedCaches writes cached data to the disk and resets the cache. +func (ds *dataSource) writeBufferedCaches() { + ds.cacheMtx.Lock() + caches := make([]*shardCache, 0, len(ds.caches)) + for _, sc := range ds.caches { + if sc.written { + continue + } + sc := sc + caches = append(caches, sc) + } + ds.cacheMtx.Unlock() + + writtenCacheShardIDs := make([]uint16, 0, len(caches)) + for _, sc := range caches { + if _, err := sc.cache.WriteTo(sc.writer); err != nil { + logger.Errorf("failed to write cache: %v", err) + continue + } + writtenCacheShardIDs = append(writtenCacheShardIDs, sc.shardID) + } + + if len(writtenCacheShardIDs) > 0 { + ds.cacheMtx.Lock() + for _, shardID := range writtenCacheShardIDs { + ds.caches[shardID].written = true + } + ds.cacheMtx.Unlock() + } +} diff --git a/pp/go/storage/remotewriter/decoder.go b/pp/go/storage/remotewriter/decoder.go new file mode 100644 index 0000000000..fed79236ee --- /dev/null +++ b/pp/go/storage/remotewriter/decoder.go @@ -0,0 +1,103 @@ +package remotewriter + +import ( + "fmt" + "io" + + "github.com/prometheus/prometheus/model/labels" + "github.com/prometheus/prometheus/pp/go/cppbridge" +) + +// +// DecodedSegment +// + +// DecodedSegment the segment decoded from the file [Wal]. +type DecodedSegment struct { + ID uint32 + Samples *cppbridge.DecodedRefSamples + MaxTimestamp int64 + OutdatedSamplesCount uint32 + DroppedSamplesCount uint32 + AddSeriesCount uint32 + DroppedSeriesCount uint32 +} + +// Decoder decodes and relabeling series in segments from a file [Wal]. +// Saves its state in the file for recovery upon restart. +type Decoder struct { + relabeler *cppbridge.StatelessRelabeler + lss *cppbridge.LabelSetStorage + outputDecoder *cppbridge.WALOutputDecoder +} + +// NewDecoder init new [Decoder]. +func NewDecoder( + externalLabels labels.Labels, + relabelConfigs []*cppbridge.RelabelConfig, + shardID uint16, + encoderVersion uint8, +) (*Decoder, error) { + relabeler, err := cppbridge.NewStatelessRelabeler(relabelConfigs) + if err != nil { + return nil, fmt.Errorf("failed to create stateless relabeler: %w", err) + } + + lss := cppbridge.NewLssStorage() + outputDecoder := cppbridge.NewWALOutputDecoder( + LabelsToCppBridgeLabels(externalLabels), + relabeler, + lss, + shardID, + encoderVersion, + ) + + return &Decoder{ + relabeler: relabeler, + lss: lss, + outputDecoder: outputDecoder, + }, nil +} + +// Decode and relabeling series in segments from a file [Wal]. +func (d *Decoder) Decode(segment []byte, minTimestamp int64) (*DecodedSegment, error) { + samples, stats, err := d.outputDecoder.Decode(segment, minTimestamp) + if err != nil { + return nil, err + } + return &DecodedSegment{ + Samples: samples, + MaxTimestamp: stats.MaxTimestamp(), + OutdatedSamplesCount: stats.OutdatedSampleCount(), + DroppedSamplesCount: stats.DroppedSampleCount(), + AddSeriesCount: stats.AddSeriesCount(), + DroppedSeriesCount: stats.DroppedSeriesCount(), + }, nil +} + +// LoadFrom loads the state from a file. +func (d *Decoder) LoadFrom(reader io.Reader) error { + state, err := io.ReadAll(reader) + if err != nil { + return fmt.Errorf("failed to read from reader: %w", err) + } + + return d.outputDecoder.LoadFrom(state) +} + +// WriteTo writes output decoder state to io.Writer. +func (d *Decoder) WriteTo(writer io.Writer) (int64, error) { + return d.outputDecoder.WriteTo(writer) +} + +// LabelsToCppBridgeLabels converts [labels.Labels] to slice [cppbridge.Label]. +func LabelsToCppBridgeLabels(lbls labels.Labels) []cppbridge.Label { + result := make([]cppbridge.Label, 0, lbls.Len()) + lbls.Range(func(l labels.Label) { + result = append(result, cppbridge.Label{ + Name: l.Name, + Value: l.Value, + }) + }) + return result +} diff --git a/pp/go/storage/remotewriter/destination.go b/pp/go/storage/remotewriter/destination.go new file mode 100644 index 0000000000..826a732c9b --- /dev/null +++ b/pp/go/storage/remotewriter/destination.go @@ -0,0 +1,517 @@ +package remotewriter + +import ( + "bytes" + "hash/crc32" + "sync" + "time" + + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/common/model" + "gopkg.in/yaml.v2" + + "github.com/prometheus/prometheus/config" + "github.com/prometheus/prometheus/model/labels" +) + +// String constants for instrumentation. +const ( + namespace = "prometheus" + subsystem = "remote_storage" + remoteName = "remote_name" + endpoint = "url" + + reasonTooOld = "too_old" + reasonDroppedSeries = "dropped_series" + + DefaultSampleAgeLimit = model.Duration(time.Hour * 24 * 30) +) + +// DestinationConfig is a remote write destination config. +type DestinationConfig struct { + config.RemoteWriteConfig + ExternalLabels labels.Labels `yaml:"external_labels"` + ReadTimeout time.Duration +} + +// EqualTo checks if the config is equal to the other config. +// +//nolint:gocritic // hugeParam // equal configs +func (c *DestinationConfig) EqualTo(other DestinationConfig) bool { + return c.ExternalLabels.Hash() == other.ExternalLabels.Hash() && + c.ReadTimeout == other.ReadTimeout && + remoteWriteConfigsAreEqual(c.RemoteWriteConfig, other.RemoteWriteConfig) +} + +// CRC32 returns the CRC32 hash of the config. +func (c *DestinationConfig) CRC32() (uint32, error) { + data, err := yaml.Marshal(c) + if err != nil { + return 0, err + } + + return crc32.ChecksumIEEE(data), nil +} + +// Destination is a remote write destination. +type Destination struct { + config DestinationConfig + metrics *DestinationMetrics +} + +// Config returns current config. +func (d *Destination) Config() DestinationConfig { + return d.config +} + +// ResetConfig resets current config to the new one. +// +//nolint:gocritic // hugeParam // resets config +func (d *Destination) ResetConfig(cfg DestinationConfig) { + d.config = cfg +} + +// NewDestination creates a new [Destination]. +// +//revive:disable-next-line:function-length // this is a constructor +//nolint:gocritic // hugeParam // this is a constructor +func NewDestination(cfg DestinationConfig) *Destination { + constLabels := prometheus.Labels{ + remoteName: cfg.Name, + endpoint: cfg.URL.Redacted(), + } + + return &Destination{ + config: cfg, + metrics: &DestinationMetrics{ + samplesTotal: prometheus.NewCounter(prometheus.CounterOpts{ + Namespace: namespace, + Subsystem: subsystem, + Name: "samples_total", + Help: "Total number of samples sent to remote storage.", + ConstLabels: constLabels, + }), + exemplarsTotal: prometheus.NewCounter(prometheus.CounterOpts{ + Namespace: namespace, + Subsystem: subsystem, + Name: "exemplars_total", + Help: "Total number of exemplars sent to remote storage.", + ConstLabels: constLabels, + }), + histogramsTotal: prometheus.NewCounter(prometheus.CounterOpts{ + Namespace: namespace, + Subsystem: subsystem, + Name: "histograms_total", + Help: "Total number of histograms sent to remote storage.", + ConstLabels: constLabels, + }), + metadataTotal: prometheus.NewCounter(prometheus.CounterOpts{ + Namespace: namespace, + Subsystem: subsystem, + Name: "metadata_total", + Help: "Total number of metadata entries sent to remote storage.", + ConstLabels: constLabels, + }), + failedSamplesTotal: prometheus.NewCounter(prometheus.CounterOpts{ + Namespace: namespace, + Subsystem: subsystem, + Name: "samples_failed_total", + Help: "Total number of samples which failed on send to remote storage, non-recoverable errors.", + ConstLabels: constLabels, + }), + failedExemplarsTotal: prometheus.NewCounter(prometheus.CounterOpts{ + Namespace: namespace, + Subsystem: subsystem, + Name: "exemplars_failed_total", + //revive:disable-next-line:line-length-limit // this is a description of the metric + Help: "Total number of exemplars which failed on send to remote storage, non-recoverable errors.", + ConstLabels: constLabels, + }), + failedHistogramsTotal: prometheus.NewCounter(prometheus.CounterOpts{ + Namespace: namespace, + Subsystem: subsystem, + Name: "histograms_failed_total", + //revive:disable-next-line:line-length-limit // this is a description of the metric + Help: "Total number of histograms which failed on send to remote storage, non-recoverable errors.", + ConstLabels: constLabels, + }), + failedMetadataTotal: prometheus.NewCounter(prometheus.CounterOpts{ + Namespace: namespace, + Subsystem: subsystem, + Name: "metadata_failed_total", + //revive:disable-next-line:line-length-limit // this is a description of the metric + Help: "Total number of metadata entries which failed on send to remote storage, non-recoverable errors.", + ConstLabels: constLabels, + }), + retriedSamplesTotal: prometheus.NewCounter(prometheus.CounterOpts{ + Namespace: namespace, + Subsystem: subsystem, + Name: "samples_retried_total", + //revive:disable-next-line:line-length-limit // this is a description of the metric + Help: "Total number of samples which failed on send to remote storage but were retried because the send error was recoverable.", + ConstLabels: constLabels, + }), + retriedExemplarsTotal: prometheus.NewCounter(prometheus.CounterOpts{ + Namespace: namespace, + Subsystem: subsystem, + Name: "exemplars_retried_total", + //revive:disable-next-line:line-length-limit // this is a description of the metric + Help: "Total number of exemplars which failed on send to remote storage but were retried because the send error was recoverable.", + ConstLabels: constLabels, + }), + retriedHistogramsTotal: prometheus.NewCounter(prometheus.CounterOpts{ + Namespace: namespace, + Subsystem: subsystem, + Name: "histograms_retried_total", + //revive:disable-next-line:line-length-limit // this is a description of the metric + Help: "Total number of histograms which failed on send to remote storage but were retried because the send error was recoverable.", + ConstLabels: constLabels, + }), + retriedMetadataTotal: prometheus.NewCounter(prometheus.CounterOpts{ + Namespace: namespace, + Subsystem: subsystem, + Name: "metadata_retried_total", + //revive:disable-next-line:line-length-limit // this is a description of the metric + Help: "Total number of metadata entries which failed on send to remote storage but were retried because the send error was recoverable.", + ConstLabels: constLabels, + }), + droppedSamplesTotal: prometheus.NewCounterVec(prometheus.CounterOpts{ + Namespace: namespace, + Subsystem: subsystem, + Name: "samples_dropped_total", + //revive:disable-next-line:line-length-limit // this is a description of the metric + Help: "Total number of samples which were dropped after being read from the WAL before being sent via remote write, either via relabelling, due to being too old or unintentionally because of an unknown reference ID.", + ConstLabels: constLabels, + }, []string{"reason"}), + addSeriesTotal: prometheus.NewCounter(prometheus.CounterOpts{ + Namespace: namespace, + Subsystem: subsystem, + Name: "series_added_total", + //revive:disable-next-line:line-length-limit // this is a description of the metric + Help: "Total number of series which were add after being read from the WAL before being sent via remote write, either via relabelling.", + ConstLabels: constLabels, + }), + droppedSeriesTotal: prometheus.NewCounter(prometheus.CounterOpts{ + Namespace: namespace, + Subsystem: subsystem, + Name: "series_dropped_total", + //revive:disable-next-line:line-length-limit // this is a description of the metric + Help: "Total number of series which were dropped after being read from the WAL before being sent via remote write, either via relabelling.", + ConstLabels: constLabels, + }), + droppedExemplarsTotal: prometheus.NewCounterVec(prometheus.CounterOpts{ + Namespace: namespace, + Subsystem: subsystem, + Name: "exemplars_dropped_total", + //revive:disable-next-line:line-length-limit // this is a description of the metric + Help: "Total number of exemplars which were dropped after being read from the WAL before being sent via remote write, either via relabelling, due to being too old or unintentionally because of an unknown reference ID.", + ConstLabels: constLabels, + }, []string{"reason"}), + droppedHistogramsTotal: prometheus.NewCounterVec(prometheus.CounterOpts{ + Namespace: namespace, + Subsystem: subsystem, + Name: "histograms_dropped_total", + //revive:disable-next-line:line-length-limit // this is a description of the metric + Help: "Total number of histograms which were dropped after being read from the WAL before being sent via remote write, either via relabelling, due to being too old or unintentionally because of an unknown reference ID.", + ConstLabels: constLabels, + }, []string{"reason"}), + enqueueRetriesTotal: prometheus.NewCounter(prometheus.CounterOpts{ + Namespace: namespace, + Subsystem: subsystem, + Name: "enqueue_retries_total", + Help: "Total number of times enqueue has failed because a shards queue was full.", + ConstLabels: constLabels, + }), + sentBatchDuration: prometheus.NewHistogram(prometheus.HistogramOpts{ + Namespace: namespace, + Subsystem: subsystem, + Name: "sent_batch_duration_seconds", + Help: "Duration of send calls to the remote storage.", + Buckets: append(prometheus.DefBuckets, 25, 60, 120, 300), + ConstLabels: constLabels, + NativeHistogramBucketFactor: 1.1, + NativeHistogramMaxBucketNumber: 100, + NativeHistogramMinResetDuration: 1 * time.Hour, + }), + highestSentTimestamp: &maxTimestamp{ + Gauge: prometheus.NewGauge(prometheus.GaugeOpts{ + Namespace: namespace, + Subsystem: subsystem, + Name: "queue_highest_sent_timestamp_seconds", + //revive:disable-next-line:line-length-limit // this is a description of the metric + Help: "Timestamp from a WAL sample, the highest timestamp successfully sent by this queue, in seconds since epoch.", + ConstLabels: constLabels, + }), + }, + pendingSamples: prometheus.NewGauge(prometheus.GaugeOpts{ + Namespace: namespace, + Subsystem: subsystem, + Name: "samples_pending", + Help: "The number of samples pending in the queues shards to be sent to the remote storage.", + ConstLabels: constLabels, + }), + pendingExemplars: prometheus.NewGauge(prometheus.GaugeOpts{ + Namespace: namespace, + Subsystem: subsystem, + Name: "exemplars_pending", + Help: "The number of exemplars pending in the queues shards to be sent to the remote storage.", + ConstLabels: constLabels, + }), + pendingHistograms: prometheus.NewGauge(prometheus.GaugeOpts{ + Namespace: namespace, + Subsystem: subsystem, + Name: "histograms_pending", + Help: "The number of histograms pending in the queues shards to be sent to the remote storage.", + ConstLabels: constLabels, + }), + shardCapacity: prometheus.NewGauge(prometheus.GaugeOpts{ + Namespace: namespace, + Subsystem: subsystem, + Name: "shard_capacity", + Help: "The capacity of each shard of the queue used for parallel sending to the remote storage.", + ConstLabels: constLabels, + }), + numShards: prometheus.NewGauge(prometheus.GaugeOpts{ + Namespace: namespace, + Subsystem: subsystem, + Name: "shards", + Help: "The number of shards used for parallel sending to the remote storage.", + ConstLabels: constLabels, + }), + maxNumShards: prometheus.NewGauge(prometheus.GaugeOpts{ + Namespace: namespace, + Subsystem: subsystem, + Name: "shards_max", + Help: "The maximum number of shards that the queue is allowed to run.", + ConstLabels: constLabels, + }), + minNumShards: prometheus.NewGauge(prometheus.GaugeOpts{ + Namespace: namespace, + Subsystem: subsystem, + Name: "shards_min", + Help: "The minimum number of shards that the queue is allowed to run.", + ConstLabels: constLabels, + }), + desiredNumShards: prometheus.NewGauge(prometheus.GaugeOpts{ + Namespace: namespace, + Subsystem: subsystem, + Name: "shards_desired", + //revive:disable-next-line:line-length-limit // this is a description of the metric + Help: "The number of shards that the queues shard calculation wants to run based on the rate of samples in vs. samples out.", + ConstLabels: constLabels, + }), + bestNumShards: prometheus.NewGauge(prometheus.GaugeOpts{ + Namespace: namespace, + Subsystem: subsystem, + Name: "shards_best", + Help: "The number of shards that are calculated from the actual number of accumulated segments.", + ConstLabels: constLabels, + }), + sentBytesTotal: prometheus.NewCounter(prometheus.CounterOpts{ + Namespace: namespace, + Subsystem: subsystem, + Name: "bytes_total", + //revive:disable-next-line:line-length-limit // this is a description of the metric + Help: "The total number of bytes of data (not metadata) sent by the queue after compression. Note that when exemplars over remote write is enabled the exemplars included in a remote write request count towards this metric.", + ConstLabels: constLabels, + }), + metadataBytesTotal: prometheus.NewCounter(prometheus.CounterOpts{ + Namespace: namespace, + Subsystem: subsystem, + Name: "metadata_bytes_total", + Help: "The total number of bytes of metadata sent by the queue after compression.", + ConstLabels: constLabels, + }), + maxSamplesPerSend: prometheus.NewGauge(prometheus.GaugeOpts{ + Namespace: namespace, + Subsystem: subsystem, + Name: "max_samples_per_send", + //revive:disable-next-line:line-length-limit // this is a description of the metric + Help: "The maximum number of samples to be sent, in a single request, to the remote storage. Note that, when sending of exemplars over remote write is enabled, exemplars count towards this limt.", + ConstLabels: constLabels, + }), + unexpectedEOFCount: prometheus.NewCounter( + prometheus.CounterOpts{ + Namespace: namespace, + Subsystem: subsystem, + Name: "unexpected_eof_count", + Help: "Number of eof occurred during reading active head wal", + ConstLabels: constLabels, + }, + ), + segmentSizeInBytes: prometheus.NewHistogram( + prometheus.HistogramOpts{ + Namespace: namespace, + Subsystem: subsystem, + Name: "segment_size_bytes", + Help: "Size of segment.", + ConstLabels: constLabels, + Buckets: []float64{ + 1 << 10, 1 << 11, 1 << 12, 1 << 13, 1 << 14, 1 << 15, + 1 << 16, 1 << 17, 1 << 18, 1 << 19, 1 << 20, + }, + }, + ), + }, + } +} + +// RegisterMetrics registers the metrics for the [Destination]. +// +//nolint:dupl // it's not duplicate, it's different function +//revive:disable-next-line:function-length // register metrics +func (d *Destination) RegisterMetrics(registerer prometheus.Registerer) { + registerer.MustRegister(d.metrics.samplesTotal) + registerer.MustRegister(d.metrics.exemplarsTotal) + registerer.MustRegister(d.metrics.histogramsTotal) + registerer.MustRegister(d.metrics.metadataTotal) + registerer.MustRegister(d.metrics.failedSamplesTotal) + registerer.MustRegister(d.metrics.failedExemplarsTotal) + registerer.MustRegister(d.metrics.failedHistogramsTotal) + registerer.MustRegister(d.metrics.failedMetadataTotal) + registerer.MustRegister(d.metrics.retriedSamplesTotal) + registerer.MustRegister(d.metrics.retriedExemplarsTotal) + registerer.MustRegister(d.metrics.retriedHistogramsTotal) + registerer.MustRegister(d.metrics.retriedMetadataTotal) + registerer.MustRegister(d.metrics.droppedSamplesTotal) + registerer.MustRegister(d.metrics.addSeriesTotal) + registerer.MustRegister(d.metrics.droppedSeriesTotal) + registerer.MustRegister(d.metrics.droppedExemplarsTotal) + registerer.MustRegister(d.metrics.droppedHistogramsTotal) + registerer.MustRegister(d.metrics.enqueueRetriesTotal) + registerer.MustRegister(d.metrics.sentBatchDuration) + registerer.MustRegister(d.metrics.highestSentTimestamp) + registerer.MustRegister(d.metrics.pendingSamples) + registerer.MustRegister(d.metrics.pendingExemplars) + registerer.MustRegister(d.metrics.pendingHistograms) + registerer.MustRegister(d.metrics.shardCapacity) + registerer.MustRegister(d.metrics.numShards) + registerer.MustRegister(d.metrics.maxNumShards) + registerer.MustRegister(d.metrics.minNumShards) + registerer.MustRegister(d.metrics.desiredNumShards) + registerer.MustRegister(d.metrics.bestNumShards) + registerer.MustRegister(d.metrics.sentBytesTotal) + registerer.MustRegister(d.metrics.metadataBytesTotal) + registerer.MustRegister(d.metrics.maxSamplesPerSend) + registerer.MustRegister(d.metrics.unexpectedEOFCount) + registerer.MustRegister(d.metrics.segmentSizeInBytes) +} + +// UnregisterMetrics unregisters the metrics for the [Destination]. +// +//nolint:dupl // it's not duplicate, it's different function +//revive:disable-next-line:function-length // register metrics +func (d *Destination) UnregisterMetrics(registerer prometheus.Registerer) { + registerer.Unregister(d.metrics.samplesTotal) + registerer.Unregister(d.metrics.exemplarsTotal) + registerer.Unregister(d.metrics.histogramsTotal) + registerer.Unregister(d.metrics.metadataTotal) + registerer.Unregister(d.metrics.failedSamplesTotal) + registerer.Unregister(d.metrics.failedExemplarsTotal) + registerer.Unregister(d.metrics.failedHistogramsTotal) + registerer.Unregister(d.metrics.failedMetadataTotal) + registerer.Unregister(d.metrics.retriedSamplesTotal) + registerer.Unregister(d.metrics.retriedExemplarsTotal) + registerer.Unregister(d.metrics.retriedHistogramsTotal) + registerer.Unregister(d.metrics.retriedMetadataTotal) + registerer.Unregister(d.metrics.droppedSamplesTotal) + registerer.Unregister(d.metrics.addSeriesTotal) + registerer.Unregister(d.metrics.droppedSeriesTotal) + registerer.Unregister(d.metrics.droppedExemplarsTotal) + registerer.Unregister(d.metrics.droppedHistogramsTotal) + registerer.Unregister(d.metrics.enqueueRetriesTotal) + registerer.Unregister(d.metrics.sentBatchDuration) + registerer.Unregister(d.metrics.highestSentTimestamp) + registerer.Unregister(d.metrics.pendingSamples) + registerer.Unregister(d.metrics.pendingExemplars) + registerer.Unregister(d.metrics.pendingHistograms) + registerer.Unregister(d.metrics.shardCapacity) + registerer.Unregister(d.metrics.numShards) + registerer.Unregister(d.metrics.maxNumShards) + registerer.Unregister(d.metrics.minNumShards) + registerer.Unregister(d.metrics.desiredNumShards) + registerer.Unregister(d.metrics.bestNumShards) + registerer.Unregister(d.metrics.sentBytesTotal) + registerer.Unregister(d.metrics.metadataBytesTotal) + registerer.Unregister(d.metrics.maxSamplesPerSend) + registerer.Unregister(d.metrics.unexpectedEOFCount) + registerer.Unregister(d.metrics.segmentSizeInBytes) +} + +// remoteWriteConfigsAreEqual compares two remote write configs. +// +//nolint:gocritic // this is a compares configs +func remoteWriteConfigsAreEqual(lrwc, rwrc config.RemoteWriteConfig) bool { + ldata, _ := yaml.Marshal(lrwc) + rdata, _ := yaml.Marshal(rwrc) + return bytes.Equal(ldata, rdata) +} + +// maxTimestamp is a metric for the highest sent timestamp. +type maxTimestamp struct { + mtx sync.Mutex + value float64 + prometheus.Gauge +} + +// Set sets the value of the metric. +func (m *maxTimestamp) Set(value float64) { + m.mtx.Lock() + defer m.mtx.Unlock() + if value > m.value { + m.value = value + m.Gauge.Set(value) + } +} + +// Get gets the value of the metric. +func (m *maxTimestamp) Get() float64 { + m.mtx.Lock() + defer m.mtx.Unlock() + return m.value +} + +// Collect collects the metric. +func (m *maxTimestamp) Collect(c chan<- prometheus.Metric) { + if m.Get() > 0 { + m.Gauge.Collect(c) + } +} + +// DestinationMetrics is container for the metrics of the [Destination]. +type DestinationMetrics struct { + samplesTotal prometheus.Counter + exemplarsTotal prometheus.Counter + histogramsTotal prometheus.Counter + metadataTotal prometheus.Counter + failedSamplesTotal prometheus.Counter + failedExemplarsTotal prometheus.Counter + failedHistogramsTotal prometheus.Counter + failedMetadataTotal prometheus.Counter + retriedSamplesTotal prometheus.Counter + retriedExemplarsTotal prometheus.Counter + retriedHistogramsTotal prometheus.Counter + retriedMetadataTotal prometheus.Counter + droppedSamplesTotal *prometheus.CounterVec + addSeriesTotal prometheus.Counter + droppedSeriesTotal prometheus.Counter + droppedExemplarsTotal *prometheus.CounterVec + droppedHistogramsTotal *prometheus.CounterVec + enqueueRetriesTotal prometheus.Counter + sentBatchDuration prometheus.Histogram + highestSentTimestamp *maxTimestamp + pendingSamples prometheus.Gauge + pendingExemplars prometheus.Gauge + pendingHistograms prometheus.Gauge + shardCapacity prometheus.Gauge + numShards prometheus.Gauge + maxNumShards prometheus.Gauge + minNumShards prometheus.Gauge + desiredNumShards prometheus.Gauge + bestNumShards prometheus.Gauge + sentBytesTotal prometheus.Counter + metadataBytesTotal prometheus.Counter + maxSamplesPerSend prometheus.Gauge + unexpectedEOFCount prometheus.Counter + segmentSizeInBytes prometheus.Histogram +} diff --git a/pp/go/storage/remotewriter/errors.go b/pp/go/storage/remotewriter/errors.go new file mode 100644 index 0000000000..7f47e172d0 --- /dev/null +++ b/pp/go/storage/remotewriter/errors.go @@ -0,0 +1,26 @@ +package remotewriter + +import ( + "errors" + "io" +) + +var ( + // ErrShardIsCorrupted error when the shard file was corrupted. + ErrShardIsCorrupted = errors.New("shard is corrupted") + // ErrEndOfBlock error indicating the end of the block. + ErrEndOfBlock = errors.New("end of block") + // ErrEmptyReadResult an error indicating an empty reading result. + ErrEmptyReadResult = errors.New("empty read result") +) + +// CloseAll closes all given closers. +func CloseAll(closers ...io.Closer) error { + var err error + for _, closer := range closers { + if closer != nil { + err = errors.Join(err, closer.Close()) + } + } + return err +} diff --git a/pp/go/storage/remotewriter/iterator.go b/pp/go/storage/remotewriter/iterator.go new file mode 100644 index 0000000000..1f4a11c23f --- /dev/null +++ b/pp/go/storage/remotewriter/iterator.go @@ -0,0 +1,528 @@ +package remotewriter + +import ( + "context" + "errors" + "fmt" + "math" + "sync" + "time" + + "github.com/cenkalti/backoff/v4" + "github.com/jonboulle/clockwork" + + "github.com/prometheus/prometheus/config" + "github.com/prometheus/prometheus/storage/remote" + + "github.com/prometheus/prometheus/pp/go/cppbridge" + "github.com/prometheus/prometheus/pp/go/logger" +) + +//go:generate -command moq go run github.com/matryer/moq --rm --skip-ensure --pkg mock --out +//go:generate moq mock/protobuf_writer.go . ProtobufWriter + +// +// DataSource +// + +// DataSource is a implementation of data source. +type DataSource interface { + Read(ctx context.Context, targetSegmentID uint32, minTimestamp int64) ([]*DecodedSegment, error) + LSSes() []*cppbridge.LabelSetStorage + WriteCaches() + Close() error +} + +// +// TargetSegmentIDSetCloser +// + +// TargetSegmentIDSetCloser is a implementation of target segment id set closer. +type TargetSegmentIDSetCloser interface { + SetTargetSegmentID(segmentID uint32) error + Close() error +} + +// +// ProtobufWriter +// + +// ProtobufWriter is a implementation of protobuf writer. +type ProtobufWriter interface { + Write(ctx context.Context, data *cppbridge.SnappyProtobufEncodedData) error +} + +type sharder struct { + min int + max int + numberOfShards int +} + +// newSharder creates a new [sharder]. +func newSharder(minShards, maxShards int) (*sharder, error) { + if minShards > maxShards || minShards <= 0 { + return nil, fmt.Errorf("failed to create sharder, min: %d, max: %d", minShards, maxShards) + } + return &sharder{ + min: minShards, + max: maxShards, + numberOfShards: minShards, + }, nil +} + +// Apply applies the value for the number of shards to the sharder. +func (s *sharder) Apply(value float64) { + newValue := int(math.Ceil(value)) + if newValue < s.min { + newValue = s.min + } else if newValue > s.max { + newValue = s.max + } + + s.numberOfShards = newValue +} + +// BestNumberOfShards clamping value between min and max. +func (s *sharder) BestNumberOfShards(value float64) int { + newValue := int(math.Ceil(value)) + if newValue < s.min { + newValue = s.min + } else if newValue > s.max { + newValue = s.max + } + + return newValue +} + +// NumberOfShards returns the number of shards. +func (s *sharder) NumberOfShards() int { + return s.numberOfShards +} + +// Iterator is a iterator for sending data to the remote storage. +type Iterator struct { + clock clockwork.Clock + queueConfig config.QueueConfig + dataSource DataSource + protobufWriter ProtobufWriter + targetSegmentIDSetCloser TargetSegmentIDSetCloser + metrics *DestinationMetrics + targetSegmentID uint32 + targetSegmentIsPartiallyRead bool + + outputSharder *sharder + + scrapeInterval time.Duration + endOfBlockReached bool +} + +// MessageShard is a shard of a message for sending to the remote storage. +type MessageShard struct { + Protobuf *cppbridge.SnappyProtobufEncodedData + Size uint64 + SampleCount uint64 + MaxTimestamp int64 + Delivered bool + PostProcessed bool +} + +// Message is a message for sending to the remote storage. +type Message struct { + MaxTimestamp int64 + Shards []*MessageShard +} + +// HasDataToDeliver checks if the message has data to deliver. +func (m *Message) HasDataToDeliver() bool { + for _, shrd := range m.Shards { + if !shrd.Delivered { + return true + } + } + + return false +} + +// IsObsoleted checks if the message is obsoleted. +func (m *Message) IsObsoleted(minTimestamp int64) bool { + return m.MaxTimestamp < minTimestamp +} + +// newIterator creates a new [Iterator]. +func newIterator( + clock clockwork.Clock, + queueConfig config.QueueConfig, + dataSource DataSource, + targetSegmentIDSetCloser TargetSegmentIDSetCloser, + targetSegmentID uint32, + readTimeout time.Duration, + protobufWriter ProtobufWriter, + metrics *DestinationMetrics, +) (*Iterator, error) { + outputSharder, err := newSharder(queueConfig.MinShards, queueConfig.MaxShards) + if err != nil { + return nil, err + } + + return &Iterator{ + clock: clock, + queueConfig: queueConfig, + dataSource: dataSource, + protobufWriter: protobufWriter, + targetSegmentIDSetCloser: targetSegmentIDSetCloser, + metrics: metrics, + targetSegmentID: targetSegmentID, + scrapeInterval: readTimeout, + outputSharder: outputSharder, + }, nil +} + +// wrapError wraps the error. +func (i *Iterator) wrapError(err error) error { + if err != nil { + return err + } + + if i.endOfBlockReached { + return ErrEndOfBlock + } + + return nil +} + +// Next reads data from the data source and writes it to the protobuf writer. +// +//revive:disable-next-line:function-length // long but readable +//revive:disable-next-line:cyclomatic // long but readable +//revive:disable-next-line:cognitive-complexity // long but readable +func (i *Iterator) Next(ctx context.Context) error { + if i.endOfBlockReached { + return i.wrapError(nil) + } + + startTime := i.clock.Now() + var deadlineReached bool + var delay time.Duration + numberOfShards := i.outputSharder.NumberOfShards() + i.metrics.numShards.Set(float64(numberOfShards)) + b := newBatch(numberOfShards, i.queueConfig.MaxSamplesPerSend) + deadline := i.clock.After(i.scrapeInterval) + +readLoop: + for { + select { + case <-ctx.Done(): + return i.wrapError(ctx.Err()) + case <-deadline: + deadlineReached = true + break readLoop + case <-i.clock.After(delay): + } + + decodedSegments, err := i.dataSource.Read(ctx, i.targetSegmentID, i.minTimestamp()) + if err != nil { + if errors.Is(err, ErrEndOfBlock) { + i.endOfBlockReached = true + break readLoop + } + + if errors.Is(err, ErrEmptyReadResult) { + delay = defaultDelay + continue + } + + logger.Errorf("datasource read failed: %v", err) + delay = defaultDelay + continue + } + + b.add(decodedSegments) + i.targetSegmentID++ + i.targetSegmentIsPartiallyRead = false + + if b.IsFilled() { + break readLoop + } + + delay = 0 + } + + readDuration := i.clock.Since(startTime) + + if b.HasDroppedSamples() { + i.metrics.droppedSamplesTotal.WithLabelValues(reasonTooOld).Add(float64(b.OutdatedSamplesCount())) + i.metrics.droppedSamplesTotal.WithLabelValues(reasonDroppedSeries).Add(float64(b.DroppedSamplesCount())) + } + + i.metrics.droppedSeriesTotal.Add(float64(b.DroppedSeriesCount())) + + if b.IsEmpty() { + return i.wrapError(nil) + } + + i.metrics.addSeriesTotal.Add(float64(b.AddSeriesCount())) + + var desiredNumberOfShards float64 + if deadlineReached { + desiredNumberOfShards = math.Ceil( + float64(b.NumberOfSamples()) / float64(b.MaxNumberOfSamplesPerShard()) * float64(numberOfShards), + ) + } else { + desiredNumberOfShards = math.Ceil(float64(i.scrapeInterval) / float64(readDuration) * float64(numberOfShards)) + } + + bestNumberOfShards := i.outputSharder.BestNumberOfShards( + float64(b.NumberOfSamples()) / float64(b.MaxNumberOfSamplesPerShard()), + ) + + i.outputSharder.Apply(desiredNumberOfShards) + i.metrics.desiredNumShards.Set(desiredNumberOfShards) + i.metrics.bestNumShards.Set(float64(bestNumberOfShards)) + + i.writeCaches() + + msg, err := i.encode(b.Data(), uint16(bestNumberOfShards)) // #nosec G115 // no overflow + if err != nil { + return i.wrapError(err) + } + + numberOfSamples := b.NumberOfSamples() + + b = nil + + sendIteration := 0 + err = backoff.Retry(func() error { + defer func() { sendIteration++ }() + if msg.IsObsoleted(i.minTimestamp()) { + for _, messageShard := range msg.Shards { + if messageShard.Delivered { + continue + } + i.metrics.droppedSamplesTotal.WithLabelValues(reasonTooOld).Add(float64(messageShard.SampleCount)) + } + return nil + } + + i.metrics.samplesTotal.Add(float64(numberOfSamples)) + + wg := &sync.WaitGroup{} + for _, shrd := range msg.Shards { + if shrd.Delivered { + continue + } + wg.Add(1) + go func(shrd *MessageShard) { + defer wg.Done() + begin := i.clock.Now() + writeErr := i.protobufWriter.Write(ctx, shrd.Protobuf) + i.metrics.sentBatchDuration.Observe(i.clock.Since(begin).Seconds()) + if writeErr != nil { + logger.Errorf("failed to send protobuf: %v", writeErr) + } + + shrd.Delivered = !errors.Is(writeErr, remote.RecoverableError{}) + }(shrd) + } + wg.Wait() + + var failedSamplesTotal uint64 + var sentBytesTotal uint64 + var highestSentTimestamp int64 + var retriedSamplesTotal uint64 + for _, shrd := range msg.Shards { + if shrd.Delivered { + if shrd.PostProcessed { + continue + } + // delivered on this iteration + shrd.PostProcessed = true + retriedSamplesTotal += shrd.SampleCount + sentBytesTotal += shrd.Size + if highestSentTimestamp < shrd.MaxTimestamp { + highestSentTimestamp = shrd.MaxTimestamp + } + continue + } + // delivery failed bool + retriedSamplesTotal += shrd.SampleCount + failedSamplesTotal += shrd.SampleCount + } + + i.metrics.failedSamplesTotal.Add(float64(failedSamplesTotal)) + i.metrics.sentBytesTotal.Add(float64(sentBytesTotal)) + i.metrics.highestSentTimestamp.Set(float64(highestSentTimestamp)) + + if sendIteration > 0 { + i.metrics.retriedSamplesTotal.Add(float64(retriedSamplesTotal)) + } + + if msg.HasDataToDeliver() { + return errors.New("not all data delivered") + } + + return nil + }, + backoff.WithContext( + backoff.NewExponentialBackOff( + backoff.WithClockProvider(i.clock), + backoff.WithMaxElapsedTime(0), + backoff.WithMaxInterval(i.scrapeInterval), + ), + ctx, + ), + ) + if err != nil { + return i.wrapError(err) + } + + if err = i.tryAck(ctx); err != nil { + logger.Errorf("failed to ack segment id: %v", err) + } + + return i.wrapError(nil) +} + +func (i *Iterator) writeCaches() { + i.dataSource.WriteCaches() +} + +func (i *Iterator) encode(segments []*DecodedSegment, numberOfShards uint16) (*Message, error) { + var maxTimestamp int64 + batchToEncode := make([]*cppbridge.DecodedRefSamples, 0, len(segments)) + for _, segment := range segments { + if maxTimestamp < segment.MaxTimestamp { + maxTimestamp = segment.MaxTimestamp + } + + batchToEncode = append(batchToEncode, segment.Samples) + } + + protobufEncoder := cppbridge.NewWALProtobufEncoder(i.dataSource.LSSes()) + protobufs, err := protobufEncoder.Encode(batchToEncode, numberOfShards) + if err != nil { + return nil, fmt.Errorf("failed to encode protobuf: %w", err) + } + shards := make([]*MessageShard, 0, len(protobufs)) + for _, protobuf := range protobufs { + proto := protobuf + var size uint64 + _ = proto.Do(func(buf []byte) error { + size = uint64(len(buf)) + return nil + }) + shards = append(shards, &MessageShard{ + Protobuf: protobuf, + Size: size, + SampleCount: protobuf.SamplesCount(), + MaxTimestamp: protobuf.MaxTimestamp(), + Delivered: false, + }) + } + return &Message{ + MaxTimestamp: maxTimestamp, + Shards: shards, + }, nil +} + +func (i *Iterator) tryAck(_ context.Context) error { + if i.targetSegmentID == 0 && i.targetSegmentIsPartiallyRead { + return nil + } + + targetSegmentID := i.targetSegmentID + if i.targetSegmentIsPartiallyRead { + targetSegmentID-- + } + + if err := i.targetSegmentIDSetCloser.SetTargetSegmentID(targetSegmentID); err != nil { + return fmt.Errorf("failed to set target segment id: %w", err) + } + + return nil +} + +func (i *Iterator) minTimestamp() int64 { + sampleAgeLimit := time.Duration(i.queueConfig.SampleAgeLimit) + return i.clock.Now().Add(-sampleAgeLimit).UnixMilli() +} + +// Close closes the iterator. +func (i *Iterator) Close() error { + return errors.Join(i.dataSource.Close(), i.targetSegmentIDSetCloser.Close()) +} + +type batch struct { + segments []*DecodedSegment + numberOfShards int + numberOfSamples int + outdatedSamplesCount uint32 + droppedSamplesCount uint32 + addSeriesCount uint32 + droppedSeriesCount uint32 + maxNumberOfSamplesPerShard int +} + +// newBatch creates a new [batch]. +func newBatch(numberOfShards, maxNumberOfSamplesPerShard int) *batch { + return &batch{ + numberOfShards: numberOfShards, + maxNumberOfSamplesPerShard: maxNumberOfSamplesPerShard, + } +} + +func (b *batch) add(segments []*DecodedSegment) { + for _, segment := range segments { + b.segments = append(b.segments, segment) + b.numberOfSamples += segment.Samples.Size() + b.outdatedSamplesCount += segment.OutdatedSamplesCount + b.droppedSamplesCount += segment.DroppedSamplesCount + b.addSeriesCount += segment.AddSeriesCount + b.droppedSeriesCount += segment.DroppedSeriesCount + } +} + +func (b *batch) IsFilled() bool { + return b.numberOfSamples > b.numberOfShards*b.maxNumberOfSamplesPerShard +} + +func (b *batch) IsEmpty() bool { + return b.numberOfSamples == 0 +} + +func (b *batch) HasDroppedSamples() bool { + return b.droppedSamplesCount > 0 || b.outdatedSamplesCount > 0 +} + +func (b *batch) OutdatedSamplesCount() uint32 { + return b.outdatedSamplesCount +} + +func (b *batch) DroppedSamplesCount() uint32 { + return b.droppedSamplesCount +} + +// AddSeriesCount number of add series. +func (b *batch) AddSeriesCount() uint32 { + return b.addSeriesCount +} + +// DroppedSeriesCount number of dropped series. +func (b *batch) DroppedSeriesCount() uint32 { + return b.droppedSeriesCount +} + +func (b *batch) NumberOfSamples() int { + return b.numberOfSamples +} + +func (b *batch) MaxNumberOfSamplesPerShard() int { + return b.maxNumberOfSamplesPerShard +} + +func (b *batch) NumberOfShards() int { + return b.numberOfShards +} + +func (b *batch) Data() []*DecodedSegment { + return b.segments +} diff --git a/pp/go/storage/remotewriter/mock/protobuf_writer.go b/pp/go/storage/remotewriter/mock/protobuf_writer.go new file mode 100644 index 0000000000..537c20b3cc --- /dev/null +++ b/pp/go/storage/remotewriter/mock/protobuf_writer.go @@ -0,0 +1,78 @@ +// Code generated by moq; DO NOT EDIT. +// github.com/matryer/moq + +package mock + +import ( + "context" + "github.com/prometheus/prometheus/pp/go/cppbridge" + "sync" +) + +// ProtobufWriterMock is a mock implementation of remotewriter.ProtobufWriter. +// +// func TestSomethingThatUsesProtobufWriter(t *testing.T) { +// +// // make and configure a mocked remotewriter.ProtobufWriter +// mockedProtobufWriter := &ProtobufWriterMock{ +// WriteFunc: func(ctx context.Context, data *cppbridge.SnappyProtobufEncodedData) error { +// panic("mock out the Write method") +// }, +// } +// +// // use mockedProtobufWriter in code that requires remotewriter.ProtobufWriter +// // and then make assertions. +// +// } +type ProtobufWriterMock struct { + // WriteFunc mocks the Write method. + WriteFunc func(ctx context.Context, data *cppbridge.SnappyProtobufEncodedData) error + + // calls tracks calls to the methods. + calls struct { + // Write holds details about calls to the Write method. + Write []struct { + // Ctx is the ctx argument value. + Ctx context.Context + // Data is the data argument value. + Data *cppbridge.SnappyProtobufEncodedData + } + } + lockWrite sync.RWMutex +} + +// Write calls WriteFunc. +func (mock *ProtobufWriterMock) Write(ctx context.Context, data *cppbridge.SnappyProtobufEncodedData) error { + if mock.WriteFunc == nil { + panic("ProtobufWriterMock.WriteFunc: method is nil but ProtobufWriter.Write was just called") + } + callInfo := struct { + Ctx context.Context + Data *cppbridge.SnappyProtobufEncodedData + }{ + Ctx: ctx, + Data: data, + } + mock.lockWrite.Lock() + mock.calls.Write = append(mock.calls.Write, callInfo) + mock.lockWrite.Unlock() + return mock.WriteFunc(ctx, data) +} + +// WriteCalls gets all the calls that were made to Write. +// Check the length with: +// +// len(mockedProtobufWriter.WriteCalls()) +func (mock *ProtobufWriterMock) WriteCalls() []struct { + Ctx context.Context + Data *cppbridge.SnappyProtobufEncodedData +} { + var calls []struct { + Ctx context.Context + Data *cppbridge.SnappyProtobufEncodedData + } + mock.lockWrite.RLock() + calls = mock.calls.Write + mock.lockWrite.RUnlock() + return calls +} diff --git a/pp/go/storage/remotewriter/remotewriter.go b/pp/go/storage/remotewriter/remotewriter.go new file mode 100644 index 0000000000..1d88242db4 --- /dev/null +++ b/pp/go/storage/remotewriter/remotewriter.go @@ -0,0 +1,162 @@ +package remotewriter + +import ( + "context" + "fmt" + "time" + + "github.com/jonboulle/clockwork" + "github.com/prometheus/client_golang/prometheus" + + "github.com/prometheus/prometheus/pp/go/storage/catalog" + "github.com/prometheus/prometheus/pp/go/storage/ready" +) + +// +// Catalog +// + +// Catalog of current head records. +type Catalog interface { + // List returns slice of records with filter and sort. + List( + filterFn func(record *catalog.Record) bool, + sortLess func(lhs, rhs *catalog.Record) bool, + ) []*catalog.Record + + // SetCorrupted set corrupted flag for ID and returns [catalog.Record] if exist. + SetCorrupted(id string) (*catalog.Record, error) +} + +// RemoteWriter sent samples to the remote write storage. +type RemoteWriter struct { + dataDir string + configQueue chan []DestinationConfig + catalog Catalog + clock clockwork.Clock + readyNotifier ready.Notifier + registerer prometheus.Registerer +} + +// New init new [RemoteWriter]. +func New( + dataDir string, + hcatalog Catalog, + clock clockwork.Clock, + readyNotifier ready.Notifier, + registerer prometheus.Registerer, +) *RemoteWriter { + return &RemoteWriter{ + dataDir: dataDir, + catalog: hcatalog, + clock: clock, + configQueue: make(chan []DestinationConfig), + readyNotifier: readyNotifier, + registerer: registerer, + } +} + +// ApplyConfig updates the state as the new config requires. +func (rw *RemoteWriter) ApplyConfig(configs ...DestinationConfig) (err error) { + select { + case rw.configQueue <- configs: + return nil + case <-time.After(time.Minute): + return fmt.Errorf("failed to apply remote write configs, timeout") + } +} + +// Run sending data via RemoteWriter. +// +//revive:disable-next-line:cyclomatic but readable +//revive:disable-next-line:function-length long but readable +//revive:disable-next-line:cognitive-complexity function is not complicated. +func (rw *RemoteWriter) Run(ctx context.Context) error { + writeLoopRunners := make(map[string]*writeLoopRunner) + defer func() { + for _, wlr := range writeLoopRunners { + wlr.stop() + } + }() + + destinations := make(map[string]*Destination) + + for { + select { + case <-ctx.Done(): + return nil + case configs := <-rw.configQueue: + destinationConfigs := make(map[string]DestinationConfig) + for i := range configs { + destinationConfigs[configs[i].Name] = configs[i] + } + + for _, destination := range destinations { + name := destination.Config().Name + if _, ok := destinationConfigs[name]; !ok { + wlr := writeLoopRunners[name] + wlr.stop() + destination.UnregisterMetrics(rw.registerer) + delete(destinations, name) + delete(writeLoopRunners, name) + } + } + + for _, config := range configs { //nolint:gocritic // hugeParam // constructor + destination, ok := destinations[config.Name] + if !ok { + destination = NewDestination(config) + destination.RegisterMetrics(rw.registerer) + wl := newWriteLoop(rw.dataDir, destination, rw.catalog, rw.clock) + wlr := newWriteLoopRunner(wl) + writeLoopRunners[config.Name] = wlr + destinations[config.Name] = destination + go wlr.run(ctx) + continue + } + + if config.EqualTo(destination.Config()) { + continue + } + + wlr := writeLoopRunners[config.Name] + wlr.stop() + destination.ResetConfig(config) + wl := newWriteLoop(rw.dataDir, destination, rw.catalog, rw.clock) + wlr = newWriteLoopRunner(wl) + writeLoopRunners[config.Name] = wlr + go wlr.run(ctx) + } + rw.readyNotifier.NotifyReady() + } + } +} + +type writeLoopRunner struct { + wl *writeLoop + stopc chan struct{} + stoppedc chan struct{} +} + +func newWriteLoopRunner(wl *writeLoop) *writeLoopRunner { + return &writeLoopRunner{ + wl: wl, + stopc: make(chan struct{}), + stoppedc: make(chan struct{}), + } +} + +func (r *writeLoopRunner) run(ctx context.Context) { + ctx, cancel := context.WithCancel(ctx) + go func() { + <-r.stopc + cancel() + }() + r.wl.run(ctx) + close(r.stoppedc) +} + +func (r *writeLoopRunner) stop() { + close(r.stopc) + <-r.stoppedc +} diff --git a/pp/go/storage/remotewriter/remotewriter_test.go b/pp/go/storage/remotewriter/remotewriter_test.go new file mode 100644 index 0000000000..cbcdeb1323 --- /dev/null +++ b/pp/go/storage/remotewriter/remotewriter_test.go @@ -0,0 +1,15 @@ +package remotewriter + +import ( + "testing" + + "github.com/jonboulle/clockwork" + "github.com/prometheus/client_golang/prometheus" + + "github.com/prometheus/prometheus/pp/go/storage/ready" +) + +func TestRemoteWriter_Run(_ *testing.T) { + rw := New("", nil, clockwork.NewFakeClock(), ready.NoOpNotifier{}, prometheus.DefaultRegisterer) + _ = rw +} diff --git a/pp/go/storage/remotewriter/walreader.go b/pp/go/storage/remotewriter/walreader.go new file mode 100644 index 0000000000..8186aae15c --- /dev/null +++ b/pp/go/storage/remotewriter/walreader.go @@ -0,0 +1,57 @@ +package remotewriter + +import ( + "bufio" + "errors" + "fmt" + "io" + + "github.com/prometheus/prometheus/pp/go/storage/head/shard/wal/reader" + "github.com/prometheus/prometheus/pp/go/util" +) + +type walReader struct { + file *util.FileReader + reader io.Reader + nextSegmentID uint32 +} + +func newWalReader(fileName string) (*walReader, uint8, error) { + file, err := util.OpenFileReader(fileName) + if err != nil { + return nil, 0, fmt.Errorf("failed to read wal file: %w", err) + } + + _, encoderVersion, _, err := reader.ReadHeader(file) + if err != nil { + return nil, 0, errors.Join(fmt.Errorf("failed to read header: %w", err), file.Close()) + } + + return &walReader{ + file: file, + reader: bufio.NewReaderSize(file, 4096), //revive:disable-line:add-constant // 4kb + }, encoderVersion, nil +} + +// Close wal file. +func (r *walReader) Close() error { + return r.file.Close() +} + +// Read [Segment] from wal and return. +func (r *walReader) Read() (segment Segment, err error) { + if _, err = segment.ReadFrom(r.reader); err != nil { + return segment, fmt.Errorf("failed to read segment: %w", err) + } + + segment.ID = r.nextSegmentID + r.nextSegmentID++ + + return segment, nil +} + +// Segment encoded segment from wal. +type Segment struct { + ID uint32 + reader.Segment +} diff --git a/pp/go/storage/remotewriter/writeloop.go b/pp/go/storage/remotewriter/writeloop.go new file mode 100644 index 0000000000..9fd5ac4769 --- /dev/null +++ b/pp/go/storage/remotewriter/writeloop.go @@ -0,0 +1,421 @@ +package remotewriter + +import ( + "context" + "errors" + "fmt" + "os" + "path/filepath" + "time" + + "github.com/jonboulle/clockwork" + "github.com/prometheus/prometheus/storage/remote" + + "github.com/prometheus/prometheus/pp/go/cppbridge" + "github.com/prometheus/prometheus/pp/go/logger" + "github.com/prometheus/prometheus/pp/go/storage/catalog" +) + +const defaultDelay = time.Second * 5 + +type writeLoop struct { + dataDir string + destination *Destination + currentHeadID *string + catalog Catalog + clock clockwork.Clock + client remote.WriteClient +} + +func newWriteLoop(dataDir string, destination *Destination, hcatalog Catalog, clock clockwork.Clock) *writeLoop { + return &writeLoop{ + dataDir: dataDir, + destination: destination, + catalog: hcatalog, + clock: clock, + } +} + +// run sending data via RemoteWriter. +// +//revive:disable-next-line:cyclomatic // but readable +//revive:disable-next-line:function-length // long but readable +//revive:disable-next-line:cognitive-complexity // long but readable +func (wl *writeLoop) run(ctx context.Context) { + var delay time.Duration + var err error + var i *Iterator + var nextI *Iterator + + rw := &readyProtobufWriter{} + + dcfg := wl.destination.Config() + wl.destination.metrics.maxNumShards.Set(float64(dcfg.QueueConfig.MaxShards)) + wl.destination.metrics.minNumShards.Set(float64(dcfg.QueueConfig.MinShards)) + + defer func() { + if i != nil { + _ = i.Close() + } + if nextI != nil { + _ = nextI.Close() + } + }() + + for { + select { + case <-ctx.Done(): + return + case <-wl.clock.After(delay): + delay = 0 + } + + if i == nil { + if nextI != nil { + i = nextI + nextI = nil + } else { + i, err = wl.nextIterator(ctx, rw) + if err != nil { + logger.Errorf("get current next iterator: %v", err) + delay = defaultDelay + continue + } + } + } + + if wl.client == nil { + wl.client, err = createClient(wl.destination.Config()) + if err != nil { + logger.Errorf("create client: %v", err) + delay = defaultDelay + continue + } + + rw.SetProtobufWriter(newProtobufWriter(wl.client)) + } + + if err = wl.write(ctx, i); err != nil { + logger.Errorf("iterator write: %v", err) + delay = defaultDelay + continue + } + + if nextI == nil { + nextI, err = wl.nextIterator(ctx, rw) + if err != nil { + logger.Errorf("get next iterator: %v", err) + delay = defaultDelay + continue + } + } + + if err = i.Close(); err != nil { + logger.Errorf("close iterator: %v", err) + delay = defaultDelay + continue + } + + i = nil + } +} + +// write writes data from iterator to the remote write storage. +func (*writeLoop) write(ctx context.Context, iterator *Iterator) error { + for { + select { + case <-ctx.Done(): + return ctx.Err() + default: + err := iterator.Next(ctx) + if err != nil { + if errors.Is(err, ErrEndOfBlock) { + return nil + } + logger.Errorf("iteration failed: %v", err) + } + } + } +} + +// nextIterator returns next iterator. +// +//revive:disable-next-line:cyclomatic // this is a constructor for new iterator +//revive:disable-next-line:function-length // this is a constructor for new iterator +func (wl *writeLoop) nextIterator(ctx context.Context, protobufWriter ProtobufWriter) (*Iterator, error) { + var nextHeadRecord *catalog.Record + var err error + var cleanStart bool + dcfg := wl.destination.Config() + if wl.currentHeadID != nil { + nextHeadRecord, err = nextHead(ctx, wl.dataDir, wl.catalog, *wl.currentHeadID) + } else { + var headFound bool + nextHeadRecord, headFound, err = scanForNextHead(ctx, wl.dataDir, wl.catalog, dcfg.Name) + cleanStart = !headFound + } + if err != nil { + return nil, fmt.Errorf("find next head: %w", err) + } + headDir := filepath.Join(wl.dataDir, nextHeadRecord.Dir()) + crw, err := NewCursorReadWriter( + filepath.Join(headDir, fmt.Sprintf("%s.cursor", dcfg.Name)), + nextHeadRecord.NumberOfShards(), + ) + if err != nil { + return nil, fmt.Errorf("create cursor: %w", err) + } + + crc32, err := dcfg.CRC32() + if err != nil { + return nil, errors.Join(fmt.Errorf("calculate crc32: %w", err), crw.Close()) + } + + var discardCache bool + if crw.GetConfigCRC32() != crc32 { + if err = crw.SetConfigCRC32(crc32); err != nil { + return nil, errors.Join(fmt.Errorf("write crc32: %w", err), crw.Close()) + } + discardCache = true + } + + ds, err := newDataSource( + headDir, + nextHeadRecord.NumberOfShards(), + dcfg, + discardCache, + newSegmentReadyChecker(nextHeadRecord), + wl.makeCorruptMarker(), + nextHeadRecord, + wl.destination.metrics.unexpectedEOFCount, + wl.destination.metrics.segmentSizeInBytes, + ) + if err != nil { + return nil, errors.Join(fmt.Errorf("create data source: %w", err), crw.Close()) + } + + headID := nextHeadRecord.ID() + ds.ID = headID + + var targetSegmentID uint32 + if cleanStart { + if nextHeadRecord.LastAppendedSegmentID() != nil { + targetSegmentID = *nextHeadRecord.LastAppendedSegmentID() + } else { + targetSegmentID = crw.GetTargetSegmentID() + } + } else { + targetSegmentID = crw.GetTargetSegmentID() + } + + i, err := newIterator( + wl.clock, + dcfg.QueueConfig, + ds, + crw, + targetSegmentID, + dcfg.ReadTimeout, + protobufWriter, + wl.destination.metrics, + ) + if err != nil { + return nil, errors.Join(fmt.Errorf("create data source: %w", err), crw.Close(), ds.Close()) + } + + wl.currentHeadID = &headID + + return i, nil +} + +// makeCorruptMarker set marker on head is corrupted. +func (wl *writeLoop) makeCorruptMarker() CorruptMarker { + return CorruptMarkerFn(func(headID string) error { + _, err := wl.catalog.SetCorrupted(headID) + return err + }) +} + +// createClient creates a new [remote.WriteClient]. +// +//nolint:gocritic // hugeParam // this is a constructor for new client +func createClient(config DestinationConfig) (client remote.WriteClient, err error) { + clientConfig := remote.ClientConfig{ + URL: config.URL, + Timeout: config.RemoteTimeout, + HTTPClientConfig: config.HTTPClientConfig, + SigV4Config: config.SigV4Config, + AzureADConfig: config.AzureADConfig, + Headers: config.Headers, + RetryOnRateLimit: true, + } + + client, err = remote.NewWriteClient(config.Name, &clientConfig) + if err != nil { + return nil, fmt.Errorf("falied to create client: %w", err) + } + + return client, nil +} + +// CorruptMarkerFn func for mark head as corrupted by ID. +type CorruptMarkerFn func(headID string) error + +// MarkCorrupted mark head as corrupted by ID. +func (fn CorruptMarkerFn) MarkCorrupted(headID string) error { + return fn(headID) +} + +// nextHead returns next head record from catalog. +// +//nolint:gocritic // hugeParam // this is a extractor +//revive:disable-next-line:cyclomatic // this is a extractor +func nextHead(ctx context.Context, dataDir string, headCatalog Catalog, headID string) (*catalog.Record, error) { + if err := contextErr(ctx); err != nil { + return nil, err + } + + headRecords := headCatalog.List( + nil, + func(lhs, rhs *catalog.Record) bool { + return lhs.CreatedAt() < rhs.CreatedAt() + }, + ) + + if len(headRecords) == 0 { + return nil, fmt.Errorf("nextHead: no new heads: empty head records") + } + + currentHeadFound := false + for _, headRecord := range headRecords { + if headRecord.ID() == headID { + currentHeadFound = true + continue + } + + if !currentHeadFound { + continue + } + + if err := validateHead(filepath.Join(dataDir, headRecord.Dir())); err != nil { + if !errors.Is(err, os.ErrNotExist) { + return nil, err + } + + switch headRecord.Status() { + case catalog.StatusNew, catalog.StatusActive: + return nil, fmt.Errorf("validate active head: %w", err) + default: + continue + } + } + + return headRecord, nil + } + + // unknown head id, selecting last head + if !currentHeadFound { + return headRecords[len(headRecords)-1], nil + } + + return nil, fmt.Errorf("nextHead: no new heads: appropriate head not found") +} + +// validateHead validates head directory. +func validateHead(headDir string) error { + dir, err := os.Open(headDir) // #nosec G304 // it's meant to be that way + if err != nil { + return err + } + + return dir.Close() +} + +// scanForNextHead scans catalog for next head record. +func scanForNextHead( + ctx context.Context, + dataDir string, + headCatalog Catalog, + destinationName string, +) (*catalog.Record, bool, error) { + if err := contextErr(ctx); err != nil { + return nil, false, err + } + + headRecords := headCatalog.List( + nil, + func(lhs, rhs *catalog.Record) bool { + return lhs.CreatedAt() > rhs.CreatedAt() + }, + ) + + if len(headRecords) == 0 { + return nil, false, fmt.Errorf("scanForNextHead: no new heads: empty head records") + } + + for _, headRecord := range headRecords { + headFound, err := scanHeadForDestination(filepath.Join(dataDir, headRecord.Dir()), destinationName) + if err != nil { + if !headRecord.Corrupted() { + logger.Errorf("head %s is corrupted: %v", headRecord.ID(), err) + if _, corruptErr := headCatalog.SetCorrupted(headRecord.ID()); corruptErr != nil { + logger.Errorf("set corrupted state: %v", corruptErr) + } + } + + continue + } + if headFound { + return headRecord, true, nil + } + } + + // track of the previous destination not found, selecting last head + return headRecords[0], false, nil +} + +// scanHeadForDestination scans head directory for [Destination]. +func scanHeadForDestination(dirPath, destinationName string) (bool, error) { + dir, err := os.Open(dirPath) // #nosec G304 // it's meant to be that way + if err != nil { + return false, fmt.Errorf("open head dir: %w", err) + } + defer func() { _ = dir.Close() }() + + fileNames, err := dir.Readdirnames(-1) + if err != nil { + return false, fmt.Errorf("read dir names: %w", err) + } + + for _, fileName := range fileNames { + if fileName == fmt.Sprintf("%s.cursor", destinationName) { + return true, nil + } + } + + return false, nil +} + +// contextErr returns error if context is done. +func contextErr(ctx context.Context) error { + select { + case <-ctx.Done(): + return ctx.Err() + default: + return nil + } +} + +// readyProtobufWriter is a writer for ready protobuf. +type readyProtobufWriter struct { + protobufWriter ProtobufWriter +} + +// SetProtobufWriter sets protobuf writer. +func (rpw *readyProtobufWriter) SetProtobufWriter(protobufWriter ProtobufWriter) { + rpw.protobufWriter = protobufWriter +} + +// Write writes protobuf to the remote write storage. +func (rpw *readyProtobufWriter) Write(ctx context.Context, protobuf *cppbridge.SnappyProtobufEncodedData) error { + return rpw.protobufWriter.Write(ctx, protobuf) +} diff --git a/pp/go/storage/remotewriter/writeloop_test.go b/pp/go/storage/remotewriter/writeloop_test.go new file mode 100644 index 0000000000..c4fe3a26c3 --- /dev/null +++ b/pp/go/storage/remotewriter/writeloop_test.go @@ -0,0 +1,335 @@ +package remotewriter + +// import ( +// "net/url" +// "path/filepath" +// "testing" +// "time" + +// "github.com/jonboulle/clockwork" +// common_config "github.com/prometheus/common/config" +// common_model "github.com/prometheus/common/model" +// "github.com/stretchr/testify/suite" + +// prom_config "github.com/prometheus/prometheus/config" +// "github.com/prometheus/prometheus/model/labels" +// "github.com/prometheus/prometheus/pp/go/storage/catalog" +// "github.com/prometheus/prometheus/pp/go/storage/remotewriter/mock" +// ) + +// type WriteLoopSuite struct { +// suite.Suite +// } + +// func TestWriteLoopSuite(t *testing.T) { +// suite.Run(t, new(WriteLoopSuite)) +// } + +// func (s *WriteLoopSuite) TestHappyPath() { +// tmpDir := s.T().TempDir() +// ctx := s.T().Context() + +// fileLog, err := catalog.NewFileLogV2(filepath.Join(tmpDir, "catalog")) +// s.Require().NoError(err) + +// clock := clockwork.NewRealClock() +// hCatalog, err := catalog.New(clock, fileLog, catalog.DefaultIDGenerator{}, catalog.DefaultMaxLogFileSize, nil) +// s.Require().NoError(err) + +// u, err := url.Parse("http://localhost:8080") +// s.Require().NoError(err) + +// destination := NewDestination(DestinationConfig{ +// RemoteWriteConfig: prom_config.RemoteWriteConfig{ +// URL: &common_config.URL{URL: u}, +// RemoteTimeout: 0, +// Headers: nil, +// WriteRelabelConfigs: nil, +// Name: "remote_write_0", +// SendExemplars: false, +// SendNativeHistograms: false, +// HTTPClientConfig: common_config.HTTPClientConfig{}, +// QueueConfig: prom_config.QueueConfig{ +// MaxSamplesPerSend: 2, +// MinShards: 3, +// MaxShards: 5, +// SampleAgeLimit: common_model.Duration(time.Hour), +// }, +// MetadataConfig: prom_config.MetadataConfig{}, +// SigV4Config: nil, +// AzureADConfig: nil, +// }, +// ExternalLabels: labels.FromStrings("lol", "kek"), +// ReadTimeout: time.Second * 3, +// }) + +// wl := newWriteLoop(tmpDir, destination, hCatalog, clock) + +// pw := &mock.ProtobufWriterMock{} +// it, err := wl.nextIterator(ctx, pw) +// s.Require().NoError(err) + +// s.Require().NoError(it.Next(ctx)) +// } + +// import ( +// "context" +// "errors" +// "net/url" +// "os" +// "path/filepath" +// "sync" +// "testing" +// "time" + +// "github.com/golang/snappy" +// "github.com/jonboulle/clockwork" +// "github.com/prometheus/client_golang/prometheus" +// config3 "github.com/prometheus/common/config" +// model2 "github.com/prometheus/common/model" +// config2 "github.com/prometheus/prometheus/config" +// "github.com/prometheus/prometheus/model/labels" +// "github.com/prometheus/prometheus/pp/go/cppbridge" +// "github.com/prometheus/prometheus/pp/go/model" +// "github.com/prometheus/prometheus/pp/go/relabeler" +// "github.com/prometheus/prometheus/pp/go/relabeler/appender" +// "github.com/prometheus/prometheus/pp/go/relabeler/config" +// "github.com/prometheus/prometheus/pp/go/relabeler/head/catalog" +// "github.com/prometheus/prometheus/pp/go/relabeler/head/manager" +// "github.com/prometheus/prometheus/prompb" +// "github.com/stretchr/testify/require" +// ) + +// const transparentRelabelerName = "transparent_relabeler" + +// type ConfigSource struct { +// inputRelabelConfigs []*config.InputRelabelerConfig +// numberOfShards uint16 +// } + +// func (s *ConfigSource) Set(inputRelabelConfigs []*config.InputRelabelerConfig, numberOfShards uint16) { +// s.inputRelabelConfigs = inputRelabelConfigs +// s.numberOfShards = numberOfShards +// } + +// func (s *ConfigSource) Get() (inputRelabelConfigs []*config.InputRelabelerConfig, numberOfShards uint16) { +// return s.inputRelabelConfigs, s.numberOfShards +// } + +// type NoOpStorage struct { +// heads []relabeler.Head +// } + +// func (s *NoOpStorage) Add(head relabeler.Head) { +// if err := head.Rotate(); err != nil { +// panic(err) +// } +// s.heads = append(s.heads, head) +// } + +// func (s *NoOpStorage) Close() error { +// var err error +// for _, head := range s.heads { +// err = errors.Join(err, head.Close()) +// } +// s.heads = nil +// return err +// } + +// type TestHeads struct { +// Dir string +// FileLog *catalog.FileLog +// Catalog *catalog.Catalog +// ConfigSource *ConfigSource +// Manager *manager.Manager +// NumberOfShards uint8 +// Storage *NoOpStorage +// Head relabeler.Head +// } + +// func NewTestHeads(dir string, inputRelabelConfigs []*config.InputRelabelerConfig, numberOfShards uint16, clock clockwork.Clock) (*TestHeads, error) { +// th := &TestHeads{ +// Dir: dir, +// } +// var err error +// th.FileLog, err = catalog.NewFileLogV2(filepath.Join(dir, "catalog")) +// if err != nil { +// return nil, err +// } + +// th.Catalog, err = catalog.New(clock, th.FileLog, catalog.DefaultIDGenerator{}, catalog.DefaultMaxLogFileSize, nil) +// if err != nil { +// return nil, errors.Join(err, th.Close()) +// } + +// th.ConfigSource = &ConfigSource{ +// inputRelabelConfigs: inputRelabelConfigs, +// numberOfShards: numberOfShards, +// } + +// th.Manager, err = manager.New(dir, clock, th.ConfigSource, th.Catalog, 0, prometheus.DefaultRegisterer) +// if err != nil { +// return nil, errors.Join(err, th.Close()) +// } + +// activeHead, _, err := th.Manager.Restore(time.Hour) +// if err != nil { +// return nil, errors.Join(err, th.Close()) +// } + +// th.Storage = &NoOpStorage{} +// th.Head = appender.NewRotatableHead(activeHead, th.Storage, th.Manager, appender.NoOpHeadActivator{}) + +// return th, nil +// } + +// func (th *TestHeads) Append(ctx context.Context, timeSeriesSlice []model.TimeSeries, relabelerID string) error { +// hx, err := cppbridge.HashdexFactory{}.GoModel(timeSeriesSlice, cppbridge.DefaultWALHashdexLimits()) +// if err != nil { +// return err +// } + +// _, _, err = th.Head.Append(ctx, &relabeler.IncomingData{Hashdex: hx}, nil, relabelerID, true) +// return err +// } + +// func (th *TestHeads) Rotate() error { +// return th.Head.Rotate() +// } + +// func (th *TestHeads) Close() error { +// return errors.Join(th.Storage.Close(), th.Head.Close(), th.FileLog.Close()) +// } + +// type remoteClient struct { +// mtx sync.Mutex +// data [][]byte +// name string +// } + +// func (c *remoteClient) Store(_ context.Context, bytes []byte, _ int) error { +// c.mtx.Lock() +// defer c.mtx.Unlock() +// c.data = append(c.data, bytes) +// return nil +// } + +// func (c *remoteClient) Name() string { +// return c.name +// } + +// func (c *remoteClient) Endpoint() string { +// return "" +// } + +// type testWriter struct { +// mtx sync.Mutex +// data []*cppbridge.SnappyProtobufEncodedData +// } + +// func (w *testWriter) Write(ctx context.Context, protobuf *cppbridge.SnappyProtobufEncodedData) error { +// w.mtx.Lock() +// defer w.mtx.Unlock() +// w.data = append(w.data, protobuf) +// return ctx.Err() +// } + +// func TestWriteLoopWrite(t *testing.T) { +// tmpDir, err := os.MkdirTemp("", "write_loop_iterate_test") +// require.NoError(t, err) +// defer func() { _ = os.RemoveAll(tmpDir) }() + +// clock := clockwork.NewRealClock() +// cfgs := []*config.InputRelabelerConfig{ +// { +// Name: transparentRelabelerName, +// RelabelConfigs: []*cppbridge.RelabelConfig{ +// { +// SourceLabels: []string{"__name__"}, +// Regex: ".*", +// Action: cppbridge.Keep, +// }, +// }, +// }, +// } +// var numberOfShards uint16 = 2 + +// testHeads, err := NewTestHeads(tmpDir, cfgs, numberOfShards, clock) +// require.NoError(t, err) + +// ctx, cancel := context.WithTimeout(context.Background(), time.Minute*5) +// defer cancel() + +// labelSets := []model.LabelSet{ +// model.NewLabelSetBuilder().Set("__name__", "test_metric_0").Build(), +// model.NewLabelSetBuilder().Set("__name__", "test_metric_1").Build(), +// model.NewLabelSetBuilder().Set("__name__", "test_metric_2").Build(), +// model.NewLabelSetBuilder().Set("__name__", "test_metric_3").Build(), +// } + +// ts := clock.Now().UnixMilli() +// batches := [][]model.TimeSeries{ +// { +// {LabelSet: labelSets[0], Timestamp: uint64(ts), Value: 0}, +// {LabelSet: labelSets[1], Timestamp: uint64(ts), Value: 1000}, +// {LabelSet: labelSets[2], Timestamp: uint64(ts), Value: 1000000}, +// {LabelSet: labelSets[3], Timestamp: uint64(ts), Value: 1000000000}, +// }, +// } + +// err = testHeads.Append(ctx, batches[0], transparentRelabelerName) +// require.NoError(t, err) + +// u, err := url.Parse("http://localhost:8080") +// require.NoError(t, err) + +// destination := NewDestination(DestinationConfig{ +// RemoteWriteConfig: config2.RemoteWriteConfig{ +// URL: &config3.URL{u}, +// RemoteTimeout: 0, +// Headers: nil, +// WriteRelabelConfigs: nil, +// Name: "remote_write_0", +// SendExemplars: false, +// SendNativeHistograms: false, +// HTTPClientConfig: config3.HTTPClientConfig{}, +// QueueConfig: config2.QueueConfig{ +// MaxSamplesPerSend: 2, +// MinShards: 3, +// MaxShards: 5, +// SampleAgeLimit: model2.Duration(time.Hour), +// }, +// MetadataConfig: config2.MetadataConfig{}, +// SigV4Config: nil, +// AzureADConfig: nil, +// }, +// ExternalLabels: labels.FromStrings("lol", "kek"), +// ReadTimeout: time.Second * 3, +// }) + +// wl := newWriteLoop(tmpDir, destination, testHeads.Catalog, clock) +// w := &testWriter{} +// i, err := wl.nextIterator(ctx, w) +// require.NoError(t, err) + +// require.NoError(t, i.Next(ctx)) +// require.NoError(t, err) + +// require.NoError(t, testHeads.Rotate()) + +// require.ErrorIs(t, i.Next(ctx), ErrEndOfBlock) + +// for _, data := range w.data { +// wr := prompb.WriteRequest{} +// err = data.Do(func(buf []byte) error { +// var decoded []byte +// decoded, err = snappy.Decode(nil, buf) +// if err != nil { +// return err +// } +// return wr.Unmarshal(decoded) +// }) +// require.NoError(t, err) +// t.Log(wr.String()) +// } +// } diff --git a/pp/go/storage/remotewriter/writer.go b/pp/go/storage/remotewriter/writer.go new file mode 100644 index 0000000000..ef1b0d03c0 --- /dev/null +++ b/pp/go/storage/remotewriter/writer.go @@ -0,0 +1,38 @@ +package remotewriter + +import ( + "context" + + "github.com/prometheus/prometheus/pp/go/cppbridge" + "github.com/prometheus/prometheus/storage/remote" +) + +// protobufWriter the wrapper over the [remote.WriteClient]. +type protobufWriter struct { + client remote.WriteClient +} + +// newProtobufWriter init new [protobufWriter]. +func newProtobufWriter(client remote.WriteClient) *protobufWriter { + return &protobufWriter{ + client: client, + } +} + +// Close implementation [io.Closer]. +func (*protobufWriter) Close() error { + return nil +} + +// Write [cppbridge.SnappyProtobufEncodedData] to [remote.WriteClient] +func (w *protobufWriter) Write(ctx context.Context, protobuf *cppbridge.SnappyProtobufEncodedData) error { + return protobuf.Do(func(buf []byte) error { + if len(buf) == 0 { + return nil + } + + // TODO WriteResponseStats + _, err := w.client.Store(ctx, buf, 0) + return err + }) +} diff --git a/pp/go/storage/storagetest/fixtures.go b/pp/go/storage/storagetest/fixtures.go new file mode 100644 index 0000000000..a91f319cee --- /dev/null +++ b/pp/go/storage/storagetest/fixtures.go @@ -0,0 +1,124 @@ +package storagetest + +import ( + "context" + + "github.com/prometheus/prometheus/model/labels" + "github.com/prometheus/prometheus/pp/go/cppbridge" + "github.com/prometheus/prometheus/pp/go/model" + "github.com/prometheus/prometheus/pp/go/storage" + "github.com/prometheus/prometheus/pp/go/storage/appender" + "github.com/prometheus/prometheus/pp/go/storage/head/services" + promstorage "github.com/prometheus/prometheus/storage" + "github.com/prometheus/prometheus/tsdb/chunkenc" + "github.com/stretchr/testify/suite" +) + +// TimeSeries test data. +type TimeSeries struct { + Labels labels.Labels + Samples []cppbridge.Sample +} + +// AppendSamples add samples to time series. +func (s *TimeSeries) AppendSamples(samples ...cppbridge.Sample) { + s.Samples = append(s.Samples, samples...) +} + +func (s *TimeSeries) toModelTimeSeries() []model.TimeSeries { + lsBuilder := model.NewLabelSetBuilder() + s.Labels.Range(func(l labels.Label) { + lsBuilder.Add(l.Name, l.Value) + }) + + ls := lsBuilder.Build() + + timeSeries := make([]model.TimeSeries, 0, len(s.Samples)) + for i := range s.Samples { + timeSeries = append(timeSeries, model.TimeSeries{ + LabelSet: ls, + Timestamp: uint64(s.Samples[i].Timestamp), // #nosec G115 // no overflow + Value: s.Samples[i].Value, + }) + } + + return timeSeries +} + +type timeSeriesDataSlice struct { + timeSeries []model.TimeSeries +} + +func (tsd *timeSeriesDataSlice) TimeSeries() []model.TimeSeries { + return tsd.timeSeries +} + +func (tsd *timeSeriesDataSlice) Destroy() { + tsd.timeSeries = nil +} + +// MustAppendTimeSeries add time series to head. +func MustAppendTimeSeries(s *suite.Suite, head *storage.Head, timeSeries []TimeSeries) { + headAppender := appender.New(head, services.CFViaRange) + + statelessRelabeler, err := cppbridge.NewStatelessRelabeler([]*cppbridge.RelabelConfig{}) + s.Require().NoError(err) + + state := cppbridge.NewStateV2WithoutLock() + state.SetStatelessRelabeler(statelessRelabeler) + + for i := range timeSeries { + tsd := timeSeriesDataSlice{timeSeries: timeSeries[i].toModelTimeSeries()} + hx, err := (cppbridge.HashdexFactory{}).GoModel(tsd.TimeSeries(), cppbridge.DefaultWALHashdexLimits()) + s.Require().NoError(err) + + _, _, err = headAppender.Append( + context.Background(), + &appender.IncomingData{Hashdex: hx, Data: &tsd}, + state, + true) + s.NoError(err) + } +} + +// SamplesMap samples map with series ID as key. +type SamplesMap map[uint32][]cppbridge.Sample + +// GetSamplesFromSerializedChunks returns sample from serialized chunks. +func GetSamplesFromSerializedChunks(chunks *cppbridge.HeadDataStorageSerializedChunks) SamplesMap { + result := make(SamplesMap) + + deserializer := cppbridge.NewHeadDataStorageDeserializer(chunks) + + n := chunks.NumberOfChunks() + for i := 0; i < n; i++ { + metadata := chunks.Metadata(i) + seriesId := metadata.SeriesID() + iterator := deserializer.CreateDecodeIterator(metadata) + for iterator.Next() { + ts, value := iterator.Sample() + result[seriesId] = append(result[seriesId], cppbridge.Sample{Timestamp: ts, Value: value}) + } + } + + return result +} + +// TimeSeriesFromSeriesSet converting seriesset to slice timeseries. +func TimeSeriesFromSeriesSet(seriesSet promstorage.SeriesSet) []TimeSeries { + var timeSeries []TimeSeries + for seriesSet.Next() { + series := seriesSet.At() + + timeSeries = append(timeSeries, TimeSeries{Labels: series.Labels()}) + currentSeries := &timeSeries[len(timeSeries)-1] + + chunkIterator := series.Iterator(nil) + for chunkIterator.Next() != chunkenc.ValNone { + ts, v := chunkIterator.At() + currentSeries.Samples = append(currentSeries.Samples, cppbridge.Sample{Timestamp: ts, Value: v}) + } + } + + return timeSeries +} diff --git a/pp/go/storage/types.go b/pp/go/storage/types.go new file mode 100644 index 0000000000..b22ed24652 --- /dev/null +++ b/pp/go/storage/types.go @@ -0,0 +1,15 @@ +package storage + +import ( + "github.com/prometheus/prometheus/pp/go/cppbridge" + "github.com/prometheus/prometheus/pp/go/storage/head/head" + "github.com/prometheus/prometheus/pp/go/storage/head/shard" + "github.com/prometheus/prometheus/pp/go/storage/head/shard/wal" + "github.com/prometheus/prometheus/pp/go/storage/head/shard/wal/writer" +) + +// Wal alias for [wal.Wal] based on [cppbridge.HeadEncodedSegment] and [writer.Buffered]. +type Wal = wal.Wal[*cppbridge.HeadEncodedSegment, *writer.Buffered[*cppbridge.HeadEncodedSegment]] + +// Head alias for [head.Head] with [shard.Shard] and [shard.PerGoroutineShard]. +type Head = head.Head[*shard.Shard, *shard.PerGoroutineShard] diff --git a/pp/go/util/closer.go b/pp/go/util/closer.go index f32bbf988e..15b48afbea 100644 --- a/pp/go/util/closer.go +++ b/pp/go/util/closer.go @@ -1,19 +1,31 @@ package util +import ( + "errors" + "io" + "sync" +) + type Closer struct { - close chan struct{} - closed chan struct{} + close chan struct{} + closeOnce sync.Once + closed chan struct{} + closedOnce sync.Once } func NewCloser() *Closer { return &Closer{ - close: make(chan struct{}), - closed: make(chan struct{}), + close: make(chan struct{}), + closeOnce: sync.Once{}, + closed: make(chan struct{}), + closedOnce: sync.Once{}, } } func (c *Closer) Done() { - close(c.closed) + c.closedOnce.Do(func() { + close(c.closed) + }) } func (c *Closer) Signal() <-chan struct{} { @@ -21,7 +33,17 @@ func (c *Closer) Signal() <-chan struct{} { } func (c *Closer) Close() error { - close(c.close) + c.closeOnce.Do(func() { + close(c.close) + }) <-c.closed return nil } + +func CloseAll(closers ...io.Closer) error { + var errs error + for _, closer := range closers { + errs = errors.Join(errs, closer.Close()) + } + return errs +} diff --git a/pp/go/util/locker/priweighted.go b/pp/go/util/locker/priweighted.go index 974dda6ace..4b0108a289 100644 --- a/pp/go/util/locker/priweighted.go +++ b/pp/go/util/locker/priweighted.go @@ -3,9 +3,13 @@ package locker // based "golang.org/x/sync/semaphore" import ( "container/list" "context" + "errors" "sync" ) +// ErrSemaphoreClosed error when the semaphore was closed. +var ErrSemaphoreClosed = errors.New("semaphore was closed") + type waiter struct { n int64 ready chan<- struct{} // Closed when semaphore acquired. @@ -27,6 +31,24 @@ type Weighted struct { waiters list.List lastPri *list.Element exclusive bool + closed bool +} + +// Close sets the flag that the semaphore is closed under the priority lock +// and after unlocking all those waiting will receive the error [ErrSemaphoreClosed]. +func (s *Weighted) Close() error { + unlock, err := s.LockWithPriority(context.Background()) + if err != nil { + return err + } + + s.mu.Lock() + s.closed = true + s.mu.Unlock() + + unlock() + + return nil } // Lock locks for exclusive operation with weight of full size. @@ -96,6 +118,11 @@ func (s *Weighted) acquireWithInserter(ctx context.Context, n int64, inserter fu done := ctx.Done() s.mu.Lock() + if s.closed { + s.mu.Unlock() + return ErrSemaphoreClosed + } + select { case <-done: // ctx becoming done has "happened before" acquiring the semaphore, @@ -106,6 +133,7 @@ func (s *Weighted) acquireWithInserter(ctx context.Context, n int64, inserter fu return ctx.Err() default: } + if ws := s.weightSize(n); s.size-s.cur >= ws && s.waiters.Len() == 0 { // Since we hold s.mu and haven't synchronized since checking done, if // ctx becomes done before we return here, it becoming done must have @@ -116,6 +144,7 @@ func (s *Weighted) acquireWithInserter(ctx context.Context, n int64, inserter fu s.exclusive = true } s.mu.Unlock() + return nil } @@ -154,6 +183,12 @@ func (s *Weighted) acquireWithInserter(ctx context.Context, n int64, inserter fu return ctx.Err() default: } + + if s.closed { + s.release(n) + return ErrSemaphoreClosed + } + return nil } } diff --git a/pp/performance_tests/benchmarks/BUILD b/pp/performance_tests/benchmarks/BUILD index e9e4aea3e3..4e18945e48 100644 --- a/pp/performance_tests/benchmarks/BUILD +++ b/pp/performance_tests/benchmarks/BUILD @@ -9,3 +9,14 @@ cc_binary( "@google_benchmark//:benchmark_main", ], ) + +cc_binary( + name = "copy_lss", + srcs = ["queryable_encoding_bimap_copy_benchmark.cpp"], + malloc = "@jemalloc", + deps = [ + "//:performance_tests_headers", + "@google_benchmark//:benchmark_main", + "//:entrypoint", + ], +) diff --git a/pp/performance_tests/benchmarks/queryable_encoding_bimap_copy_benchmark.cpp b/pp/performance_tests/benchmarks/queryable_encoding_bimap_copy_benchmark.cpp new file mode 100644 index 0000000000..ddd962f363 --- /dev/null +++ b/pp/performance_tests/benchmarks/queryable_encoding_bimap_copy_benchmark.cpp @@ -0,0 +1,150 @@ +// ./bazel-bin/performance_tests/benchmarks/copy_lss --benchmark_context=wal_file="performance_tests/test_data/new/lss_real" --benchmark_counters_tabular=true +// --benchmark_repetitions=25 +#include + +#include +#include +#include +#include +#include +#include + +#include "primitives/snug_composites.h" +#include "series_index/queryable_encoding_bimap.h" +#include "series_index/trie/cedarpp_tree.h" + +namespace { +using Lss = + series_index::QueryableEncodingBimap; + +template +using LssCopier = series_index::QueryableEncodingBimapCopier; + +std::string GetWalFileName() { + if (auto& context = benchmark::internal::GetGlobalContext(); context != nullptr) { + return context->operator[]("wal_file"); + } + return {}; +} + +void mark_all_series_as_added(const std::shared_ptr& lss) { + for (auto label_set : *lss) { + lss->find_or_emplace(label_set); + } +} + +std::shared_ptr LoadLssFromFile() { + auto file_name = GetWalFileName(); + auto lss = std::make_shared(); + + std::ifstream istrm(file_name, std::ios::binary); + istrm >> *lss; + mark_all_series_as_added(lss); + return lss; +} + +std::vector copy_added_series_times; +std::vector copy_ls_id_set_times; +std::vector build_trie_index_times; +std::vector build_ls_id_hashset_times; +std::vector build_reverse_index_times; + +void BM_CopyAllStepsWithTiming(benchmark::State& state) { + using std::chrono::nanoseconds; + using std::chrono::steady_clock; + + static auto lss = LoadLssFromFile(); + lss->build_deferred_indexes(); + + for ([[maybe_unused]] auto _ : state) { + Lss lss_copy; + LssCopier copier(*lss, lss->sorting_index(), lss->added_series(), lss_copy); + + { + auto start = steady_clock::now(); + copier.copy_added_series(); + auto end = steady_clock::now(); + copy_added_series_times.push_back(duration_cast(end - start).count()); + } + + { + auto start = steady_clock::now(); + copier.copy_ls_id_set(); + auto end = steady_clock::now(); + copy_ls_id_set_times.push_back(duration_cast(end - start).count()); + } + + { + auto start = steady_clock::now(); + copier.build_trie_index(); + auto end = steady_clock::now(); + build_trie_index_times.push_back(duration_cast(end - start).count()); + } + + { + auto start = steady_clock::now(); + copier.build_ls_id_hashset(); + auto end = steady_clock::now(); + build_ls_id_hashset_times.push_back(duration_cast(end - start).count()); + } + + { + auto start = steady_clock::now(); + copier.build_reverse_index(); + auto end = steady_clock::now(); + build_reverse_index_times.push_back(duration_cast(end - start).count()); + } + } + + benchmark::DoNotOptimize(copy_added_series_times); + benchmark::DoNotOptimize(copy_ls_id_set_times); + benchmark::DoNotOptimize(build_trie_index_times); + benchmark::DoNotOptimize(build_ls_id_hashset_times); + benchmark::DoNotOptimize(build_reverse_index_times); +} + +BENCHMARK(BM_CopyAllStepsWithTiming); + +uint64_t Min(const std::vector& v) { + return v.empty() ? 0uz : static_cast(*std::ranges::min_element(v)); +} + +void PrintMinStats() { + std::cout << "\n=== Min method timings (ns) ===\n"; + + constexpr int words_width = 20; + constexpr int numbers_width = 10; + + std::cout << std::left << std::setw(words_width) << "copy_added_series" + << ": " << std::right << std::setw(numbers_width) << Min(copy_added_series_times) << '\n'; + + std::cout << std::left << std::setw(words_width) << "copy_ls_id_set" + << ": " << std::right << std::setw(numbers_width) << Min(copy_ls_id_set_times) << '\n'; + + std::cout << std::left << std::setw(words_width) << "build_trie_index" + << ": " << std::right << std::setw(numbers_width) << Min(build_trie_index_times) << '\n'; + + std::cout << std::left << std::setw(words_width) << "build_ls_id_hashset" + << ": " << std::right << std::setw(numbers_width) << Min(build_ls_id_hashset_times) << '\n'; + + std::cout << std::left << std::setw(words_width) << "build_reverse_index" + << ": " << std::right << std::setw(numbers_width) << Min(build_reverse_index_times) << '\n'; + + std::cout << "-------------------------------\n"; + + std::cout << std::left << std::setw(words_width) << "total sum" + << ": " << std::right << std::setw(numbers_width) + << (Min(copy_added_series_times) + Min(copy_ls_id_set_times) + Min(build_trie_index_times) + Min(build_ls_id_hashset_times) + + Min(build_reverse_index_times)) + << '\n'; + + std::cout << "===============================\n"; +} + +} // namespace + +int main(int argc, char** argv) { + ::benchmark::Initialize(&argc, argv); + ::benchmark::RunSpecifiedBenchmarks(); + PrintMinStats(); +} diff --git a/pp/primitives/go_slice.h b/pp/primitives/go_slice.h index 4a8ce297df..e32e113b4c 100644 --- a/pp/primitives/go_slice.h +++ b/pp/primitives/go_slice.h @@ -125,23 +125,6 @@ template struct SliceControlBlock { using SizeType = size_t; - SliceControlBlock() = default; - SliceControlBlock(const SliceControlBlock&) = delete; - - SliceControlBlock(SliceControlBlock&& other) noexcept - : data(std::exchange(other.data, nullptr)), items_count(std::exchange(other.items_count, 0)), data_size(std::exchange(other.data_size, 0)) {} - - SliceControlBlock& operator=(const SliceControlBlock&) = delete; - PROMPP_ALWAYS_INLINE SliceControlBlock& operator=(SliceControlBlock&& other) noexcept { - if (this != &other) [[likely]] { - data = std::exchange(other.data, nullptr); - data_size = std::exchange(other.data_size, 0); - items_count = std::exchange(other.items_count, 0); - } - - return *this; - } - T* data{}; union { diff --git a/pp/primitives/label_set.h b/pp/primitives/label_set.h index c268fea8fa..4ff3609f89 100644 --- a/pp/primitives/label_set.h +++ b/pp/primitives/label_set.h @@ -8,8 +8,6 @@ namespace PromPP::Primitives { template class Container = BareBones::Vector> class BasicLabelSet { - Container labels_; - public: using label_type = LabelType; @@ -68,6 +66,17 @@ class BasicLabelSet { } } + template + PROMPP_ALWAYS_INLINE void append_unsorted(const LabelSet& label_set) { + labels_.reserve(labels_.size() + label_set.size()); + + for (const auto& label : label_set) { + append(label.first, label.second); + } + + sort(); + } + template PROMPP_ALWAYS_INLINE void add(const LabelSet& label_set) { labels_.reserve(labels_.size() + label_set.size()); @@ -102,6 +111,8 @@ class BasicLabelSet { [[nodiscard]] PROMPP_ALWAYS_INLINE const_iterator end() const noexcept { return labels_.end(); } PROMPP_ALWAYS_INLINE iterator end() noexcept { return labels_.end(); } + [[nodiscard]] PROMPP_ALWAYS_INLINE const LabelType& operator[](uint32_t index) const noexcept { return labels_[index]; } + [[nodiscard]] PROMPP_ALWAYS_INLINE uint32_t allocated_memory() const noexcept { return BareBones::mem::allocated_memory(labels_); } template @@ -169,6 +180,13 @@ class BasicLabelSet { }; [[nodiscard]] PROMPP_ALWAYS_INLINE Names names() const noexcept { return Names(*this); } + + private: + Container labels_; + + PROMPP_ALWAYS_INLINE void sort() noexcept { + std::ranges::sort(labels_, [](const auto& a, const auto& b) { return a.first < b.first; }); + } }; using LabelSet = BasicLabelSet; diff --git a/pp/primitives/labels_builder.h b/pp/primitives/labels_builder.h index d9253f0f72..cc458c9c8d 100644 --- a/pp/primitives/labels_builder.h +++ b/pp/primitives/labels_builder.h @@ -9,20 +9,11 @@ #include "primitives.h" namespace PromPP::Primitives { -class LabelsBuilderStateMap { - PromPP::Primitives::LabelViewSet building_buf_view_; - PromPP::Primitives::LabelSet building_buf_; - phmap::flat_hash_map buffer_; - template - void sort_labels(Labels& labels) { - std::ranges::sort(labels, [](const auto& a, const auto& b) { - if (a.first == b.first) { - return a.second < b.second; - } - return a.first < b.first; - }); - } +class LabelsBuilder { + LabelViewSet building_buf_view_; + LabelSet building_buf_; + phmap::flat_hash_map buffer_; public: // del add label name to remove from label set. @@ -33,8 +24,8 @@ class LabelsBuilderStateMap { // extract we extract(move) the lebel from the builder. PROMPP_ALWAYS_INLINE Label extract(const std::string_view& lname) { - if (auto it = buffer_.find(lname); it != buffer_.end()) { - auto node = buffer_.extract(it); + if (const auto it = buffer_.find(lname); it != buffer_.end()) { + auto&& node = buffer_.extract(it); return {std::move(const_cast(node.key())), std::move(node.mapped())}; } @@ -42,17 +33,17 @@ class LabelsBuilderStateMap { } // get returns the value for the label with the given name. Returns an empty string if the label doesn't exist. - PROMPP_ALWAYS_INLINE std::string_view get(const std::string_view lname) { - if (auto it = buffer_.find(lname); it != buffer_.end()) { - return (*it).second; + PROMPP_ALWAYS_INLINE std::string_view get(const std::string_view& lname) { + if (const auto it = buffer_.find(lname); it != buffer_.end()) { + return it->second; } return ""; } // contains check the given name if exist. - [[nodiscard]] PROMPP_ALWAYS_INLINE bool contains(const std::string_view lname) const noexcept { - if (auto it = buffer_.find(lname); it != buffer_.end()) { + [[nodiscard]] PROMPP_ALWAYS_INLINE bool contains(const std::string_view& lname) const noexcept { + if (const auto it = buffer_.find(lname); it != buffer_.end()) { return true; } @@ -68,7 +59,7 @@ class LabelsBuilderStateMap { } if (auto it = buffer_.find(lname); it != buffer_.end()) { - (*it).second = lvalue; + it->second = lvalue; return; } @@ -76,61 +67,34 @@ class LabelsBuilderStateMap { } // returns size of building labels. - PROMPP_ALWAYS_INLINE size_t size() { return buffer_.size(); } + [[nodiscard]] PROMPP_ALWAYS_INLINE size_t size() const { return buffer_.size(); } // returns true if ls represents an empty set of labels. - PROMPP_ALWAYS_INLINE bool is_empty() { return buffer_.size() == 0; } + [[nodiscard]] PROMPP_ALWAYS_INLINE bool is_empty() const { return buffer_.empty(); } // label_view_set - returns the label_view set from the builder. If no modifications were made, the original labels are returned. PROMPP_ALWAYS_INLINE const LabelViewSet& label_view_set() { building_buf_view_.clear(); - for (const auto& it : buffer_) { - if (it.second == "") [[unlikely]] { - continue; - } - - building_buf_view_.add(LabelView{it.first, it.second}); - } - - if (building_buf_view_.size() != 0) { - sort_labels(building_buf_view_); - } - + building_buf_view_.append_unsorted(buffer_); return building_buf_view_; } // label_set - returns the label set from the builder. If no modifications were made, the original labels are returned. PROMPP_ALWAYS_INLINE const LabelSet& label_set() { building_buf_.clear(); - - for (const auto& it : buffer_) { - if (it.second == "") [[unlikely]] { - continue; - } - - building_buf_.add(Label{it.first, it.second}); - } - - if (building_buf_.size() != 0) { - sort_labels(building_buf_); - } - + building_buf_.append_unsorted(buffer_); return building_buf_; } - // range - calls f on each label in the builder. - // TODO without copy buffer_, all changes in a another cycle. + // range - calls f on each label in the builder. Don't modify LabelsBuilderStateMap inside callback template - PROMPP_ALWAYS_INLINE void range(Callback func) { - // take a copy of add and del, so they are unaffected by calls to set() or del(). - phmap::flat_hash_map cbuffer_ = buffer_; - - for (const auto& it : cbuffer_) { - if (it.second == "") [[unlikely]] { + PROMPP_ALWAYS_INLINE void range(Callback func) const { + for (const auto& it : buffer_) { + if (it.second.empty()) [[unlikely]] { continue; } - if (bool ok = func(it.first, it.second); !ok) { + if (!func(it.first, it.second)) { return; } } @@ -145,83 +109,15 @@ class LabelsBuilderStateMap { // reset - clears all current state for the builder. template - PROMPP_ALWAYS_INLINE void reset(SomeLabelSet& base) { - building_buf_view_.clear(); - building_buf_.clear(); - buffer_.clear(); + PROMPP_ALWAYS_INLINE void reset(const SomeLabelSet& base) { + reset(); for (const auto& [lname, lvalue] : base) { - if (lvalue == "") { - continue; + if (!lvalue.empty()) [[likely]] { + buffer_[lname] = lvalue; } - - buffer_[lname] = lvalue; } } }; -// LabelsBuilder - builder for label set. -template -class LabelsBuilder { - BuilderState& state_; - - public: - PROMPP_ALWAYS_INLINE explicit LabelsBuilder(BuilderState& state) : state_(state) {} - - template - PROMPP_ALWAYS_INLINE explicit LabelsBuilder(BuilderState& state, SomeLabelSet* ls) : state_(state) { - reset(ls); - } - - // del - add label name to remove from label set. - template - PROMPP_ALWAYS_INLINE void del(const LNameType& lname) { - state_.del(lname); - } - - // extract we extract(move) the lebel from the builder. - PROMPP_ALWAYS_INLINE Label extract(const std::string_view& lname) { return state_.extract(lname); } - - // get - returns the value for the label with the given name. Returns an empty string if the label doesn't exist. - PROMPP_ALWAYS_INLINE std::string_view get(const std::string_view lname) { return state_.get(lname); } - - // contains check the given name if exist. - [[nodiscard]] PROMPP_ALWAYS_INLINE bool contains(const std::string_view lname) const noexcept { return state_.contains(lname); } - - // returns size of building labels. - PROMPP_ALWAYS_INLINE size_t size() { return state_.size(); } - - // returns true if ls represents an empty set of labels. - PROMPP_ALWAYS_INLINE bool is_empty() { return state_.is_empty(); } - - // label_view_set - returns the label_view set from the builder. If no modifications were made, the original labels are returned. - PROMPP_ALWAYS_INLINE const PromPP::Primitives::LabelViewSet& label_view_set() { return state_.label_view_set(); } - - // label_set - returns the label set from the builder. If no modifications were made, the original labels are returned. - PROMPP_ALWAYS_INLINE const PromPP::Primitives::LabelSet& label_set() { return state_.label_set(); } - - // range - calls f on each label in the builder. - template - PROMPP_ALWAYS_INLINE void range(Callback func) { - state_.range(func); - } - - // reset - clears all current state for the builder. - PROMPP_ALWAYS_INLINE void reset() { state_.reset(); } - - // reset - clears all current state for the builder and init from LabelSet. - template - PROMPP_ALWAYS_INLINE void reset(const SomeLabelSet& ls) { - state_.reset(ls); - } - - // set - the name/value pair as a label. A value of "" means delete that label. - template - PROMPP_ALWAYS_INLINE void set(const LNameType& lname, const LValueType& lvalue) { - state_.set(lname, lvalue); - } - - PROMPP_ALWAYS_INLINE LabelsBuilder(LabelsBuilder&&) noexcept = default; - PROMPP_ALWAYS_INLINE ~LabelsBuilder() = default; -}; } // namespace PromPP::Primitives \ No newline at end of file diff --git a/pp/primitives/snug_composites_filaments.h b/pp/primitives/snug_composites_filaments.h index c34105f35e..b2f7421f63 100644 --- a/pp/primitives/snug_composites_filaments.h +++ b/pp/primitives/snug_composites_filaments.h @@ -144,7 +144,10 @@ class Symbol { return max_ui32 - this->size(); } - PROMPP_ALWAYS_INLINE void reserve(const data_type& other) { Vector::reserve(other.size()); } + template + PROMPP_ALWAYS_INLINE void reserve(const DataType& other) { + Vector::reserve(other.size()); + } }; using composite_type = std::string_view; @@ -369,7 +372,8 @@ class LabelNameSet { return symbols_table.allocated_memory() + BareBones::mem::allocated_memory(symbols_ids_sequences); } - PROMPP_ALWAYS_INLINE void reserve(const data_type& other) noexcept { + template + PROMPP_ALWAYS_INLINE void reserve(const DataType& other) noexcept { symbols_table.reserve(other.symbols_table); symbols_ids_sequences.reserve(other.symbols_ids_sequences.size()); } @@ -691,7 +695,8 @@ class LabelSet { symbols_tables.resize(symbols_tables_checkpoints.size()); } - void reserve(const data_type& other) { + template + void reserve(const DataType& other) { symbols_ids_sequences.reserve(other.symbols_ids_sequences.size()); symbols_tables.reserve(other.symbols_tables.size()); label_name_sets_table.reserve(other.label_name_sets_table); diff --git a/pp/primitives/tests/labels_builder_tests.cpp b/pp/primitives/tests/labels_builder_tests.cpp index 6454b4b487..8ef24b03cc 100644 --- a/pp/primitives/tests/labels_builder_tests.cpp +++ b/pp/primitives/tests/labels_builder_tests.cpp @@ -5,8 +5,7 @@ namespace { struct TestLabelsBuilder : public testing::Test { - PromPP::Primitives::LabelsBuilderStateMap builder_state_; - PromPP::Primitives::LabelsBuilder builder_{builder_state_}; + PromPP::Primitives::LabelsBuilder builder_; PromPP::Primitives::LabelViewSet ls_view_; PromPP::Primitives::LabelSet ls_; std::vector> DATA{{"qwe", "ewq"}, {"asd", "dsa"}, {"zxc", "cxz"}}; @@ -68,7 +67,7 @@ TEST_F(TestLabelsBuilder, SetEmpty) { ls_.add({DATA[i].first, DATA[i].second}); } - std::string empty = ""; + const std::string empty; builder_.set(DATA[0].first, empty); EXPECT_EQ(builder_.label_view_set(), ls_view_); @@ -85,7 +84,7 @@ TEST_F(TestLabelsBuilder, SetChange) { ls_.add({lname, lvalue}); } - std::string value = "zxcv"; + const std::string value = "zxcv"; builder_.set(DATA[0].first, value); EXPECT_NE(builder_.label_view_set(), ls_view_); @@ -110,7 +109,7 @@ TEST_F(TestLabelsBuilder, Extract) { builder_.set(lname, lvalue); } - auto l = builder_.extract(DATA[0].first); + const auto l = builder_.extract(DATA[0].first); EXPECT_EQ(DATA[0], l); EXPECT_EQ(builder_.get(l.first), ""); @@ -123,7 +122,7 @@ TEST_F(TestLabelsBuilder, Del) { builder_.del(DATA[0].first); - std::string_view b_lvalue = builder_.get(DATA[0].first); + const std::string_view b_lvalue = builder_.get(DATA[0].first); EXPECT_EQ(b_lvalue, ""); } @@ -142,72 +141,6 @@ TEST_F(TestLabelsBuilder, SetDelSet) { EXPECT_EQ(builder_.label_set(), ls_); } -TEST_F(TestLabelsBuilder, Range) { - for (auto& [lname, lvalue] : DATA) { - ls_view_.add({lvalue, lname}); - builder_.set(lname, lvalue); - ls_.add({lvalue, lname}); - } - - builder_.range([&](LNameType& lname, LValueType& lvalue) PROMPP_LAMBDA_INLINE -> bool { - builder_.del(lname); - builder_.set(lvalue, lname); - return true; - }); - - EXPECT_EQ(builder_.label_view_set(), ls_view_); - EXPECT_EQ(builder_.label_set(), ls_); -} - -TEST_F(TestLabelsBuilder, RangeFastExit) { - for (size_t i = 0; i < DATA.size(); ++i) { - builder_.set(DATA[i].first, DATA[i].second); - if (i == 2) { - // for last label not swap lname and lvalue - ls_view_.add({DATA[i].first, DATA[i].second}); - ls_.add({DATA[i].first, DATA[i].second}); - continue; - } - ls_view_.add({DATA[i].second, DATA[i].first}); - ls_.add({DATA[i].second, DATA[i].first}); - } - - size_t count{0}; - builder_.range([&](LNameType& lname, LValueType& lvalue) PROMPP_LAMBDA_INLINE -> bool { - builder_.del(lname); - builder_.set(lvalue, lname); - ++count; - if (count == 2) { - return false; - }; - return true; - }); - - EXPECT_EQ(count, 2); - EXPECT_EQ(builder_.label_view_set(), ls_view_); - EXPECT_EQ(builder_.label_set(), ls_); -} - -TEST_F(TestLabelsBuilder, ResetRange) { - PromPP::Primitives::LabelViewSet ls; - for (auto& [lname, lvalue] : DATA) { - ls.add({lname, lvalue}); - ls_view_.add({lvalue, lname}); - ls_.add({lvalue, lname}); - } - - builder_.reset(ls); - - builder_.range([&](LNameType& lname, LValueType& lvalue) PROMPP_LAMBDA_INLINE -> bool { - builder_.del(lname); - builder_.set(lvalue, lname); - return true; - }); - - EXPECT_EQ(builder_.label_view_set(), ls_view_); - EXPECT_EQ(builder_.label_set(), ls_); -} - TEST_F(TestLabelsBuilder, NotIsEmpty) { for (auto& [lname, lvalue] : DATA) { builder_.set(lname, lvalue); diff --git a/pp/prometheus/relabeler.h b/pp/prometheus/relabeler.h index a57a08438e..f6bb8df606 100644 --- a/pp/prometheus/relabeler.h +++ b/pp/prometheus/relabeler.h @@ -1,6 +1,5 @@ #pragma once -#include #include #include #include @@ -29,14 +28,19 @@ struct MetricLimits { size_t label_value_length_limit{0}; size_t sample_limit{0}; - PROMPP_ALWAYS_INLINE bool label_limit_exceeded(size_t labels_count) { return label_limit > 0 && labels_count > label_limit; } - - PROMPP_ALWAYS_INLINE bool samples_limit_exceeded(size_t samples_count) { return sample_limit > 0 && samples_count >= sample_limit; } + [[nodiscard]] PROMPP_ALWAYS_INLINE bool label_limit_exceeded(size_t labels_count) const { return label_limit > 0 && labels_count > label_limit; } + [[nodiscard]] PROMPP_ALWAYS_INLINE bool samples_limit_exceeded(size_t samples_count) const { return sample_limit > 0 && samples_count >= sample_limit; } + [[nodiscard]] PROMPP_ALWAYS_INLINE bool label_name_length_limit_exceeded(size_t label_name_length) const { + return label_name_length_limit > 0 && label_name_length > label_name_length_limit; + } + [[nodiscard]] PROMPP_ALWAYS_INLINE bool label_value_length_limit_exceeded(size_t label_value_length) const { + return label_value_length_limit > 0 && label_value_length > label_value_length_limit; + } }; // hard_validate on empty, name label(__name__) mandatory, valid label name and value) validate label set. template -PROMPP_ALWAYS_INLINE void hard_validate(relabelStatus& rstatus, LabelsBuilder& builder, MetricLimits* limits) { +PROMPP_ALWAYS_INLINE void hard_validate(relabelStatus& rstatus, LabelsBuilder& builder, const MetricLimits* limits) { if (rstatus == rsDrop) { return; } @@ -54,13 +58,17 @@ PROMPP_ALWAYS_INLINE void hard_validate(relabelStatus& rstatus, LabelsBuilder& b } // validate labels - builder.range([&](LNameType& lname, LValueType& lvalue) PROMPP_LAMBDA_INLINE -> bool { - if (lname == kMetricLabelName && !metric_name_value_is_valid(lvalue)) { - rstatus = rsInvalid; - return false; + builder.range([&](const auto& lname, const auto& lvalue) PROMPP_LAMBDA_INLINE -> bool { + if (lname == kMetricLabelName) [[unlikely]] { + if (!metric_name_value_is_valid(lvalue)) [[unlikely]] { + rstatus = rsInvalid; + return false; + } + + return true; } - if (!label_name_is_valid(lname) || !label_value_is_valid(lvalue)) { + if (!label_name_is_valid(lname) || !label_value_is_valid(lvalue)) [[unlikely]] { rstatus = rsInvalid; return false; } @@ -86,13 +94,13 @@ PROMPP_ALWAYS_INLINE void hard_validate(relabelStatus& rstatus, LabelsBuilder& b } // check limit len label name and value - builder.range([&](LNameType& lname, LValueType& lvalue) PROMPP_LAMBDA_INLINE -> bool { - if (limits->label_name_length_limit > 0 && lname.size() > limits->label_name_length_limit) { + builder.range([&](const auto& lname, auto& lvalue) PROMPP_LAMBDA_INLINE -> bool { + if (limits->label_name_length_limit_exceeded(lname.size())) { rstatus = rsInvalid; return false; } - if (limits->label_value_length_limit > 0 && lvalue.size() > limits->label_value_length_limit) { + if (limits->label_value_length_limit_exceeded(lvalue.size())) { rstatus = rsInvalid; return false; } @@ -106,7 +114,7 @@ PROMPP_ALWAYS_INLINE void hard_validate(relabelStatus& rstatus, LabelsBuilder& b // samples - incoming samples; // ls_id - relabeling ls id from lss; struct InnerSerie { - PromPP::Primitives::Sample sample; + Primitives::Sample sample; uint32_t ls_id; PROMPP_ALWAYS_INLINE bool operator==(const InnerSerie& rt) const noexcept = default; @@ -121,17 +129,29 @@ class InnerSeries { BareBones::Vector data_; public: - PROMPP_ALWAYS_INLINE const BareBones::Vector& data() const { return data_; } + [[nodiscard]] PROMPP_ALWAYS_INLINE const BareBones::Vector& data() const { return data_; } - PROMPP_ALWAYS_INLINE size_t size() const { return size_; } + [[nodiscard]] PROMPP_ALWAYS_INLINE size_t size() const { return size_; } PROMPP_ALWAYS_INLINE void reserve(size_t n) { data_.reserve(n); } - PROMPP_ALWAYS_INLINE void emplace_back(const PromPP::Primitives::Sample& sample, const uint32_t& ls_id) { + PROMPP_ALWAYS_INLINE void emplace_back(const Primitives::Sample& sample, uint32_t ls_id) { data_.emplace_back(sample, ls_id); ++size_; } + PROMPP_ALWAYS_INLINE void emplace_back(auto const& samples, uint32_t ls_id) { + data_.reserve_and_write(samples.size(), [&](InnerSerie* series_buffer, uint32_t series_size) { + for (const auto& sample : samples) { + std::construct_at(series_buffer, sample, ls_id); + ++series_buffer; + } + return series_size; + }); + + size_ += samples.size(); + } + PROMPP_ALWAYS_INLINE void clear() noexcept { data_.clear(); size_ = 0; @@ -145,8 +165,8 @@ class InnerSeries { // hash - hash sum from ls; // ls_id - incoming ls id from lss; struct RelabeledSerie { - PromPP::Primitives::LabelSet ls; - BareBones::Vector samples; + Primitives::LabelSet ls; + BareBones::Vector samples; size_t hash; uint32_t ls_id; }; @@ -160,17 +180,25 @@ class RelabeledSeries { std::vector data_; public: - PROMPP_ALWAYS_INLINE const std::vector& data() const { return data_; } + [[nodiscard]] PROMPP_ALWAYS_INLINE const std::vector& data() const { return data_; } - PROMPP_ALWAYS_INLINE size_t size() const { return size_; } + [[nodiscard]] PROMPP_ALWAYS_INLINE size_t size() const { return size_; } - PROMPP_ALWAYS_INLINE void emplace_back(PromPP::Primitives::LabelSet& ls, - const BareBones::Vector& samples, + PROMPP_ALWAYS_INLINE void emplace_back(const Primitives::LabelSet& ls, + const BareBones::Vector& samples, const size_t hash, const uint32_t ls_id) { data_.emplace_back(ls, samples, hash, ls_id); ++size_; } + + PROMPP_ALWAYS_INLINE void emplace_back(const Primitives::LabelSet& ls, + BareBones::Vector&& samples, + const size_t hash, + const uint32_t ls_id) { + data_.emplace_back(ls, std::move(samples), hash, ls_id); + ++size_; + } }; // CacheValue - value for cache map. @@ -189,15 +217,15 @@ struct IncomingAndRelabeledLsID { }; // RelabelerStateUpdate - container for update states. -using RelabelerStateUpdate = PromPP::Primitives::Go::Slice; +using RelabelerStateUpdate = Primitives::Go::Slice; class NoOpStaleNaNsState { public: - PROMPP_ALWAYS_INLINE void add_input([[maybe_unused]] uint32_t id) {} - PROMPP_ALWAYS_INLINE void add_target([[maybe_unused]] uint32_t id) {} + PROMPP_ALWAYS_INLINE static void add_input([[maybe_unused]] uint32_t id) {} + PROMPP_ALWAYS_INLINE static void add_target([[maybe_unused]] uint32_t id) {} template - PROMPP_ALWAYS_INLINE void swap([[maybe_unused]] InputCallback input_fn, [[maybe_unused]] TargetCallback target_fn) {} + PROMPP_ALWAYS_INLINE static void swap([[maybe_unused]] InputCallback input_fn, [[maybe_unused]] TargetCallback target_fn) {} }; // StaleNaNsState state for stale nans. @@ -208,8 +236,6 @@ class StaleNaNsState { roaring::Roaring prev_target_bitset_{}; public: - PROMPP_ALWAYS_INLINE explicit StaleNaNsState() {} - PROMPP_ALWAYS_INLINE void add_input(uint32_t id) { input_bitset_.add(id); } PROMPP_ALWAYS_INLINE void add_target(uint32_t id) { target_bitset_.add(id); } @@ -230,32 +256,19 @@ class StaleNaNsState { // drop old, store new.. prev_target_bitset_ = std::move(target_bitset_); } - - PROMPP_ALWAYS_INLINE void reset() { - input_bitset_ = roaring::Roaring{}; - target_bitset_ = roaring::Roaring{}; - prev_input_bitset_ = roaring::Roaring{}; - prev_target_bitset_ = roaring::Roaring{}; - } }; // Cache stateless cache for relabeler. class Cache { size_t cache_allocated_memory_{0}; - phmap::parallel_flat_hash_map, - std::equal_to<>, - BareBones::Allocator>> - cache_relabel_{{}, {}, BareBones::Allocator>{cache_allocated_memory_}}; + phmap::parallel_flat_hash_map, std::equal_to<>, BareBones::Allocator>> + cache_relabel_{{}, {}, BareBones::Allocator>{cache_allocated_memory_}}; roaring::Roaring cache_keep_{}; roaring::Roaring cache_drop_{}; public: - PROMPP_ALWAYS_INLINE explicit Cache() {} - // allocated_memory return size of allocated memory for caches. - PROMPP_ALWAYS_INLINE size_t allocated_memory() const noexcept { + [[nodiscard]] PROMPP_ALWAYS_INLINE size_t allocated_memory() const noexcept { return cache_allocated_memory_ + cache_keep_.getSizeInBytes() + cache_drop_.getSizeInBytes(); } @@ -276,19 +289,13 @@ class Cache { cache_drop_.runOptimize(); } - PROMPP_ALWAYS_INLINE void reset() { - cache_relabel_.clear(); - cache_keep_ = roaring::Roaring{}; - cache_drop_ = roaring::Roaring{}; - } - - PROMPP_ALWAYS_INLINE double part_of_drops() { + [[nodiscard]] PROMPP_ALWAYS_INLINE double part_of_drops() const { if (cache_drop_.cardinality() == 0) { return 0; } return std::bit_cast(cache_drop_.cardinality()) / - std::bit_cast(cache_drop_.cardinality() + cache_keep_.cardinality() + static_cast(cache_relabel_.size())); + std::bit_cast(cache_drop_.cardinality() + cache_keep_.cardinality() + cache_relabel_.size()); } struct CheckResult { @@ -306,13 +313,12 @@ class Cache { template PROMPP_ALWAYS_INLINE CheckResult check(const InputLSS& input_lss, const TargetLSS& target_lss, LabelSet& label_set, size_t hash) { - if (std::optional ls_id = input_lss.find(label_set, hash); ls_id.has_value()) { - auto res = check_input(ls_id.value()); - if (res.status != CheckResult::kNotFound) { + if (const auto ls_id = input_lss.find(label_set, hash); ls_id.has_value()) { + if (auto res = check_input(ls_id.value()); res.status != CheckResult::kNotFound) { return res; } } - if (std::optional ls_id = target_lss.find(label_set, hash); ls_id.has_value()) { + if (const auto ls_id = target_lss.find(label_set, hash); ls_id.has_value()) { return check_target(ls_id.value()); } return {}; @@ -323,24 +329,32 @@ class Cache { return {.status = CheckResult::Status::kDrop}; } - if (auto it = cache_relabel_.find(ls_id); it != cache_relabel_.end()) { + if (const auto it = cache_relabel_.find(ls_id); it != cache_relabel_.end()) { return {.status = CheckResult::Status::kRelabel, .shard_id = it->second.shard_id, .ls_id = it->second.ls_id, .source_ls_id = ls_id}; } return {}; } - PROMPP_ALWAYS_INLINE CheckResult check_target(uint32_t ls_id) { + [[nodiscard]] PROMPP_ALWAYS_INLINE CheckResult check_target(uint32_t ls_id) const { if (cache_keep_.contains(ls_id)) { return {.status = CheckResult::Status::kKeep, .ls_id = ls_id}; } return {}; } + + // third stage + // update add to cache relabled data. + PROMPP_ALWAYS_INLINE void update(const RelabelerStateUpdate* relabeler_state_update, const uint16_t relabeled_shard_id) { + for (const auto& update : *relabeler_state_update) { + add_relabel(update.incoming_ls_id, update.relabeled_ls_id, relabeled_shard_id); + } + } }; struct RelabelerOptions { - PromPP::Primitives::Go::SliceView> target_labels{}; + Primitives::Go::SliceView> target_labels{}; MetricLimits* metric_limits{nullptr}; bool honor_labels{false}; bool track_timestamps_staleness{false}; @@ -356,21 +370,20 @@ struct RelabelerOptions { // shard_id_ - current shard id; // log_shards_ - logarithm to the base 2 of total shards count; class PerShardRelabeler { - std::stringstream buf_; - PromPP::Primitives::LabelsBuilderStateMap builder_state_; - std::vector external_labels_{}; - PromPP::Primitives::TimeseriesSemiview timeseries_buf_; + std::string buf_; + Primitives::LabelsBuilder builder_; + std::vector external_labels_{}; + Primitives::TimeseriesSemiview timeseries_buf_; StatelessRelabeler* stateless_relabeler_; uint16_t number_of_shards_; uint16_t shard_id_; public: // PerShardRelabeler - constructor. Init only with pre-initialized LSS* and StatelessRelabeler*. - PROMPP_ALWAYS_INLINE PerShardRelabeler( - PromPP::Primitives::Go::SliceView>& external_labels, - StatelessRelabeler* stateless_relabeler, - const uint16_t number_of_shards, - const uint16_t shard_id) + PROMPP_ALWAYS_INLINE PerShardRelabeler(Primitives::Go::SliceView>& external_labels, + StatelessRelabeler* stateless_relabeler, + const uint16_t number_of_shards, + const uint16_t shard_id) : stateless_relabeler_(stateless_relabeler), number_of_shards_(number_of_shards), shard_id_(shard_id) { if (stateless_relabeler_ == nullptr) [[unlikely]] { throw BareBones::Exception(0xabd6db40882fd6aa, "stateless relabeler is null pointer"); @@ -383,18 +396,18 @@ class PerShardRelabeler { } private: - PROMPP_ALWAYS_INLINE bool resolve_timestamps(PromPP::Primitives::Timestamp def_timestamp, - BareBones::Vector& samples, - const RelabelerOptions& o) { + PROMPP_ALWAYS_INLINE static bool resolve_timestamps(Primitives::Timestamp def_timestamp, + BareBones::Vector& samples, + const RelabelerOptions& o) { // skip resolve without stalenans - if (def_timestamp == PromPP::Primitives::kNullTimestamp) { + if (def_timestamp == Primitives::kNullTimestamp) { return false; } bool track_staleness{true}; for (auto& sample : samples) { // replace null timestamp on def timestamp - if (sample.timestamp() == PromPP::Primitives::kNullTimestamp) { + if (sample.timestamp() == Primitives::kNullTimestamp) { sample.timestamp() = def_timestamp; continue; } @@ -431,14 +444,15 @@ class PerShardRelabeler { const Hashdex& hashdex, const RelabelerOptions& o, Stats& stats, - PromPP::Primitives::Go::SliceView& shards_inner_series, - PromPP::Primitives::Go::SliceView& shards_relabeled_series, + Primitives::Go::SliceView& shards_inner_series, + Primitives::Go::SliceView& shards_relabeled_series, StNaNsState& stale_nan_state, - PromPP::Primitives::Timestamp def_timestamp) { + Primitives::Timestamp def_timestamp) { assert(number_of_shards_ > 0); - size_t n = std::min(static_cast(hashdex.size()), static_cast((hashdex.size() * (1 - cache.part_of_drops()) * 1.1) / number_of_shards_)); - for (auto i = 0; i < number_of_shards_; ++i) { + const size_t n = + std::min(static_cast(hashdex.size()), static_cast((hashdex.size() * (1 - cache.part_of_drops()) * 1.1) / number_of_shards_)); + for (uint16_t i = 0; i < number_of_shards_; ++i) { if (shards_inner_series[i]->size() >= n) { continue; } @@ -446,7 +460,6 @@ class PerShardRelabeler { shards_inner_series[i]->reserve(n); } - PromPP::Primitives::LabelsBuilder builder{builder_state_}; size_t samples_count{0}; for (auto it = skip_shard_inner_series(hashdex, shards_inner_series[shard_id_]->size()); it != hashdex.end(); ++it) { @@ -460,14 +473,9 @@ class PerShardRelabeler { Cache::CheckResult check_result = cache.check(input_lss, target_lss, timeseries_buf_.label_set(), it->hash()); switch (check_result.status) { case Cache::CheckResult::kNotFound: { - builder.reset(timeseries_buf_.label_set()); - auto rstatus = relabel(o, builder); - switch (rstatus) { - case rsDrop: { - cache.add_drop(input_lss.find_or_emplace(timeseries_buf_.label_set(), it->hash())); - ++stats.series_drop; - continue; - } + builder_.reset(timeseries_buf_.label_set()); + switch (relabel(o, builder_)) { + case rsDrop: case rsInvalid: { cache.add_drop(input_lss.find_or_emplace(timeseries_buf_.label_set(), it->hash())); ++stats.series_drop; @@ -477,11 +485,11 @@ class PerShardRelabeler { auto ls_id = target_lss.find_or_emplace(timeseries_buf_.label_set(), it->hash()); cache.add_keep(ls_id); auto& samples = timeseries_buf_.samples(); - bool all_samples_reseted_to_scrape_ts = resolve_timestamps(def_timestamp, samples, o); + const bool all_samples_reseted_to_scrape_ts = resolve_timestamps(def_timestamp, samples, o); if (o.track_timestamps_staleness || all_samples_reseted_to_scrape_ts) { stale_nan_state.add_target(ls_id); } - for (const PromPP::Primitives::Sample& sample : samples) { + for (const Primitives::Sample& sample : samples) { shards_inner_series[shard_id_]->emplace_back(sample, ls_id); } ++stats.series_added; @@ -490,11 +498,11 @@ class PerShardRelabeler { } case rsRelabel: { auto ls_id = input_lss.find_or_emplace(timeseries_buf_.label_set(), it->hash()); - PromPP::Primitives::LabelSet new_label_set = builder.label_set(); + const auto& new_label_set = builder_.label_set(); size_t new_hash = hash_value(new_label_set); - size_t new_shard_id = new_hash % number_of_shards_; + const size_t new_shard_id = new_hash % number_of_shards_; auto& samples = timeseries_buf_.samples(); - bool all_samples_reseted_to_scrape_ts = resolve_timestamps(def_timestamp, samples, o); + const bool all_samples_reseted_to_scrape_ts = resolve_timestamps(def_timestamp, samples, o); if (o.track_timestamps_staleness || all_samples_reseted_to_scrape_ts) { stale_nan_state.add_input(ls_id); } @@ -509,11 +517,11 @@ class PerShardRelabeler { } case Cache::CheckResult::kKeep: { auto& samples = timeseries_buf_.samples(); - bool all_samples_reseted_to_scrape_ts = resolve_timestamps(def_timestamp, samples, o); + const bool all_samples_reseted_to_scrape_ts = resolve_timestamps(def_timestamp, samples, o); if (o.track_timestamps_staleness || all_samples_reseted_to_scrape_ts) { stale_nan_state.add_target(check_result.ls_id); } - for (const PromPP::Primitives::Sample& sample : samples) { + for (const Primitives::Sample& sample : samples) { shards_inner_series[shard_id_]->emplace_back(sample, check_result.ls_id); } @@ -521,11 +529,11 @@ class PerShardRelabeler { } case Cache::CheckResult::kRelabel: { auto& samples = timeseries_buf_.samples(); - bool all_samples_reseted_to_scrape_ts = resolve_timestamps(def_timestamp, samples, o); + const bool all_samples_reseted_to_scrape_ts = resolve_timestamps(def_timestamp, samples, o); if (o.track_timestamps_staleness || all_samples_reseted_to_scrape_ts) { stale_nan_state.add_input(check_result.source_ls_id); } - for (const PromPP::Primitives::Sample& sample : samples) { + for (const Primitives::Sample& sample : samples) { shards_inner_series[check_result.shard_id]->emplace_back(sample, check_result.ls_id); } @@ -535,7 +543,7 @@ class PerShardRelabeler { continue; } - stats.samples_added += static_cast(timeseries_buf_.samples().size()); + stats.samples_added += timeseries_buf_.samples().size(); if (o.metric_limits == nullptr) { continue; @@ -547,15 +555,15 @@ class PerShardRelabeler { } } - PromPP::Primitives::Sample smpl{def_timestamp, kStaleNan}; + const Primitives::Sample smpl{def_timestamp, kStaleNan}; stale_nan_state.swap( [&](uint32_t ls_id) { - if (auto res = cache.check_input(ls_id); res.status == Cache::CheckResult::kRelabel) { + if (const auto res = cache.check_input(ls_id); res.status == Cache::CheckResult::kRelabel) { shards_inner_series[res.shard_id]->emplace_back(smpl, res.ls_id); } }, [&](uint32_t ls_id) { - if (auto res = cache.check_target(ls_id); res.status == Cache::CheckResult::kKeep) { + if (const auto res = cache.check_target(ls_id); res.status == Cache::CheckResult::kKeep) { shards_inner_series[shard_id_]->emplace_back(smpl, res.ls_id); } }); @@ -569,18 +577,19 @@ class PerShardRelabeler { const Hashdex& hashdex, const RelabelerOptions& o, Stats& stats, - PromPP::Primitives::Go::SliceView& shards_inner_series, + Primitives::Go::SliceView& shards_inner_series, StNaNsState& stale_nan_state, - PromPP::Primitives::Timestamp def_timestamp) { + Primitives::Timestamp def_timestamp) { assert(number_of_shards_ > 0); - size_t n = std::min(static_cast(hashdex.size()), static_cast((hashdex.size() * (1 - cache.part_of_drops()) * 1.1) / number_of_shards_)); - for (auto i = 0; i < number_of_shards_; ++i) { + const size_t n = + std::min(static_cast(hashdex.size()), static_cast((hashdex.size() * (1 - cache.part_of_drops()) * 1.1) / number_of_shards_)); + for (uint16_t i = 0; i < number_of_shards_; ++i) { shards_inner_series[i]->reserve(n); } size_t samples_count{0}; - PromPP::Primitives::TimeseriesSemiview timeseries_buf; + Primitives::TimeseriesSemiview timeseries_buf; for (const auto& item : hashdex) { if ((item.hash() % number_of_shards_) != shard_id_) { @@ -589,18 +598,18 @@ class PerShardRelabeler { timeseries_buf.clear(); item.read(timeseries_buf); - Cache::CheckResult check_result = cache.check(input_lss, target_lss, timeseries_buf.label_set(), item.hash()); + const auto check_result = cache.check(input_lss, target_lss, timeseries_buf.label_set(), item.hash()); switch (check_result.status) { case Cache::CheckResult::kNotFound: { return false; }; case Cache::CheckResult::kKeep: { auto& samples = timeseries_buf.samples(); - bool all_samples_reseted_to_scrape_ts = resolve_timestamps(def_timestamp, samples, o); + const bool all_samples_reseted_to_scrape_ts = resolve_timestamps(def_timestamp, samples, o); if (o.track_timestamps_staleness || all_samples_reseted_to_scrape_ts) { stale_nan_state.add_target(check_result.ls_id); } - for (const PromPP::Primitives::Sample& sample : samples) { + for (const Primitives::Sample& sample : samples) { shards_inner_series[shard_id_]->emplace_back(sample, check_result.ls_id); } @@ -608,11 +617,11 @@ class PerShardRelabeler { } case Cache::CheckResult::kRelabel: { auto& samples = timeseries_buf.samples(); - bool all_samples_reseted_to_scrape_ts = resolve_timestamps(def_timestamp, samples, o); + const bool all_samples_reseted_to_scrape_ts = resolve_timestamps(def_timestamp, samples, o); if (o.track_timestamps_staleness || all_samples_reseted_to_scrape_ts) { stale_nan_state.add_input(check_result.source_ls_id); } - for (const PromPP::Primitives::Sample& sample : samples) { + for (const Primitives::Sample& sample : samples) { shards_inner_series[check_result.shard_id]->emplace_back(sample, check_result.ls_id); } @@ -622,7 +631,7 @@ class PerShardRelabeler { continue; } - stats.samples_added += static_cast(timeseries_buf.samples().size()); + stats.samples_added += timeseries_buf.samples().size(); if (o.metric_limits == nullptr) { continue; @@ -634,15 +643,15 @@ class PerShardRelabeler { } } - PromPP::Primitives::Sample smpl{def_timestamp, kStaleNan}; + const Primitives::Sample smpl{def_timestamp, kStaleNan}; stale_nan_state.swap( [&](uint32_t ls_id) { - if (auto res = cache.check_input(ls_id); res.status == Cache::CheckResult::kRelabel) { + if (const auto res = cache.check_input(ls_id); res.status == Cache::CheckResult::kRelabel) { shards_inner_series[res.shard_id]->emplace_back(smpl, res.ls_id); } }, [&](uint32_t ls_id) { - if (auto res = cache.check_target(ls_id); res.status == Cache::CheckResult::kKeep) { + if (const auto res = cache.check_target(ls_id); res.status == Cache::CheckResult::kKeep) { shards_inner_series[shard_id_]->emplace_back(smpl, res.ls_id); } }); @@ -652,7 +661,7 @@ class PerShardRelabeler { template PROMPP_ALWAYS_INLINE relabelStatus relabel(const RelabelerOptions& o, LabelsBuilder& builder) { - bool changed = inject_target_labels(builder, o); + const bool changed = inject_target_labels(builder, o); relabelStatus rstatus = stateless_relabeler_->relabeling_process(buf_, builder); hard_validate(rstatus, builder, o.metric_limits); @@ -664,7 +673,7 @@ class PerShardRelabeler { } // calculate_samples counts the number of samples excluding stale_nan. - PROMPP_ALWAYS_INLINE size_t calculate_samples(const BareBones::Vector& samples) noexcept { + PROMPP_ALWAYS_INLINE static size_t calculate_samples(const BareBones::Vector& samples) noexcept { size_t samples_count{0}; for (const auto smpl : samples) { if (is_stale_nan(smpl.value())) { @@ -697,9 +706,9 @@ class PerShardRelabeler { return changed; } - std::vector conflicting_exposed_labels; + std::vector conflicting_exposed_labels; for (const auto& [lname, lvalue] : o.target_labels) { - PromPP::Primitives::Label existing_label = target_builder.extract(static_cast(lname)); + Primitives::Label existing_label = target_builder.extract(static_cast(lname)); if (!existing_label.second.empty()) [[likely]] { conflicting_exposed_labels.emplace_back(std::move(existing_label)); } @@ -719,9 +728,9 @@ class PerShardRelabeler { // resolve_conflicting_exposed_labels add prefix to conflicting label name. template - PROMPP_ALWAYS_INLINE void resolve_conflicting_exposed_labels(LabelsBuilder& builder, std::vector& conflicting_exposed_labels) { + PROMPP_ALWAYS_INLINE void resolve_conflicting_exposed_labels(LabelsBuilder& builder, std::vector& conflicting_exposed_labels) { std::stable_sort(conflicting_exposed_labels.begin(), conflicting_exposed_labels.end(), - [](PromPP::Primitives::LabelView a, PromPP::Primitives::LabelView b) { return a.first.size() < b.first.size(); }); + [](const Primitives::LabelView& a, const Primitives::LabelView& b) { return a.first.size() < b.first.size(); }); for (auto& [ln, lv] : conflicting_exposed_labels) { while (true) { @@ -741,11 +750,10 @@ class PerShardRelabeler { const Hashdex& hashdex, const RelabelerOptions& o, Stats& stats, - PromPP::Primitives::Go::SliceView& shards_inner_series, - PromPP::Primitives::Go::SliceView& shards_relabeled_series) { + Primitives::Go::SliceView& shards_inner_series, + Primitives::Go::SliceView& shards_relabeled_series) { NoOpStaleNaNsState state{}; - input_relabeling_internal(input_lss, target_lss, cache, hashdex, o, stats, shards_inner_series, shards_relabeled_series, state, - PromPP::Primitives::kNullTimestamp); + input_relabeling_internal(input_lss, target_lss, cache, hashdex, o, stats, shards_inner_series, shards_relabeled_series, state, Primitives::kNullTimestamp); } template @@ -755,10 +763,10 @@ class PerShardRelabeler { const Hashdex& hashdex, const RelabelerOptions& o, Stats& stats, - PromPP::Primitives::Go::SliceView& shards_inner_series, - PromPP::Primitives::Go::SliceView& shards_relabeled_series, + Primitives::Go::SliceView& shards_inner_series, + Primitives::Go::SliceView& shards_relabeled_series, StaleNaNsState& state, - PromPP::Primitives::Timestamp def_timestamp) { + Primitives::Timestamp def_timestamp) { input_relabeling_internal(input_lss, target_lss, cache, hashdex, o, stats, shards_inner_series, shards_relabeled_series, state, def_timestamp); } @@ -769,10 +777,9 @@ class PerShardRelabeler { const Hashdex& hashdex, const RelabelerOptions& o, Stats& stats, - PromPP::Primitives::Go::SliceView& shards_inner_series) { + Primitives::Go::SliceView& shards_inner_series) { NoOpStaleNaNsState state{}; - return input_relabeling_from_cache_internal(input_lss, target_lss, cache, hashdex, o, stats, shards_inner_series, state, - PromPP::Primitives::kNullTimestamp); + return input_relabeling_from_cache_internal(input_lss, target_lss, cache, hashdex, o, stats, shards_inner_series, state, Primitives::kNullTimestamp); } template @@ -782,25 +789,25 @@ class PerShardRelabeler { const Hashdex& hashdex, const RelabelerOptions& o, Stats& stats, - PromPP::Primitives::Go::SliceView& shards_inner_series, + Primitives::Go::SliceView& shards_inner_series, StaleNaNsState& state, - PromPP::Primitives::Timestamp def_timestamp) { + Primitives::Timestamp def_timestamp) { return input_relabeling_from_cache_internal(input_lss, target_lss, cache, hashdex, o, stats, shards_inner_series, state, def_timestamp); } PROMPP_ALWAYS_INLINE void input_collect_stalenans(Cache& cache, - PromPP::Primitives::Go::SliceView& shards_inner_series, + Primitives::Go::SliceView& shards_inner_series, StaleNaNsState& state, - PromPP::Primitives::Timestamp stale_ts) { - PromPP::Primitives::Sample smpl{stale_ts, kStaleNan}; + Primitives::Timestamp stale_ts) const { + const Primitives::Sample smpl{stale_ts, kStaleNan}; state.swap( [&](uint32_t ls_id) { - if (auto res = cache.check_input(ls_id); res.status == Cache::CheckResult::kRelabel) { + if (const auto res = cache.check_input(ls_id); res.status == Cache::CheckResult::kRelabel) { shards_inner_series[res.shard_id]->emplace_back(smpl, res.ls_id); } }, [&](uint32_t ls_id) { - if (auto res = cache.check_target(ls_id); res.status == Cache::CheckResult::kKeep) { + if (const auto res = cache.check_target(ls_id); res.status == Cache::CheckResult::kKeep) { shards_inner_series[shard_id_]->emplace_back(smpl, res.ls_id); } }); @@ -809,14 +816,14 @@ class PerShardRelabeler { // append_relabeler_series add relabeled ls to lss, add to result and add to cache update(second stage). template - PROMPP_ALWAYS_INLINE void append_relabeler_series(LSS& lss, - InnerSeries* inner_series, - const RelabeledSeries* relabeled_series, - RelabelerStateUpdate* relabeler_state_update) { + PROMPP_ALWAYS_INLINE static void append_relabeler_series(LSS& lss, + InnerSeries* inner_series, + const RelabeledSeries* relabeled_series, + RelabelerStateUpdate* relabeler_state_update) { relabeler_state_update->reserve(relabeler_state_update->size() + relabeled_series->size()); inner_series->reserve(inner_series->size() + relabeled_series->size()); if constexpr (BareBones::concepts::has_reserve) { - lss.reserve(lss.size() + relabeled_series->size()); + lss.reserve(static_cast(lss.size() + relabeled_series->size())); } for (const auto& relabeled_serie : relabeled_series->data()) { @@ -830,7 +837,7 @@ class PerShardRelabeler { } // update_relabeler_state - add to cache relabled data(third stage). - PROMPP_ALWAYS_INLINE void update_relabeler_state(Cache& cache, const RelabelerStateUpdate* relabeler_state_update, const uint16_t relabeled_shard_id) { + PROMPP_ALWAYS_INLINE static void update_relabeler_state(Cache& cache, const RelabelerStateUpdate* relabeler_state_update, const uint16_t relabeled_shard_id) { for (const auto& update : *relabeler_state_update) { cache.add_relabel(update.incoming_ls_id, update.relabeled_ls_id, relabeled_shard_id); } @@ -841,18 +848,15 @@ class PerShardRelabeler { PROMPP_ALWAYS_INLINE void output_relabeling(const LSS& lss, Cache& cache, RelabeledSeries* relabeled_series, - PromPP::Primitives::Go::SliceView& incoming_inner_series, - PromPP::Primitives::Go::SliceView& encoders_inner_series) { - std::ranges::for_each(incoming_inner_series, [&](const InnerSeries* inner_series) PROMPP_LAMBDA_INLINE { + Primitives::Go::SliceView& incoming_inner_series, + Primitives::Go::SliceView& encoders_inner_series) { + for (const auto inner_series : incoming_inner_series) { if (inner_series == nullptr || inner_series->size() == 0) { - return; + continue; } - // TODO move ctor builder from ranges for; - PromPP::Primitives::LabelsBuilder builder{builder_state_}; - std::ranges::for_each(inner_series->data(), [&](const InnerSerie& inner_serie) PROMPP_LAMBDA_INLINE { - auto res = cache.check_input(inner_serie.ls_id); + const auto res = cache.check_input(inner_serie.ls_id); if (res.status == Cache::CheckResult::kDrop) { return; } @@ -865,30 +869,27 @@ class PerShardRelabeler { if (inner_serie.ls_id >= lss.size()) [[unlikely]] { throw BareBones::Exception(0x7763a97e1717e835, "ls_id out of range: %d size: %d shard_id: %d", inner_serie.ls_id, lss.size(), shard_id_); } - typename LSS::value_type labels = lss[inner_serie.ls_id]; - builder.reset(labels); - process_external_labels(builder, external_labels_); + builder_.reset(lss[inner_serie.ls_id]); + process_external_labels(builder_, external_labels_); - relabelStatus rstatus = stateless_relabeler_->relabeling_process(buf_, builder); - soft_validate(rstatus, builder); + relabelStatus rstatus = stateless_relabeler_->relabeling_process(buf_, builder_); + soft_validate(rstatus, builder_); if (rstatus == rsDrop) { cache.add_drop(inner_serie.ls_id); return; } - PromPP::Primitives::LabelSet new_label_set = builder.label_set(); - relabeled_series->emplace_back(new_label_set, BareBones::Vector{inner_serie.sample}, hash_value(new_label_set), - inner_serie.ls_id); + const auto& new_label_set = builder_.label_set(); + relabeled_series->emplace_back(new_label_set, BareBones::Vector{inner_serie.sample}, hash_value(new_label_set), inner_serie.ls_id); }); - }); + } cache.optimize(); } // reset set new number_of_shards and external_labels. - PROMPP_ALWAYS_INLINE void reset_to( - const PromPP::Primitives::Go::SliceView>& external_labels, - const uint16_t number_of_shards) { + PROMPP_ALWAYS_INLINE void reset_to(const Primitives::Go::SliceView>& external_labels, + const uint16_t number_of_shards) { number_of_shards_ = number_of_shards; external_labels_.clear(); external_labels_.reserve(external_labels.size()); @@ -896,8 +897,473 @@ class PerShardRelabeler { external_labels_.emplace_back(static_cast(ln), static_cast(lv)); } } +}; + +// +// PerGoroutineRelabeler +// + +// PerGoroutineRelabeler stateful relabeler for shard goroutines. +class PerGoroutineRelabeler { + std::string buf_; + Primitives::LabelsBuilder builder_; + Primitives::TimeseriesSemiview timeseries_buf_; + uint16_t number_of_shards_; + uint16_t shard_id_; + + public: + // PerShardRelabeler constructor. + PROMPP_ALWAYS_INLINE PerGoroutineRelabeler(const uint16_t number_of_shards, const uint16_t shard_id) + : number_of_shards_(number_of_shards), shard_id_(shard_id) {} + + private: + PROMPP_ALWAYS_INLINE static size_t non_stale_nan_samples_count(const BareBones::Vector& samples) noexcept { + return std::ranges::count_if(samples, [](const Primitives::Sample& sample) { return !is_stale_nan(sample.value()); }); + } + + // inject_target_labels add labels from target to builder. + template + PROMPP_ALWAYS_INLINE bool inject_target_labels(LabelsBuilder& target_builder, const RelabelerOptions& options) { + if (options.target_labels.empty()) { + return false; + } + + bool changed{false}; + + if (options.honor_labels) { + for (const auto& [lname, lvalue] : options.target_labels) { + if (target_builder.contains(static_cast(lname))) [[unlikely]] { + continue; + } + + target_builder.set(static_cast(lname), static_cast(lvalue)); + changed = true; + } + + return changed; + } + + std::vector conflicting_exposed_labels; + for (const auto& [lname, lvalue] : options.target_labels) { + Primitives::Label existing_label = target_builder.extract(static_cast(lname)); + if (!existing_label.second.empty()) [[likely]] { + conflicting_exposed_labels.emplace_back(std::move(existing_label)); + } + + // It is now safe to set the target label. + target_builder.set(static_cast(lname), static_cast(lvalue)); + changed = true; + } + + // resolve conflict + if (!conflicting_exposed_labels.empty()) { + resolve_conflicting_exposed_labels(target_builder, conflicting_exposed_labels); + } + + return changed; + } + + template + PROMPP_ALWAYS_INLINE bool input_relabeling_from_cache_internal(InputLSS& input_lss, + TargetLSS& target_lss, + Cache& cache, + const Hashdex& hashdex, + const RelabelerOptions& options, + Stats& stats, + Primitives::Go::SliceView& shards_inner_series, + StNaNsState& stale_nan_state, + Primitives::Timestamp def_timestamp) { + bool result{true}; + size_t samples_count{}; + fill_inner_series(hashdex, hashdex.begin(), shards_inner_series, [&](auto& item) { + Cache::CheckResult check_result = cache.check(input_lss, target_lss, timeseries_buf_.label_set(), item.hash()); + switch (check_result.status) { + case Cache::CheckResult::kNotFound: { + result = false; + return false; + }; + + case Cache::CheckResult::kKeep: { + auto& samples = timeseries_buf_.samples(); + const bool all_samples_reseted_to_scrape_ts = resolve_timestamps(def_timestamp, samples, options); + if (options.track_timestamps_staleness || all_samples_reseted_to_scrape_ts) { + stale_nan_state.add_target(check_result.ls_id); + } + + shards_inner_series[shard_id_]->emplace_back(samples, check_result.ls_id); + break; + } + + case Cache::CheckResult::kRelabel: { + auto& samples = timeseries_buf_.samples(); + const bool all_samples_reseted_to_scrape_ts = resolve_timestamps(def_timestamp, samples, options); + if (options.track_timestamps_staleness || all_samples_reseted_to_scrape_ts) { + stale_nan_state.add_input(check_result.source_ls_id); + } + + shards_inner_series[check_result.shard_id]->emplace_back(samples, check_result.ls_id); + break; + } + + default: { + return true; + } + } + + stats.samples_added += timeseries_buf_.samples().size(); + + if (options.metric_limits != nullptr) { + samples_count += non_stale_nan_samples_count(timeseries_buf_.samples()); + if (options.metric_limits->samples_limit_exceeded(samples_count)) { + return false; + } + } + + return true; + }); + + if (result) { + add_stale_nans(cache, shards_inner_series, stale_nan_state, def_timestamp); + } + + return result; + } - PROMPP_ALWAYS_INLINE ~PerShardRelabeler() = default; + template + PROMPP_ALWAYS_INLINE void input_relabeling_internal(InputLSS& input_lss, + TargetLSS& target_lss, + Cache& cache, + const Hashdex& hashdex, + const RelabelerOptions& options, + const StatelessRelabeler& stateless_relabeler, + Stats& stats, + Primitives::Go::SliceView& shards_inner_series, + Primitives::Go::SliceView& shards_relabeled_series, + StNaNsState& stale_nan_state, + Primitives::Timestamp def_timestamp) { + size_t samples_count{}; + fill_inner_series(hashdex, skip_shard_inner_series(hashdex, shards_inner_series[shard_id_]->size()), shards_inner_series, [&](auto& item) { + Cache::CheckResult check_result = cache.check(input_lss, target_lss, timeseries_buf_.label_set(), item.hash()); + switch (check_result.status) { + case Cache::CheckResult::kNotFound: { + builder_.reset(timeseries_buf_.label_set()); + switch (relabel(options, stateless_relabeler, builder_)) { + case rsDrop: + case rsInvalid: { + cache.add_drop(input_lss.find_or_emplace(timeseries_buf_.label_set(), item.hash())); + ++stats.series_drop; + return true; + } + + case rsKeep: { + auto ls_id = target_lss.find_or_emplace(timeseries_buf_.label_set(), item.hash()); + cache.add_keep(ls_id); + auto& samples = timeseries_buf_.samples(); + const bool all_samples_reseted_to_scrape_ts = resolve_timestamps(def_timestamp, samples, options); + if (options.track_timestamps_staleness || all_samples_reseted_to_scrape_ts) { + stale_nan_state.add_target(ls_id); + } + + shards_inner_series[shard_id_]->emplace_back(samples, ls_id); + ++stats.series_added; + + break; + } + + case rsRelabel: { + auto ls_id = input_lss.find_or_emplace(timeseries_buf_.label_set(), item.hash()); + const auto& new_label_set = builder_.label_set(); + size_t new_hash = hash_value(new_label_set); + const size_t new_shard_id = new_hash % number_of_shards_; + auto& samples = timeseries_buf_.samples(); + const bool all_samples_reseted_to_scrape_ts = resolve_timestamps(def_timestamp, samples, options); + if (options.track_timestamps_staleness || all_samples_reseted_to_scrape_ts) { + stale_nan_state.add_input(ls_id); + } + + shards_relabeled_series[new_shard_id]->emplace_back(new_label_set, samples, new_hash, ls_id); + ++stats.series_added; + + break; + } + } + + break; + } + + case Cache::CheckResult::kKeep: { + auto& samples = timeseries_buf_.samples(); + const bool all_samples_reseted_to_scrape_ts = resolve_timestamps(def_timestamp, samples, options); + if (options.track_timestamps_staleness || all_samples_reseted_to_scrape_ts) { + stale_nan_state.add_target(check_result.ls_id); + } + + shards_inner_series[shard_id_]->emplace_back(samples, check_result.ls_id); + break; + } + + case Cache::CheckResult::kRelabel: { + auto& samples = timeseries_buf_.samples(); + const bool all_samples_reseted_to_scrape_ts = resolve_timestamps(def_timestamp, samples, options); + if (options.track_timestamps_staleness || all_samples_reseted_to_scrape_ts) { + stale_nan_state.add_input(check_result.source_ls_id); + } + + shards_inner_series[check_result.shard_id]->emplace_back(samples, check_result.ls_id); + break; + } + + default: { + break; + } + } + + stats.samples_added += timeseries_buf_.samples().size(); + + if (options.metric_limits != nullptr) { + samples_count += non_stale_nan_samples_count(timeseries_buf_.samples()); + if (options.metric_limits->samples_limit_exceeded(samples_count)) { + return false; + } + } + + return true; + }); + + add_stale_nans(cache, shards_inner_series, stale_nan_state, def_timestamp); + cache.optimize(); + } + + template + PROMPP_ALWAYS_INLINE relabelStatus relabel(const RelabelerOptions& options, const StatelessRelabeler& stateless_relabeler, LabelsBuilder& builder) { + const bool changed = inject_target_labels(builder, options); + + relabelStatus rstatus = stateless_relabeler.relabeling_process(buf_, builder); + hard_validate(rstatus, builder, options.metric_limits); + if (changed && rstatus == rsKeep) { + rstatus = rsRelabel; + } + + return rstatus; + } + + // resolve_conflicting_exposed_labels add prefix to conflicting label name. + template + PROMPP_ALWAYS_INLINE void resolve_conflicting_exposed_labels(LabelsBuilder& builder, std::vector& conflicting_exposed_labels) { + std::stable_sort(conflicting_exposed_labels.begin(), conflicting_exposed_labels.end(), + [](const Primitives::LabelView& a, const Primitives::LabelView& b) { return a.first.size() < b.first.size(); }); + + for (auto& [ln, lv] : conflicting_exposed_labels) { + while (true) { + ln.insert(0, "exported_"); + if (builder.get(ln).empty()) { + builder.set(ln, lv); + break; + } + } + } + } + + PROMPP_ALWAYS_INLINE static bool resolve_timestamps(Primitives::Timestamp def_timestamp, + BareBones::Vector& samples, + const RelabelerOptions& options) { + // skip resolve without stalenans + if (def_timestamp == Primitives::kNullTimestamp) { + return false; + } + + bool track_staleness{true}; + for (auto& sample : samples) { + // replace null timestamp on def timestamp + if (sample.timestamp() == Primitives::kNullTimestamp) { + sample.timestamp() = def_timestamp; + continue; + } + + // replace incoming timestamp on def timestamp + if (!options.honor_timestamps) { + sample.timestamp() = def_timestamp; + continue; + } + + track_staleness = false; + } + + return track_staleness; + } + + template + [[nodiscard]] PROMPP_ALWAYS_INLINE auto skip_shard_inner_series(const Hashdex& hashdex, size_t i) { + auto it = hashdex.begin(); + for (; it != hashdex.end() && i > 0; ++it) { + if ((it->hash() % number_of_shards_) != shard_id_) { + continue; + } + --i; + } + + return it; + } + + public: + // first stage + template + PROMPP_ALWAYS_INLINE void input_relabeling(InputLSS& input_lss, + TargetLSS& target_lss, + Cache& cache, + const Hashdex& hashdex, + const RelabelerOptions& options, + const StatelessRelabeler& stateless_relabeler, + Stats& stats, + Primitives::Go::SliceView& shards_inner_series, + Primitives::Go::SliceView& shards_relabeled_series) { + NoOpStaleNaNsState state{}; + input_relabeling_internal(input_lss, target_lss, cache, hashdex, options, stateless_relabeler, stats, shards_inner_series, shards_relabeled_series, state, + Primitives::kNullTimestamp); + } + + template + PROMPP_ALWAYS_INLINE bool input_relabeling_from_cache(InputLSS& input_lss, + TargetLSS& target_lss, + Cache& cache, + const Hashdex& hashdex, + const RelabelerOptions& options, + Stats& stats, + Primitives::Go::SliceView& shards_inner_series) { + NoOpStaleNaNsState state{}; + return input_relabeling_from_cache_internal(input_lss, target_lss, cache, hashdex, options, stats, shards_inner_series, state, Primitives::kNullTimestamp); + } + + template + PROMPP_ALWAYS_INLINE void input_relabeling_with_stalenans(InputLSS& input_lss, + TargetLSS& target_lss, + Cache& cache, + const Hashdex& hashdex, + const RelabelerOptions& options, + const StatelessRelabeler& stateless_relabeler, + Stats& stats, + Primitives::Go::SliceView& shards_inner_series, + Primitives::Go::SliceView& shards_relabeled_series, + StaleNaNsState& state, + Primitives::Timestamp def_timestamp) { + input_relabeling_internal(input_lss, target_lss, cache, hashdex, options, stateless_relabeler, stats, shards_inner_series, shards_relabeled_series, state, + def_timestamp); + } + + template + PROMPP_ALWAYS_INLINE bool input_relabeling_with_stalenans_from_cache(InputLSS& input_lss, + TargetLSS& target_lss, + Cache& cache, + const Hashdex& hashdex, + const RelabelerOptions& options, + Stats& stats, + Primitives::Go::SliceView& shards_inner_series, + StaleNaNsState& state, + Primitives::Timestamp def_timestamp) { + return input_relabeling_from_cache_internal(input_lss, target_lss, cache, hashdex, options, stats, shards_inner_series, state, def_timestamp); + } + + // input_transition_relabeling transparent relabeling. + template + PROMPP_ALWAYS_INLINE void input_transition_relabeling(TargetLSS& target_lss, + const Hashdex& hashdex, + Stats& stats, + Primitives::Go::SliceView& shards_inner_series) { + fill_inner_series(hashdex, skip_shard_inner_series(hashdex, shards_inner_series[shard_id_]->size()), shards_inner_series, [&](auto& item) { + const auto previous_size = target_lss.size(); + auto ls_id = target_lss.find_or_emplace(timeseries_buf_.label_set(), item.hash()); + shards_inner_series[shard_id_]->emplace_back(timeseries_buf_.samples(), ls_id); + + if (target_lss.size() > previous_size) { + ++stats.series_added; + } + stats.samples_added += timeseries_buf_.samples().size(); + return true; + }); + } + + // input_transition_relabeling_from_cache transparent relabeling with only reading from the lss. + template + PROMPP_ALWAYS_INLINE bool input_transition_relabeling_only_read(TargetLSS& target_lss, + const Hashdex& hashdex, + Stats& stats, + Primitives::Go::SliceView& shards_inner_series) { + bool result = true; + fill_inner_series(hashdex, hashdex.begin(), shards_inner_series, [&](auto& item) { + if (auto ls_id = target_lss.find(timeseries_buf_.label_set(), item.hash()); ls_id.has_value()) { + shards_inner_series[shard_id_]->emplace_back(timeseries_buf_.samples(), *ls_id); + stats.samples_added += timeseries_buf_.samples().size(); + return true; + } + + result = false; + return false; + }); + + return result; + } + + // second stage + // append_relabeler_series add relabeled ls to lss, add to result and add to cache update. + template + PROMPP_ALWAYS_INLINE static void append_relabeler_series(LSS& target_lss, + InnerSeries* inner_series, + const RelabeledSeries* relabeled_series, + RelabelerStateUpdate* relabeler_state_update) { + relabeler_state_update->reserve(relabeler_state_update->size() + relabeled_series->size()); + inner_series->reserve(inner_series->size() + relabeled_series->size()); + if constexpr (BareBones::concepts::has_reserve) { + target_lss.reserve(target_lss.size() + relabeled_series->size()); + } + + for (const auto& relabeled_serie : relabeled_series->data()) { + uint32_t ls_id = target_lss.find_or_emplace(relabeled_serie.ls, relabeled_serie.hash); + inner_series->emplace_back(relabeled_serie.samples, ls_id); + relabeler_state_update->emplace_back(relabeled_serie.ls_id, ls_id); + } + } + + template + void fill_inner_series(const Hashdex& hashdex, auto hashdex_it, Primitives::Go::SliceView& shards_inner_series, Handler handler) { + assert(number_of_shards_ > 0); + + const size_t n = std::min(static_cast(hashdex.size()), static_cast((hashdex.size() * 1.1) / number_of_shards_)); + for (uint16_t i = 0; i < number_of_shards_; ++i) { + shards_inner_series[i]->reserve(n); + } + + for (; hashdex_it != hashdex.end(); ++hashdex_it) { + if ((hashdex_it->hash() % number_of_shards_) != shard_id_) { + continue; + } + + timeseries_buf_.clear(); + hashdex_it->read(timeseries_buf_); + + if (!handler(*hashdex_it)) { + break; + } + } + } + + template + PROMPP_ALWAYS_INLINE void add_stale_nans(Cache& cache, + Primitives::Go::SliceView& shards_inner_series, + StNaNsState& stale_nan_state, + Primitives::Timestamp def_timestamp) { + const Primitives::Sample smpl{def_timestamp, kStaleNan}; + stale_nan_state.swap( + [&](uint32_t ls_id) { + if (const auto res = cache.check_input(ls_id); res.status == Cache::CheckResult::kRelabel) { + shards_inner_series[res.shard_id]->emplace_back(smpl, res.ls_id); + } + }, + [&](uint32_t ls_id) { + if (const auto res = cache.check_target(ls_id); res.status == Cache::CheckResult::kKeep) { + shards_inner_series[shard_id_]->emplace_back(smpl, res.ls_id); + } + }); + } }; } // namespace PromPP::Prometheus::Relabel diff --git a/pp/prometheus/stateless_relabeler.h b/pp/prometheus/stateless_relabeler.h index 33a3a8c7db..885e014109 100644 --- a/pp/prometheus/stateless_relabeler.h +++ b/pp/prometheus/stateless_relabeler.h @@ -1,20 +1,14 @@ #pragma once -#include #include #include #include #include #include -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wold-style-cast" -#pragma GCC diagnostic ignored "-Wpedantic" -#pragma GCC diagnostic ignored "-Wswitch" -#include "re2/re2.h" -#pragma GCC diagnostic pop #include #include +#include "re2/re2.h" #include "bare_bones/bit.h" #include "bare_bones/exception.h" @@ -24,8 +18,8 @@ namespace PromPP::Prometheus::Relabel { // label_name_is_valid validate label name. -PROMPP_ALWAYS_INLINE bool label_name_is_valid(std::string_view name) { - if (name.size() == 0) { +PROMPP_ALWAYS_INLINE bool label_name_is_valid(const std::string_view& name) { + if (name.empty()) { return false; } @@ -41,13 +35,13 @@ PROMPP_ALWAYS_INLINE bool label_name_is_valid(std::string_view name) { } // label_value_is_valid validate label value. -PROMPP_ALWAYS_INLINE bool label_value_is_valid(std::string_view value) noexcept { +PROMPP_ALWAYS_INLINE bool label_value_is_valid(const std::string_view& value) noexcept { return simdutf::validate_utf8(value.data(), value.length()); } // metric_name_value_is_valid validate value for metric name(__name__). -PROMPP_ALWAYS_INLINE bool metric_name_value_is_valid(std::string_view value) { - if (value.size() == 0) { +PROMPP_ALWAYS_INLINE bool metric_name_value_is_valid(const std::string_view& value) { + if (value.empty()) { return false; } @@ -85,15 +79,13 @@ class PatternPart { PROMPP_ALWAYS_INLINE explicit PatternPart(int g) : type_(pGroup), data_{.group_ = g} {} // write - convert parts to out. - PROMPP_ALWAYS_INLINE void write(std::ostream& out, std::vector& groups) { + PROMPP_ALWAYS_INLINE void write(std::string& out, const std::vector& groups) const { if (type_ == pGroup) { - out << groups[data_.group_]; + out += groups[data_.group_]; } else { - out << data_.string_; + out += data_.string_; } } - - ~PatternPart() = default; }; // Regexp - wrapper on re2. @@ -103,17 +95,14 @@ class Regexp { public: // Regexp - work without ("^(?:" + std::string(s) + ")$"). - PROMPP_ALWAYS_INLINE explicit Regexp(const std::string_view& s) noexcept : re_(std::make_unique(std::string(s))) {} - - PROMPP_ALWAYS_INLINE Regexp(Regexp&&) noexcept = default; - PROMPP_ALWAYS_INLINE ~Regexp() = default; + PROMPP_ALWAYS_INLINE explicit Regexp(const std::string_view& s) noexcept : re_(std::make_unique(s)) {} // number_of_capturing_groups - return the number of capturing sub-patterns, or -1 if the regexp wasn't valid on construction. The overall match ($0) does not // count. Use in test. - PROMPP_ALWAYS_INLINE int number_of_capturing_groups() { return re_->NumberOfCapturingGroups(); } + [[nodiscard]] PROMPP_ALWAYS_INLINE int number_of_capturing_groups() const { return re_->NumberOfCapturingGroups(); } // groups - get named capturing groups and number groups. - PROMPP_ALWAYS_INLINE std::map groups() { + [[nodiscard]] PROMPP_ALWAYS_INLINE std::map groups() const { // get named capturing groups std::map named_groups = re_->NamedCapturingGroups(); // add number groups to named capturing groups @@ -125,8 +114,8 @@ class Regexp { } // match_to_args - match expression and return result args. - PROMPP_ALWAYS_INLINE bool match_to_args(std::string_view src, std::vector& res) { - int n = number_of_capturing_groups(); + PROMPP_ALWAYS_INLINE bool match_to_args(std::string_view src, std::vector& res) const { + const int n = number_of_capturing_groups(); // search full match to args, where size - number of capturing groups res.resize(n + 1); @@ -139,60 +128,65 @@ class Regexp { re_args.emplace_back(&res[i]); re_args_ptr.emplace_back(&re_args[i - 1]); } - bool ok = RE2::FullMatchN(src, *re_.get(), &re_args_ptr[0], n); - return ok; + + if (!RE2::FullMatchN(src, *re_, &re_args_ptr[0], n)) { + res.clear(); + return false; + } + + return true; } // replace_with_args - replace in template with incoming args. - PROMPP_ALWAYS_INLINE std::string replace_with_args(std::stringstream& buf, std::vector& args, std::vector& tmpl) { - if (tmpl.size() == 0) [[unlikely]] { + PROMPP_ALWAYS_INLINE static void replace_with_args(std::string& buf, const std::vector& args, const std::vector& tmpl) { + buf.clear(); + + if (tmpl.empty()) [[unlikely]] { // no template or source data - return ""; + return; } - buf.str(""); for (auto& val : tmpl) { val.write(buf, args); } - - return buf.str(); } // replace_full - find match for source and replace in template. - PROMPP_ALWAYS_INLINE std::string replace_full(std::stringstream& out, std::string_view src, std::vector& tmpl) { - if (src.size() == 0 || tmpl.size() == 0) [[unlikely]] { + PROMPP_ALWAYS_INLINE void replace_full(std::string& out, std::string_view src, const std::vector& tmpl) const { + out.clear(); + + if (src.empty() || tmpl.empty()) [[unlikely]] { // no template or source data - return ""; + return; } - std::vector res_args; - bool ok = match_to_args(src, res_args); - if (!ok) { + std::vector res_args; + if (!match_to_args(src, res_args)) { // no entries in regexp - return ""; + return; } - return replace_with_args(out, res_args, tmpl); + replace_with_args(out, res_args, tmpl); } // full_match - check text for full match regexp. - PROMPP_ALWAYS_INLINE bool full_match(std::string_view str) { return RE2::FullMatch(str, *re_.get()); } + [[nodiscard]] PROMPP_ALWAYS_INLINE bool full_match(std::string_view str) const { return RE2::FullMatch(str, *re_); } }; struct GORelabelConfig { // source_labels - a list of labels from which values are taken and concatenated with the configured separator in order. - PromPP::Primitives::Go::SliceView source_labels; + Primitives::Go::SliceView source_labels; // separator - is the string between concatenated values from the source labels. - PromPP::Primitives::Go::String separator; + Primitives::Go::String separator; // regex - against which the concatenation is matched. - PromPP::Primitives::Go::String regex; + Primitives::Go::String regex; // modulus - to take of the hash of concatenated values from the source labels. uint64_t modulus; // target_label - is the label to which the resulting string is written in a replacement. // Regexp interpolation is allowed for the replace action. - PromPP::Primitives::Go::String target_label; + Primitives::Go::String target_label; // replacement - is the regex replacement pattern to be used. - PromPP::Primitives::Go::String replacement; + Primitives::Go::String replacement; // action - is the action to be performed for the relabeling. uint8_t action; }; @@ -260,27 +254,26 @@ class RelabelConfig { std::vector replacement_parts_; // extract - extract from source letter or digit value. - PROMPP_ALWAYS_INLINE std::string extract(re2::RE2& rgx_validate, std::string_view src) { + PROMPP_ALWAYS_INLINE static std::string extract(const re2::RE2& rgx_validate, const std::string_view& src) { std::string name; RE2::PartialMatch(src, rgx_validate, &name); return name; } // is_valid_name - validate source on letter or digit value. - PROMPP_ALWAYS_INLINE bool is_valid_name(re2::RE2& rgx_validate, std::string_view src) { return RE2::FullMatch(src, rgx_validate); } + PROMPP_ALWAYS_INLINE static bool is_valid_name(const re2::RE2& rgx_validate, std::string_view src) { return RE2::FullMatch(src, rgx_validate); } // parse - parse template on parts. - PROMPP_ALWAYS_INLINE void parse(Regexp& regexp, re2::RE2& rgx_validate, std::string_view tmpl, std::vector& src_parts) { + PROMPP_ALWAYS_INLINE static void parse(const Regexp& regexp, const re2::RE2& rgx_validate, std::string_view tmpl, std::vector& src_parts) { std::map groups = regexp.groups(); - std::string_view p = std::string_view(tmpl); + auto p = std::string_view(tmpl); while (true) { - if (p.size() == 0) { + if (p.empty()) { break; } // search '$' and cut before - size_t i = p.find('$'); - std::string_view substr_p = p.substr(0, i); - if (substr_p.size() != 0) { + const size_t i = p.find('$'); + if (std::string_view substr_p = p.substr(0, i); !substr_p.empty()) { src_parts.emplace_back(substr_p); } if (i == std::string_view::npos) { @@ -298,7 +291,7 @@ class RelabelConfig { // if contains '{...}' case '{': { p.remove_prefix(1); - size_t j = p.find('}'); + const size_t j = p.find('}'); if (j == std::string_view::npos) { // if '}' not found cut - "${" src_parts.emplace_back(tmpl.substr(tmpl.size() - p.size() - 2, 2)); @@ -306,16 +299,14 @@ class RelabelConfig { } std::string_view g_name = p.substr(0, j); - auto rec = groups.find(std::string(g_name)); - if (rec != groups.end()) { + if (auto rec = groups.find(std::string(g_name)); rec != groups.end()) { // if g_name found in map add as group(int) src_parts.emplace_back(rec->second); p.remove_prefix(g_name.size() + 1); continue; } - bool ok = is_valid_name(rgx_validate, g_name); - if (!ok) { + if (!is_valid_name(rgx_validate, g_name)) { // if g_name invalid add as is - "${" + std::string{g_name} + "}" src_parts.emplace_back(tmpl.substr(tmpl.size() - p.size() - 2, g_name.size() + 3)); p.remove_prefix(g_name.size() + 1); @@ -330,10 +321,10 @@ class RelabelConfig { default: { // search '$' and extract g_name - int j = p.find('$'); + const auto j = p.find('$'); std::string_view g_name = p.substr(0, j); std::string name = extract(rgx_validate, g_name); - if (name.size() == 0) { + if (name.empty()) { // if name invalid add as is - "$" src_parts.emplace_back(tmpl.substr(tmpl.size() - p.size() - 1, 1)); continue; @@ -343,7 +334,7 @@ class RelabelConfig { if (rec != groups.end()) { // if g_name found in map add as group(int) src_parts.emplace_back(rec->second); - if (substr_g_name.size() != 0) { + if (!substr_g_name.empty()) { src_parts.emplace_back(substr_g_name); } p.remove_prefix(g_name.size()); @@ -351,7 +342,7 @@ class RelabelConfig { } // if g_name not found in map cut g_name - if (substr_g_name.size() != 0) { + if (!substr_g_name.empty()) { src_parts.emplace_back(substr_g_name); } p.remove_prefix(g_name.size()); @@ -372,12 +363,28 @@ class RelabelConfig { return BareBones::Bit::be(*reinterpret_cast(&digest[shift])); } + template + PROMPP_ALWAYS_INLINE std::string get_value(LabelsBuilder& builder) const { + std::string value; + + for (size_t i = 0; i < source_labels_.size(); ++i) { + const auto lv = builder.get(source_labels_[i]); + if (i == 0) [[unlikely]] { + value += lv; + continue; + } + value += separator_; + value += lv; + } + + return value; + } + public: // RelabelConfig - constructor for RelabelConfig from go-config. template PROMPP_ALWAYS_INLINE explicit RelabelConfig(GORelabelConfig* go_rc) noexcept - : source_labels_{}, - separator_{static_cast(go_rc->separator)}, + : separator_{static_cast(go_rc->separator)}, regexp_(static_cast(go_rc->regex)), modulus_{go_rc->modulus}, target_label_{static_cast(go_rc->target_label)}, @@ -393,164 +400,168 @@ class RelabelConfig { parse(regexp_, rgx_validate, replacement_, replacement_parts_); } - PROMPP_ALWAYS_INLINE RelabelConfig(RelabelConfig&&) noexcept = default; - // source_labels - a list of labels from which values are taken and concatenated with the configured separator in order. - PROMPP_ALWAYS_INLINE const std::vector& source_labels() const noexcept { return source_labels_; } + [[nodiscard]] PROMPP_ALWAYS_INLINE const std::vector& source_labels() const noexcept { return source_labels_; } // separator - is the string between concatenated values from the source labels. - PROMPP_ALWAYS_INLINE const std::string_view& separator() const noexcept { return separator_; } + [[nodiscard]] PROMPP_ALWAYS_INLINE const std::string_view& separator() const noexcept { return separator_; } // regexp - against which the concatenation is matched. - PROMPP_ALWAYS_INLINE const Regexp& regexp() const noexcept { return regexp_; } + [[nodiscard]] PROMPP_ALWAYS_INLINE const Regexp& regexp() const noexcept { return regexp_; } // modulus - to take of the hash of concatenated values from the source labels. - PROMPP_ALWAYS_INLINE const uint64_t& modulus() const noexcept { return modulus_; } + [[nodiscard]] PROMPP_ALWAYS_INLINE uint64_t modulus() const noexcept { return modulus_; } // target_label - is the label to which the resulting string is written in a replacement. // Regexp interpolation is allowed for the replace action. - PROMPP_ALWAYS_INLINE const std::string_view& target_label() const noexcept { return target_label_; } + [[nodiscard]] PROMPP_ALWAYS_INLINE const std::string_view& target_label() const noexcept { return target_label_; } // replacement - is the regex replacement pattern to be used. - PROMPP_ALWAYS_INLINE const std::string_view& replacement() const noexcept { return replacement_; } + [[nodiscard]] PROMPP_ALWAYS_INLINE const std::string_view& replacement() const noexcept { return replacement_; } // action - is the action to be performed for the relabeling. - PROMPP_ALWAYS_INLINE const rAction& action() const noexcept { return action_; } + [[nodiscard]] PROMPP_ALWAYS_INLINE rAction action() const noexcept { return action_; } // target_label_parts - dismantled target_label. - PROMPP_ALWAYS_INLINE const std::vector& target_label_parts() const noexcept { return target_label_parts_; } + [[nodiscard]] PROMPP_ALWAYS_INLINE const std::vector& target_label_parts() const noexcept { return target_label_parts_; } // replacement_parts - dismantled replacement. - PROMPP_ALWAYS_INLINE const std::vector& replacement_parts() const noexcept { return replacement_parts_; } + [[nodiscard]] PROMPP_ALWAYS_INLINE const std::vector& replacement_parts() const noexcept { return replacement_parts_; } // relabel - building relabeling labels. template - PROMPP_ALWAYS_INLINE relabelStatus relabel(std::stringstream& buf, LabelsBuilder& builder) { - std::string value; - for (size_t i = 0; i < source_labels_.size(); ++i) { - std::string_view lv = builder.get(source_labels_[i]); - if (i == 0) { - value += std::string(lv); - continue; - } - value += separator_; - value += lv; - } - + PROMPP_ALWAYS_INLINE relabelStatus relabel(std::string& buf, LabelsBuilder& builder) const { switch (action_) { case rDrop: { - if (regexp_.full_match(value)) { + if (regexp_.full_match(get_value(builder))) { return rsDrop; } break; } case rKeep: { - if (!regexp_.full_match(value)) { + if (!regexp_.full_match(get_value(builder))) { return rsDrop; } break; } case rDropEqual: { - if (builder.get(target_label_) == value) { + if (builder.get(target_label_) == get_value(builder)) { return rsDrop; } break; } case rKeepEqual: { - if (builder.get(target_label_) != value) { + if (builder.get(target_label_) != get_value(builder)) { return rsDrop; } break; } case rReplace: { - std::vector res_args; - bool ok = regexp_.match_to_args(value, res_args); - if (!ok) { + const auto value = get_value(builder); + std::vector res_args; + if (!regexp_.match_to_args(value, res_args)) { break; } - std::string lname = regexp_.replace_with_args(buf, res_args, target_label_parts_); - if (!label_name_is_valid(lname)) { + Regexp::replace_with_args(buf, res_args, target_label_parts_); + if (!label_name_is_valid(buf)) { break; } - std::string lvalue = regexp_.replace_with_args(buf, res_args, replacement_parts_); - if (lvalue.size() == 0) { + std::string lname = buf; + + Regexp::replace_with_args(buf, res_args, replacement_parts_); + if (buf.empty()) { if (builder.contains(lname)) { builder.del(lname); return rsRelabel; } break; } - builder.set(lname, lvalue); + builder.set(lname, buf); return rsRelabel; } case rLowercase: { - std::string lvalue{value}; - std::ranges::transform(lvalue, lvalue.begin(), [](unsigned char c) { return std::tolower(c); }); - builder.set(target_label_, lvalue); + auto value = get_value(builder); + std::ranges::transform(value, value.begin(), [](unsigned char c) { return std::tolower(c); }); + builder.set(target_label_, value); return rsRelabel; } case rUppercase: { - std::string lvalue{value}; - std::ranges::transform(lvalue, lvalue.begin(), [](unsigned char c) { return std::toupper(c); }); - builder.set(target_label_, lvalue); + auto value = get_value(builder); + std::ranges::transform(value, value.begin(), [](unsigned char c) { return std::toupper(c); }); + builder.set(target_label_, value); return rsRelabel; } case rHashMod: { - std::string lvalue{std::to_string(make_hash_uint64(value) % modulus_)}; + std::string lvalue{std::to_string(make_hash_uint64(get_value(builder)) % modulus_)}; builder.set(target_label_, lvalue); return rsRelabel; } case rLabelMap: { - bool change{false}; - builder.range([&](LNameType& lname, LValueType& lvalue) PROMPP_LAMBDA_INLINE -> bool { + std::vector labels_for_set; + builder.range([&](const auto& lname, const auto& lvalue) PROMPP_LAMBDA_INLINE -> bool { if (regexp_.full_match(lname)) { - std::string rlname = regexp_.replace_full(buf, lname, replacement_parts_); - builder.set(rlname, lvalue); - change = true; + regexp_.replace_full(buf, lname, replacement_parts_); + labels_for_set.emplace_back(buf, lvalue); } return true; }); - if (change) { + + if (!labels_for_set.empty()) { + for (const auto& label : labels_for_set) { + builder.set(label.first, label.second); + } + return rsRelabel; } + break; } case rLabelDrop: { - bool change{false}; - builder.range([&](LNameType& lname, [[maybe_unused]] LValueType& lvalue) PROMPP_LAMBDA_INLINE -> bool { + std::vector labels_for_del; + builder.range([&](const auto& lname, const auto&) PROMPP_LAMBDA_INLINE -> bool { if (regexp_.full_match(lname)) { - builder.del(lname); - change = true; + labels_for_del.emplace_back(lname); } return true; }); - if (change) { + + if (!labels_for_del.empty()) { + for (const auto& name : labels_for_del) { + builder.del(name); + } + return rsRelabel; } + break; } case rLabelKeep: { - bool change{false}; - builder.range([&](LNameType& lname, [[maybe_unused]] LValueType& lvalue) PROMPP_LAMBDA_INLINE -> bool { + std::vector labels_for_del; + builder.range([&](const auto& lname, const auto&) PROMPP_LAMBDA_INLINE -> bool { if (!regexp_.full_match(lname)) { - builder.del(lname); - change = true; + labels_for_del.emplace_back(lname); } return true; }); - if (change) { + + if (!labels_for_del.empty()) { + for (const auto& name : labels_for_del) { + builder.del(name); + } + return rsRelabel; } + break; } @@ -561,9 +572,6 @@ class RelabelConfig { return rsKeep; } - - // ~RelabelConfig - destructor for RelabelConfig from go-config. - PROMPP_ALWAYS_INLINE ~RelabelConfig() = default; }; // StatelessRelabeler - stateless relabeler with relabel configs. @@ -576,18 +584,15 @@ class StatelessRelabeler { // StatelessRelabeler - constructor for StatelessRelabeler, converting go-config. template PROMPP_ALWAYS_INLINE explicit StatelessRelabeler(const GORelabelConfigs& go_rcfgs) noexcept { - configs_.reserve(go_rcfgs.size()); - for (const auto go_rcfg : go_rcfgs) { - configs_.emplace_back(go_rcfg); - } + reset_to(go_rcfgs); } // relabeling_process caller passes a LabelsBuilder containing the initial set of labels, which is mutated by the rules. template - PROMPP_ALWAYS_INLINE relabelStatus relabeling_process(std::stringstream& buf, LabelsBuilder& builder) { + PROMPP_ALWAYS_INLINE relabelStatus relabeling_process(std::string& buf, LabelsBuilder& builder) const { relabelStatus rstatus{rsKeep}; for (auto& rcfg : configs_) { - relabelStatus status = rcfg.relabel(buf, builder); + const relabelStatus status = rcfg.relabel(buf, builder); if (status == rsDrop) { return rsDrop; } @@ -602,9 +607,8 @@ class StatelessRelabeler { // relabeling_process_with_soft_validate caller passes a LabelsBuilder containing the initial set of labels, which is mutated by the rules with soft(on empty) // validate label set. template - PROMPP_ALWAYS_INLINE relabelStatus relabeling_process_with_soft_validate(std::stringstream& buf, LabelsBuilder& builder) { - relabelStatus rstatus = relabeling_process(buf, builder); - + PROMPP_ALWAYS_INLINE relabelStatus relabeling_process_with_soft_validate(std::ostringstream& buf, LabelsBuilder& builder) { + const relabelStatus rstatus = relabeling_process(buf, builder); if (rstatus == rsDrop) { return rsDrop; } @@ -625,23 +629,21 @@ class StatelessRelabeler { configs_.emplace_back(go_rcfg); } } - - PROMPP_ALWAYS_INLINE ~StatelessRelabeler() = default; }; // processExternalLabels merges externalLabels into ls. If ls contains // a label in externalLabels, the value in ls wins. template -PROMPP_ALWAYS_INLINE void process_external_labels(LabelsBuilder& builder, ExternalLabels& external_labels) { +PROMPP_ALWAYS_INLINE void process_external_labels(LabelsBuilder& builder, const ExternalLabels& external_labels) { if (external_labels.size() == 0) { return; } std::size_t j{0}; - builder.range([&](LNameType& lname, [[maybe_unused]] LValueType& lvalue) PROMPP_LAMBDA_INLINE -> bool { - for (; j < external_labels.size() && lname > external_labels[j].first;) { - builder.set(external_labels[j].first, external_labels[j].second); - ++j; + std::vector indexes_for_set; + builder.range([&](const auto& lname, const auto&) PROMPP_LAMBDA_INLINE -> bool { + for (; j < external_labels.size() && lname > external_labels[j].first; ++j) { + indexes_for_set.emplace_back(j); } if (j < external_labels.size() && lname == external_labels[j].first) { @@ -650,6 +652,10 @@ PROMPP_ALWAYS_INLINE void process_external_labels(LabelsBuilder& builder, Extern return true; }); + for (auto index : indexes_for_set) { + builder.set(external_labels[index].first, external_labels[index].second); + } + for (; j < external_labels.size(); j++) { builder.set(external_labels[j].first, external_labels[j].second); } @@ -664,7 +670,6 @@ PROMPP_ALWAYS_INLINE void soft_validate(relabelStatus& rstatus, LabelsBuilder& b if (builder.is_empty()) [[unlikely]] { rstatus = rsDrop; - return; } }; diff --git a/pp/prometheus/tests/relabeler_tests.cpp b/pp/prometheus/tests/relabeler_tests.cpp index d34a1b4ddb..591fd4dff3 100644 --- a/pp/prometheus/tests/relabeler_tests.cpp +++ b/pp/prometheus/tests/relabeler_tests.cpp @@ -1,7 +1,3 @@ -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wunknown-pragmas" -#pragma GCC optimize("no-var-tracking") // to speed up compilation - #include #include #include @@ -15,326 +11,36 @@ namespace { -using namespace PromPP::Prometheus; // NOLINT -using namespace PromPP::Primitives; // NOLINT - -using LabelViewForTest = std::pair; -using LabelForTest = std::pair; - -class SampleForTest { - private: - int64_t timestamp_; - double value_; - - public: - PROMPP_ALWAYS_INLINE SampleForTest(int64_t timestamp, double value) noexcept : timestamp_(timestamp), value_(value) {} - - PROMPP_ALWAYS_INLINE SampleForTest() noexcept = default; - - int64_t timestamp() const noexcept { return timestamp_; } - int64_t& timestamp() noexcept { return timestamp_; } - - double value() const noexcept { return value_; } - double& value() noexcept { return value_; } -}; - -class NamesSetForTest : public std::vector { - using Base = std::vector; - - public: - using Base::Base; - [[maybe_unused]] friend size_t hash_value(const NamesSetForTest& lns) { return PromPP::Primitives::hash::hash_of_string_list(lns); } -}; - -class LabelViewSetForTest : public std::vector { - using Base = std::vector; - - public: - using Base::Base; - - PROMPP_ALWAYS_INLINE void add(const LabelViewForTest& label) noexcept { - if (__builtin_expect(Base::empty() || std::get<0>(label) > std::get<0>(Base::back()), true)) { - Base::emplace_back(label); - } else if (__builtin_expect(std::get<0>(label) == std::get<0>(Base::back()), false)) { - std::get<1>(Base::back()) = std::get<1>(label); - } else { - auto i = std::lower_bound(Base::begin(), Base::end(), std::get<0>(label), - [](const LabelViewForTest& a, const std::string_view& b) { return std::get<0>(a) < b; }); - if (__builtin_expect(std::get<0>(*i) == std::get<0>(label), false)) { - std::get<1>(*i) = std::get<1>(label); - } else { - Base::insert(i, label); - } - } - } - - template - PROMPP_ALWAYS_INLINE SymbolType get(const SymbolType& label_name) noexcept { - for (const auto& [ln, lv] : *this) { - if (ln == label_name) [[unlikely]] { - return lv; - } - } - - return ""; - } - - NamesSetForTest names() const { - NamesSetForTest tns; - - for (auto [label_name, _] : *this) { - tns.push_back(label_name); - } - - return tns; - } - - [[maybe_unused]] friend size_t hash_value(const LabelViewSetForTest& tls) { return PromPP::Primitives::hash::hash_of_label_set(tls); } -}; - -class TimeseriesForTest { - LabelViewSetForTest label_set_; - std::vector samples_; - - public: - TimeseriesForTest() noexcept = default; - TimeseriesForTest(const TimeseriesForTest&) noexcept = default; - TimeseriesForTest& operator=(const TimeseriesForTest&) noexcept = default; - TimeseriesForTest(TimeseriesForTest&&) noexcept = default; - TimeseriesForTest& operator=(TimeseriesForTest&&) noexcept = default; - - TimeseriesForTest(const LabelViewSetForTest& label_set, const std::vector& samples) noexcept : label_set_(label_set), samples_(samples) {} - - PROMPP_ALWAYS_INLINE auto& label_set() noexcept { return label_set_; } - - PROMPP_ALWAYS_INLINE const auto& label_set() const noexcept { return label_set_; } - - PROMPP_ALWAYS_INLINE const auto& samples() const noexcept { return samples_; } - - PROMPP_ALWAYS_INLINE auto& samples() noexcept { return samples_; } - - PROMPP_ALWAYS_INLINE void clear() noexcept { - label_set().clear(); - samples().clear(); - } -}; - -// LabelsBuilderForTest - builder for label set. -template -class LabelsBuilderForTest { - LabelSet* base_ = nullptr; - LabelViewSetForTest buf_; - std::vector add_; - std::vector del_; - - public: - PROMPP_ALWAYS_INLINE LabelsBuilderForTest() noexcept {} - - // del - add label name to remove from label set. - PROMPP_ALWAYS_INLINE void del(std::string& lname) { - std::erase_if(add_, [lname](LabelForTest& lv) { return lv.first == lname; }); - - if (auto i = std::ranges::find_if(del_.begin(), del_.end(), [lname](const std::string_view& ln) { return ln == lname; }); i != del_.end()) { - return; - } - - del_.emplace_back(lname); - } - - // del - add label name to remove from label set. - PROMPP_ALWAYS_INLINE void del(std::string_view lname) { - std::erase_if(add_, [lname](LabelForTest& lv) { return lv.first == lname; }); - - if (auto i = std::ranges::find_if(del_.begin(), del_.end(), [lname](const std::string_view& ln) { return ln == lname; }); i != del_.end()) { - return; - } - - del_.emplace_back(lname); - } - - // get - returns the value for the label with the given name. Returns an empty string if the label doesn't exist. - PROMPP_ALWAYS_INLINE std::string_view get(std::string_view lname) { - if (auto i = std::ranges::find_if(add_.begin(), add_.end(), [lname](const LabelForTest& l) { return l.first == lname; }); i != add_.end()) { - return (*i).second; - } - - if (auto i = std::ranges::find_if(del_.begin(), del_.end(), [lname](const std::string_view& ln) { return ln == lname; }); i != del_.end()) { - return ""; - } - - if (base_ != nullptr) [[likely]] { - for (const auto& [ln, lv] : *base_) { - if (ln == lname) { - return lv; - } - } - } - - return ""; - } - - PROMPP_ALWAYS_INLINE bool contains(std::string_view lname) { - if (auto i = std::ranges::find_if(add_, [&lname](const LabelForTest& l) { return l.first == lname; }); i != add_.end()) { - return true; - } - - if (base_ != nullptr) [[likely]] { - for (const auto& [ln, lv] : *base_) { - if (ln == lname) { - return true; - } - } - } - - return false; - } - - // returns size of building labels. - PROMPP_ALWAYS_INLINE size_t size() { - size_t count{0}; - if (base_ != nullptr) [[likely]] { - for (const auto& ls : *base_) { - if (auto i = std::ranges::find_if(add_.begin(), add_.end(), [ls](const LabelForTest& l) { return l.first == ls.first; }); i != add_.end()) { - continue; - } - - if (auto i = std::ranges::find_if(del_.begin(), del_.end(), [ls](const std::string_view& ln) { return ln == ls.first; }); i != del_.end()) { - continue; - } - - ++count; - } - } - - count += add_.size(); - return count; - } - - // returns true if ls represents an empty set of labels. - PROMPP_ALWAYS_INLINE bool is_empty() { return size() == 0; } - - // labels - returns the labels from the builder. If no modifications were made, the original labels are returned. - PROMPP_ALWAYS_INLINE LabelViewSetForTest labels() { - if (base_ != nullptr) [[likely]] { - for (const auto& ls : *base_) { - if (auto i = std::ranges::find_if(add_.begin(), add_.end(), [ls](const LabelForTest& l) { return l.first == ls.first; }); i != add_.end()) { - continue; - } - - if (auto i = std::ranges::find_if(del_.begin(), del_.end(), [ls](const std::string_view& ln) { return ln == ls.first; }); i != del_.end()) { - continue; - } - - buf_.add(ls); - } - } - - if (add_.size() != 0) { - std::ranges::for_each(add_.begin(), add_.end(), [&](const LabelForTest& l) { buf_.add(l); }); - std::ranges::sort(buf_.begin(), buf_.end(), [](const LabelViewForTest& a, const LabelViewForTest& b) { - if (a.first == b.first) { - return a.second < b.second; - } - return a.first < b.first; - }); - } - - return buf_; - } - - // range - calls f on each label in the builder. - template - PROMPP_ALWAYS_INLINE void range(Callback func) { - // take a copy of add and del, so they are unaffected by calls to set() or del(). - std::vector cadd; - cadd.reserve(add_.size()); - std::ranges::copy(add_.begin(), add_.end(), std::back_inserter(cadd)); - - std::vector cdel; - cdel.reserve(del_.size()); - std::ranges::copy(cdel.begin(), cdel.end(), std::back_inserter(cdel)); - - if (__builtin_expect(base_ != nullptr, true)) { - for (const auto& ls : *base_) { - if (auto i = std::ranges::find_if(cadd.begin(), cadd.end(), [ls](const LabelViewForTest& l) { return l.first == ls.first; }); i != cadd.end()) { - continue; - } - - if (auto i = std::ranges::find_if(cdel.begin(), cdel.end(), [ls](const std::string_view& ln) { return ln == ls.first; }); i != cdel.end()) { - continue; - } - - func(ls.first, ls.second); - } - } - - std::ranges::for_each(cadd.begin(), cadd.end(), [&](const LabelViewForTest& l) { func(l.first, l.second); }); - } - - // reset - clears all current state for the builder. - PROMPP_ALWAYS_INLINE void reset() { - buf_.clear(); - add_.clear(); - del_.clear(); - base_ = nullptr; - } - - // reset - clears all current state for the builder and init from LabelSet. - PROMPP_ALWAYS_INLINE void reset(LabelSet& ls) { - reset(); - base_ = &ls; - } - - // set - the name/value pair as a label. A value of "" means delete that label. - PROMPP_ALWAYS_INLINE void set(std::string& lname, std::string& lvalue) { - if (__builtin_expect(lvalue.size() == 0, false)) { - del(lname); - return; - } - - if (auto i = std::ranges::find_if(add_.begin(), add_.end(), [lname](const LabelForTest& l) { return l.first == lname; }); i != add_.end()) { - (*i).second = lvalue; - return; - } - - add_.emplace_back(lname, lvalue); - } - - // set - the name/value pair as a label. A value of "" means delete that label. - PROMPP_ALWAYS_INLINE void set(std::string_view lname, std::string& lvalue) { - if (__builtin_expect(lvalue.size() == 0, false)) { - del(lname); - return; - } - - if (auto i = std::ranges::find_if(add_.begin(), add_.end(), [lname](const LabelForTest& l) { return l.first == lname; }); i != add_.end()) { - (*i).second = lvalue; - return; - } - - add_.emplace_back(lname, lvalue); - } - - // set - the name/value pair as a label. A value of "" means delete that label. - PROMPP_ALWAYS_INLINE void set(std::string& lname, std::string_view lvalue) { - if (__builtin_expect(lvalue.size() == 0, false)) { - del(lname); - return; - } - - if (auto i = std::ranges::find_if(add_.begin(), add_.end(), [lname](const LabelForTest& l) { return l.first == lname; }); i != add_.end()) { - (*i).second = lvalue; - return; - } - - add_.emplace_back(lname, lvalue); - } - - PROMPP_ALWAYS_INLINE LabelsBuilderForTest(LabelsBuilderForTest&&) noexcept = default; - PROMPP_ALWAYS_INLINE ~LabelsBuilderForTest() = default; -}; - -struct RelabelConfigTest { +using PromPP::Primitives::Label; +using PromPP::Primitives::LabelsBuilder; +using PromPP::Primitives::LabelSet; +using PromPP::Primitives::LabelView; +using PromPP::Primitives::LabelViewSet; +using PromPP::Primitives::Sample; +using PromPP::Primitives::Timestamp; +using PromPP::Primitives::Go::SliceView; +using PromPP::Primitives::SnugComposites::LabelSet::EncodingBimap; +using PromPP::Primitives::SnugComposites::LabelSet::OrderedEncodingBimap; +using PromPP::Prometheus::Relabel::hard_validate; +using PromPP::Prometheus::Relabel::InnerSerie; +using PromPP::Prometheus::Relabel::InnerSeries; +using PromPP::Prometheus::Relabel::MetricLimits; +using PromPP::Prometheus::Relabel::PerGoroutineRelabeler; +using PromPP::Prometheus::Relabel::PerShardRelabeler; +using PromPP::Prometheus::Relabel::RelabelerStateUpdate; +using PromPP::Prometheus::Relabel::relabelStatus; +using PromPP::Prometheus::Relabel::StaleNaNsState; +using PromPP::Prometheus::Relabel::StatelessRelabeler; +using enum PromPP::Prometheus::Relabel::rAction; +using enum relabelStatus; + +using GoString = PromPP::Primitives::Go::String; +using PromPP::Primitives::kNullTimestamp; +using PromPP::Prometheus::kStaleNan; + +using GoLabel = std::pair; + +struct RelabelConfig { std::vector source_labels{}; std::string_view separator{}; std::string_view regex{}; @@ -344,43 +50,25 @@ struct RelabelConfigTest { uint8_t action{0}; }; -PROMPP_ALWAYS_INLINE LabelViewSetForTest make_label_set(std::initializer_list lvs) { - LabelViewSetForTest labels; - for (const LabelViewForTest& lv : lvs) { - labels.add(lv); - } - - return labels; -} - -PROMPP_ALWAYS_INLINE std::vector make_samples(std::initializer_list samples) { - std::vector sampleses; - for (const SampleForTest& s : samples) { - sampleses.push_back(s); - } - - return sampleses; -} - -struct ItemTest { - size_t hash_; - LabelViewSetForTest labelview_set_; - std::vector samples_; +class ItemTest { + public: + ItemTest(LabelViewSet&& label_set, std::vector&& samples) + : label_set_(std::move(label_set)), samples_(std::move(samples)), hash_(hash_value(label_set_)) {} - PROMPP_ALWAYS_INLINE explicit ItemTest(size_t hash, LabelViewSetForTest& labelview_set, std::vector& samples) - : hash_(hash), labelview_set_(labelview_set), samples_(samples) {} - PROMPP_ALWAYS_INLINE size_t hash() const { return hash_; } + [[nodiscard]] PROMPP_ALWAYS_INLINE size_t hash() const { return hash_; } template PROMPP_ALWAYS_INLINE void read(Timeseries& timeseries) const { - for (const auto& labelview : labelview_set_) { - timeseries.label_set().add({labelview.first, labelview.second}); - } - + timeseries.label_set().add(label_set_); for (const auto& sample : samples_) { timeseries.samples().emplace_back(sample.timestamp(), sample.value()); } } + + private: + LabelViewSet label_set_; + std::vector samples_; + size_t hash_; }; class HashdexTest : public std::vector { @@ -389,6 +77,8 @@ class HashdexTest : public std::vector { public: using Base::Base; + void emplace_back(LabelViewSet&& label_set, std::vector&& samples) { Base::emplace_back(std::move(label_set), std::move(samples)); } + [[nodiscard]] PROMPP_ALWAYS_INLINE const auto& metrics() const noexcept { return *this; } [[nodiscard]] static PROMPP_ALWAYS_INLINE auto metadata() noexcept { struct Stub {}; @@ -398,109 +88,76 @@ class HashdexTest : public std::vector { static_assert(PromPP::Prometheus::hashdex::HashdexInterface); -PROMPP_ALWAYS_INLINE void make_hashdex(HashdexTest& hx, LabelViewSetForTest label_set, std::vector samples) { - hx.emplace_back(hash_value(label_set), label_set, samples); -} - -// -// TestValidate -// - -struct TestValidate : public testing::Test { - PromPP::Primitives::LabelsBuilderStateMap builder_state_; +struct HardValidateCase { + LabelViewSet labels; + std::optional limits{std::nullopt}; + relabelStatus expected; }; -TEST_F(TestValidate, HardValid) { - LabelViewSetForTest incoming_labels = make_label_set({{"__name__", "value"}, {"job", "abc"}}); - PromPP::Primitives::LabelsBuilder builder{builder_state_}; - builder.reset(incoming_labels); - - Relabel::relabelStatus rstatus{Relabel::rsKeep}; - Relabel::hard_validate(rstatus, builder, nullptr); - EXPECT_EQ(Relabel::rsKeep, rstatus); -} - -TEST_F(TestValidate, HardInvalid) { - LabelViewSetForTest incoming_labels = make_label_set({{"__value__", "value"}, {"job", "abc"}}); - PromPP::Primitives::LabelsBuilder builder{builder_state_}; - builder.reset(incoming_labels); - - Relabel::relabelStatus rstatus{Relabel::rsKeep}; - Relabel::hard_validate(rstatus, builder, nullptr); - EXPECT_EQ(Relabel::rsInvalid, rstatus); -} - -TEST_F(TestValidate, InvalidLabelLimit) { - LabelViewSetForTest incoming_labels = make_label_set({{"__name__", "value"}, {"job", "abc"}, {"jub", "buj"}}); - PromPP::Primitives::LabelsBuilder builder{builder_state_}; - builder.reset(incoming_labels); - - // label_limit 0 - Relabel::MetricLimits ll{}; - Relabel::relabelStatus rstatus{Relabel::rsKeep}; - Relabel::hard_validate(rstatus, builder, &ll); - EXPECT_EQ(Relabel::rsKeep, rstatus); - - // label_limit 2 - ll.label_limit = 2; - Relabel::hard_validate(rstatus, builder, &ll); - EXPECT_EQ(Relabel::rsInvalid, rstatus); - - // label_name_length_limit 3 - rstatus = Relabel::rsKeep; - ll.label_limit = 3; - ll.label_name_length_limit = 3; - Relabel::hard_validate(rstatus, builder, &ll); - EXPECT_EQ(Relabel::rsInvalid, rstatus); - - // label_value_length_limit 3 - rstatus = Relabel::rsKeep; - ll.label_limit = 3; - ll.label_name_length_limit = 10; - ll.label_value_length_limit = 3; - Relabel::hard_validate(rstatus, builder, &ll); - EXPECT_EQ(Relabel::rsInvalid, rstatus); -} +class HardValidateFixture : public testing::TestWithParam { + protected: + LabelsBuilder builder_; + relabelStatus rstatus_{rsKeep}; +}; -// -// PerShardRelabeler -// +TEST_P(HardValidateFixture, Test) { + // Arrange + builder_.reset(GetParam().labels); + + // Act + hard_validate(rstatus_, builder_, GetParam().limits ? &GetParam().limits.value() : nullptr); + + // Assert + EXPECT_EQ(GetParam().expected, rstatus_); +} + +INSTANTIATE_TEST_SUITE_P(Valid, HardValidateFixture, testing::Values(HardValidateCase{.labels = {{"__name__", "value"}, {"job", "abc"}}, .expected = rsKeep})); +INSTANTIATE_TEST_SUITE_P(Invalid, + HardValidateFixture, + testing::Values(HardValidateCase{.labels = {{"__value__", "value"}, {"job", "abc"}}, .expected = rsInvalid})); +INSTANTIATE_TEST_SUITE_P( + NoLimit, + HardValidateFixture, + testing::Values(HardValidateCase{.labels = {{"__name__", "value"}, {"job", "abc"}, {"jub", "buj"}}, .limits = MetricLimits{}, .expected = rsKeep})); +INSTANTIATE_TEST_SUITE_P(LabelCountLimitExceeded, + HardValidateFixture, + testing::Values(HardValidateCase{.labels = {{"__name__", "value"}, {"job", "abc"}, {"jub", "buj"}}, + .limits = MetricLimits{.label_limit = 2}, + .expected = rsInvalid})); +INSTANTIATE_TEST_SUITE_P(LabelNameLengthLimitExceeded, + HardValidateFixture, + testing::Values(HardValidateCase{.labels = {{"__name__", "value"}, {"job", "abc"}, {"jub", "buj"}}, + .limits = MetricLimits{.label_name_length_limit = 3}, + .expected = rsInvalid})); +INSTANTIATE_TEST_SUITE_P(LabelValueLengthLimitExceeded, + HardValidateFixture, + testing::Values(HardValidateCase{.labels = {{"__name__", "value"}, {"job", "abc"}, {"jub", "buj"}}, + .limits = MetricLimits{.label_value_length_limit = 3}, + .expected = rsInvalid})); struct Stats { uint32_t samples_added{0}; uint32_t series_added{0}; uint32_t series_drop{0}; -}; -struct TestPerShardRelabeler : public testing::Test { - // shards_inner_series - std::vector> vector_shards_inner_series_; - PromPP::Primitives::Go::SliceView shards_inner_series_; + bool operator==(const Stats& other) const noexcept = default; +}; - // relabeled_results - std::vector> vector_relabeled_results_; - PromPP::Primitives::Go::SliceView relabeled_results_; +class PerGoroutineRelabelerFixture : public testing::Test { + protected: + static constexpr uint16_t kNumberOfShards = 2; - // external_labels - std::vector> vector_external_labels_; - PromPP::Primitives::Go::SliceView> external_labels_; + std::vector> vector_shards_inner_series_; + SliceView shards_inner_series_{}; - // target_labels - std::vector> vector_target_labels_{}; + std::vector> vector_relabeled_results_; + SliceView relabeled_results_{}; - // Options + std::vector vector_target_labels_{}; PromPP::Prometheus::Relabel::RelabelerOptions o_; - - // Stats Stats stats_; - - // Hashdex HashdexTest hx_; - - // LSS - PromPP::Primitives::SnugComposites::LabelSet::EncodingBimap lss_; - - // Cache + EncodingBimap lss_; PromPP::Prometheus::Relabel::Cache cache_{}; void reset() { @@ -508,526 +165,432 @@ struct TestPerShardRelabeler : public testing::Test { SetUp(); } - void add_target_labels(std::vector>& list_target_labels) { - vector_target_labels_.resize(list_target_labels.size()); + void add_target_labels(const LabelViewSet& target_labels) { + vector_target_labels_.resize(target_labels.size()); for (size_t i = 0; i < vector_target_labels_.size(); i++) { - vector_target_labels_[i].first.reset_to(list_target_labels[i].first.data(), list_target_labels[i].first.size()); - vector_target_labels_[i].second.reset_to(list_target_labels[i].second.data(), list_target_labels[i].second.size()); + vector_target_labels_[i].first.reset_to(target_labels[i].first.data(), target_labels[i].first.size()); + vector_target_labels_[i].second.reset_to(target_labels[i].second.data(), target_labels[i].second.size()); } o_.target_labels.reset_to(vector_target_labels_.data(), vector_target_labels_.size(), vector_target_labels_.size()); } void SetUp() final { - // shards_inner_series - vector_shards_inner_series_.emplace_back(std::make_unique()); - vector_shards_inner_series_.emplace_back(std::make_unique()); - shards_inner_series_.reset_to(reinterpret_cast(vector_shards_inner_series_.data()), - vector_shards_inner_series_.size(), vector_shards_inner_series_.size()); + vector_shards_inner_series_.emplace_back(std::make_unique()); + vector_shards_inner_series_.emplace_back(std::make_unique()); + shards_inner_series_.reset_to(reinterpret_cast(vector_shards_inner_series_.data()), vector_shards_inner_series_.size(), + vector_shards_inner_series_.size()); - // relabeled_results vector_relabeled_results_.emplace_back(std::make_unique()); vector_relabeled_results_.emplace_back(std::make_unique()); relabeled_results_.reset_to(reinterpret_cast(vector_relabeled_results_.data()), vector_shards_inner_series_.size(), vector_shards_inner_series_.size()); - // external_labels - external_labels_.reset_to(vector_external_labels_.data(), vector_external_labels_.size(), vector_external_labels_.size()); - - // target_labels o_.target_labels.reset_to(vector_target_labels_.data(), vector_target_labels_.size(), vector_target_labels_.size()); } void TearDown() final { - // clear memory vector_shards_inner_series_.clear(); vector_relabeled_results_.clear(); } }; -TEST_F(TestPerShardRelabeler, KeepEQ) { - RelabelConfigTest rct{.source_labels = std::vector{"job"}, .regex = "abc", .action = 2}; // Keep - Relabel::StatelessRelabeler sr(std::vector{&rct}); - PromPP::Prometheus::Relabel::PerShardRelabeler prs(external_labels_, &sr, 2, 1); - make_hashdex(hx_, make_label_set({{"__name__", "value"}, {"job", "abc"}}), make_samples({{1712567046855, 0.1}})); +TEST_F(PerGoroutineRelabelerFixture, KeepOnNotFoundInCache) { + // Arrange + const RelabelConfig config{.source_labels = {{"job"}}, .regex = "abc", .action = rKeep}; + const StatelessRelabeler stateless_relabeler(std::initializer_list{&config}); + hx_.emplace_back({{"__name__", "value"}, {"job", "abc"}}, {{1000, 0.1}}); + PerGoroutineRelabeler relabeler{kNumberOfShards, 1}; - prs.input_relabeling(lss_, lss_, cache_, hx_, o_, stats_, shards_inner_series_, relabeled_results_); - // shard id 1 + // Act + relabeler.input_relabeling(lss_, lss_, cache_, hx_, o_, stateless_relabeler, stats_, shards_inner_series_, relabeled_results_); + + // Assert EXPECT_EQ(relabeled_results_[0]->size(), 0); EXPECT_EQ(relabeled_results_[1]->size(), 0); EXPECT_EQ(shards_inner_series_[0]->size(), 0); - EXPECT_EQ(shards_inner_series_[1]->size(), 1); - EXPECT_EQ(stats_.samples_added, 1); - EXPECT_EQ(stats_.series_added, 1); - - PromPP::Prometheus::Relabel::RelabelerStateUpdate update_data{}; - prs.append_relabeler_series(lss_, shards_inner_series_[1], relabeled_results_[1], &update_data); - EXPECT_EQ(shards_inner_series_[1]->size(), 1); - EXPECT_EQ(update_data.size(), 0); - - vector_shards_inner_series_[1] = std::make_unique(); - prs.input_relabeling(lss_, lss_, cache_, hx_, o_, stats_, shards_inner_series_, relabeled_results_); - EXPECT_EQ(shards_inner_series_[1]->size(), 1); - EXPECT_EQ(stats_.samples_added, 2); - EXPECT_EQ(stats_.series_added, 1); + EXPECT_TRUE(std::ranges::equal(std::vector{{.sample = Sample(1000, 0.1), .ls_id = 0}}, shards_inner_series_[1]->data())); + EXPECT_EQ((Stats{.samples_added = 1, .series_added = 1}), stats_); } -TEST_F(TestPerShardRelabeler, KeepEQ_OrderedEncodingBimap) { - PromPP::Primitives::SnugComposites::LabelSet::OrderedEncodingBimap lss; - RelabelConfigTest rct{.source_labels = std::vector{"job"}, .regex = "abc", .action = 2}; // Keep - Relabel::StatelessRelabeler sr(std::vector{&rct}); - PromPP::Prometheus::Relabel::PerShardRelabeler prs(external_labels_, &sr, 2, 1); - make_hashdex(hx_, make_label_set({{"__name__", "value"}, {"job", "abc"}}), make_samples({{1712567046855, 0.1}})); +TEST_F(PerGoroutineRelabelerFixture, InnerSeriesAlreadyAdded) { + // Arrange + const RelabelConfig config{.source_labels = {{"job"}}, .regex = "abc", .action = rKeep}; + const StatelessRelabeler stateless_relabeler(std::initializer_list{&config}); + hx_.emplace_back({{"__name__", "value"}, {"job", "abc"}}, {{1000, 0.1}}); + PerGoroutineRelabeler relabeler{kNumberOfShards, 1}; + + // Act + relabeler.input_relabeling(lss_, lss_, cache_, hx_, o_, stateless_relabeler, stats_, shards_inner_series_, relabeled_results_); + relabeler.input_relabeling(lss_, lss_, cache_, hx_, o_, stateless_relabeler, stats_, shards_inner_series_, relabeled_results_); - prs.input_relabeling(lss, lss, cache_, hx_, o_, stats_, shards_inner_series_, relabeled_results_); - // shard id 1 + // Assert EXPECT_EQ(relabeled_results_[0]->size(), 0); EXPECT_EQ(relabeled_results_[1]->size(), 0); EXPECT_EQ(shards_inner_series_[0]->size(), 0); - EXPECT_EQ(shards_inner_series_[1]->size(), 1); - EXPECT_EQ(stats_.samples_added, 1); - EXPECT_EQ(stats_.series_added, 1); - - PromPP::Prometheus::Relabel::RelabelerStateUpdate update_data{}; - prs.append_relabeler_series(lss_, shards_inner_series_[1], relabeled_results_[1], &update_data); - EXPECT_EQ(shards_inner_series_[1]->size(), 1); - EXPECT_EQ(update_data.size(), 0); - - vector_shards_inner_series_[1] = std::make_unique(); - prs.input_relabeling(lss, lss, cache_, hx_, o_, stats_, shards_inner_series_, relabeled_results_); - EXPECT_EQ(shards_inner_series_[1]->size(), 1); - EXPECT_EQ(stats_.samples_added, 2); - EXPECT_EQ(stats_.series_added, 1); + EXPECT_TRUE(std::ranges::equal(std::vector{{.sample = Sample(1000, 0.1), .ls_id = 0}}, shards_inner_series_[1]->data())); + EXPECT_EQ((Stats{.samples_added = 1, .series_added = 1}), stats_); } -TEST_F(TestPerShardRelabeler, KeepNE) { - RelabelConfigTest rct{.source_labels = std::vector{"job"}, .regex = "no-match", .action = 2}; // Keep - Relabel::StatelessRelabeler sr(std::vector{&rct}); - PromPP::Prometheus::Relabel::PerShardRelabeler prs(external_labels_, &sr, 2, 1); - make_hashdex(hx_, make_label_set({{"__name__", "value"}, {"job", "abc"}}), make_samples({{1712567046855, 0.1}})); +TEST_F(PerGoroutineRelabelerFixture, KeepOnFoundInCache) { + // Arrange + const RelabelConfig config{.source_labels = {{"job"}}, .regex = "abc", .action = rKeep}; + const StatelessRelabeler stateless_relabeler(std::initializer_list{&config}); + hx_.emplace_back({{"__name__", "value"}, {"job", "abc"}}, {{1000, 0.1}}); + PerGoroutineRelabeler relabeler{kNumberOfShards, 1}; + + // Act + relabeler.input_relabeling(lss_, lss_, cache_, hx_, o_, stateless_relabeler, stats_, shards_inner_series_, relabeled_results_); + shards_inner_series_[1]->clear(); + relabeler.input_relabeling(lss_, lss_, cache_, hx_, o_, stateless_relabeler, stats_, shards_inner_series_, relabeled_results_); - prs.input_relabeling(lss_, lss_, cache_, hx_, o_, stats_, shards_inner_series_, relabeled_results_); - // shard id 1 + // Assert EXPECT_EQ(relabeled_results_[0]->size(), 0); EXPECT_EQ(relabeled_results_[1]->size(), 0); EXPECT_EQ(shards_inner_series_[0]->size(), 0); - EXPECT_EQ(shards_inner_series_[1]->size(), 0); - EXPECT_EQ(stats_.samples_added, 0); - EXPECT_EQ(stats_.series_added, 0); + EXPECT_TRUE(std::ranges::equal(std::vector{{.sample = Sample(1000, 0.1), .ls_id = 0}}, shards_inner_series_[1]->data())); + EXPECT_EQ((Stats{.samples_added = 2, .series_added = 1}), stats_); } -TEST_F(TestPerShardRelabeler, KeepEQNE) { - RelabelConfigTest rct{.source_labels = std::vector{"job"}, .regex = "abc", .action = 2}; // Keep - Relabel::StatelessRelabeler sr(std::vector{&rct}); - PromPP::Prometheus::Relabel::PerShardRelabeler prs(external_labels_, &sr, 2, 1); - make_hashdex(hx_, make_label_set({{"__name__", "value"}, {"job", "abc"}}), make_samples({{1712567046855, 0.1}})); - - prs.input_relabeling(lss_, lss_, cache_, hx_, o_, stats_, shards_inner_series_, relabeled_results_); - // shard id 1 - EXPECT_EQ(relabeled_results_[0]->size(), 0); - EXPECT_EQ(relabeled_results_[1]->size(), 0); - EXPECT_EQ(shards_inner_series_[0]->size(), 0); - EXPECT_EQ(shards_inner_series_[1]->size(), 1); - EXPECT_EQ(stats_.samples_added, 1); - EXPECT_EQ(stats_.series_added, 1); +TEST_F(PerGoroutineRelabelerFixture, KeepNotEqual) { + // Arrange + const RelabelConfig config{.source_labels = {{"job"}}, .regex = "no-match", .action = rKeep}; + const StatelessRelabeler stateless_relabeler(std::initializer_list{&config}); + hx_.emplace_back({{"__name__", "value"}, {"job", "abc"}}, {{1000, 0.1}}); + PerGoroutineRelabeler relabeler{kNumberOfShards, 1}; - PromPP::Prometheus::Relabel::RelabelerStateUpdate update_data{}; - prs.append_relabeler_series(lss_, shards_inner_series_[1], relabeled_results_[1], &update_data); - EXPECT_EQ(shards_inner_series_[1]->size(), 1); - EXPECT_EQ(update_data.size(), 0); + // Act + relabeler.input_relabeling(lss_, lss_, cache_, hx_, o_, stateless_relabeler, stats_, shards_inner_series_, relabeled_results_); - reset(); - hx_.clear(); - make_hashdex(hx_, make_label_set({{"__name__", "value"}, {"job", "abcd"}}), make_samples({{1712567046855, 0.1}})); - prs.input_relabeling(lss_, lss_, cache_, hx_, o_, stats_, shards_inner_series_, relabeled_results_); - // shard id 1 skip + // Assert EXPECT_EQ(relabeled_results_[0]->size(), 0); EXPECT_EQ(relabeled_results_[1]->size(), 0); EXPECT_EQ(shards_inner_series_[0]->size(), 0); EXPECT_EQ(shards_inner_series_[1]->size(), 0); - EXPECT_EQ(stats_.samples_added, 1); - EXPECT_EQ(stats_.series_added, 1); + EXPECT_EQ(Stats{.series_drop = 1}, stats_); } -TEST_F(TestPerShardRelabeler, ReplaceToNewLS2) { - RelabelConfigTest rct{.source_labels = std::vector{"__name__"}, - .separator = ";", - .regex = ".*(o).*", - .target_label = "replaced", - .replacement = "$1", - .action = 5}; // Replace - Relabel::StatelessRelabeler sr(std::vector{&rct}); - PromPP::Prometheus::Relabel::PerShardRelabeler prs(external_labels_, &sr, 2, 1); - make_hashdex(hx_, make_label_set({{"__name__", "booom"}, {"jab", "baj"}, {"job", "baj"}}), make_samples({{1712567046855, 0.1}})); - - prs.input_relabeling(lss_, lss_, cache_, hx_, o_, stats_, shards_inner_series_, relabeled_results_); - // shard id 1 - EXPECT_EQ(relabeled_results_[0]->size(), 0); - EXPECT_EQ(relabeled_results_[1]->size(), 1); - EXPECT_EQ(shards_inner_series_[0]->size(), 0); - EXPECT_EQ(shards_inner_series_[1]->size(), 0); - EXPECT_EQ(stats_.samples_added, 1); - EXPECT_EQ(stats_.series_added, 1); - - PromPP::Prometheus::Relabel::RelabelerStateUpdate update_data{}; - prs.append_relabeler_series(lss_, shards_inner_series_[1], relabeled_results_[1], &update_data); - EXPECT_EQ(shards_inner_series_[1]->size(), 1); - EXPECT_EQ(update_data.size(), 1); - - prs.update_relabeler_state(cache_, &update_data, 1); +TEST_F(PerGoroutineRelabelerFixture, KeepEqualThenNotEqual) { + // Arrange + const RelabelConfig config{.source_labels = {{"job"}}, .regex = "abc", .action = rKeep}; + const StatelessRelabeler stateless_relabeler(std::initializer_list{&config}); + hx_.emplace_back({{"__name__", "value"}, {"job", "abc"}}, {{1000, 0.1}}); + PerGoroutineRelabeler relabeler{kNumberOfShards, 1}; - auto rlabels = lss_[update_data[0].relabeled_ls_id]; - LabelViewSetForTest expected_labels = make_label_set({{"__name__", "booom"}, {"jab", "baj"}, {"job", "baj"}, {"replaced", "o"}}); - - EXPECT_EQ(rlabels, expected_labels); -} - -TEST_F(TestPerShardRelabeler, ReplaceToNewLS3) { - RelabelConfigTest rct{.separator = ";", .regex = ".*", .target_label = "replaced", .replacement = "blabla", .action = 5}; // Replace - Relabel::StatelessRelabeler sr(std::vector{&rct}); - PromPP::Prometheus::Relabel::PerShardRelabeler prs(external_labels_, &sr, 2, 1); - make_hashdex(hx_, make_label_set({{"__name__", "booom"}, {"jab", "baj"}, {"job", "baj"}}), make_samples({{1712567046855, 0.1}})); + // Act + relabeler.input_relabeling(lss_, lss_, cache_, hx_, o_, stateless_relabeler, stats_, shards_inner_series_, relabeled_results_); + reset(); + hx_ = HashdexTest{{{{"__name__", "value"}, {"job", "abcd"}}, {{1000, 0.1}}}}; + relabeler.input_relabeling(lss_, lss_, cache_, hx_, o_, stateless_relabeler, stats_, shards_inner_series_, relabeled_results_); - prs.input_relabeling(lss_, lss_, cache_, hx_, o_, stats_, shards_inner_series_, relabeled_results_); - // shard id 1 - EXPECT_EQ(relabeled_results_[0]->size(), 1); + // Assert + EXPECT_EQ(relabeled_results_[0]->size(), 0); EXPECT_EQ(relabeled_results_[1]->size(), 0); EXPECT_EQ(shards_inner_series_[0]->size(), 0); EXPECT_EQ(shards_inner_series_[1]->size(), 0); - EXPECT_EQ(stats_.samples_added, 1); - EXPECT_EQ(stats_.series_added, 1); - - PromPP::Prometheus::Relabel::RelabelerStateUpdate update_data{}; - prs.append_relabeler_series(lss_, shards_inner_series_[0], relabeled_results_[0], &update_data); - EXPECT_EQ(shards_inner_series_[0]->size(), 1); - EXPECT_EQ(update_data.size(), 1); - - prs.update_relabeler_state(cache_, &update_data, 1); + EXPECT_EQ((Stats{.samples_added = 1, .series_added = 1, .series_drop = 1}), stats_); +} - auto rlabels = lss_[update_data[0].relabeled_ls_id]; - LabelViewSetForTest expected_labels = make_label_set({{"__name__", "booom"}, {"jab", "baj"}, {"job", "baj"}, {"replaced", "blabla"}}); +TEST_F(PerGoroutineRelabelerFixture, ReplaceToNewLS2) { + // Arrange + const RelabelConfig config{ + .source_labels = {{"__name__"}}, .separator = ";", .regex = ".*(o).*", .target_label = "replaced", .replacement = "$1", .action = rReplace}; + const StatelessRelabeler stateless_relabeler(std::initializer_list{&config}); + hx_.emplace_back({{"__name__", "booom"}, {"jab", "baj"}, {"job", "baj"}}, {{1000, 0.1}}); + PerGoroutineRelabeler relabeler{kNumberOfShards, 1}; + RelabelerStateUpdate update_data; - EXPECT_EQ(rlabels, expected_labels); -} + // Act + relabeler.input_relabeling(lss_, lss_, cache_, hx_, o_, stateless_relabeler, stats_, shards_inner_series_, relabeled_results_); + PerShardRelabeler::append_relabeler_series(lss_, shards_inner_series_[1], relabeled_results_[1], &update_data); -TEST_F(TestPerShardRelabeler, ReplaceToNewLS2_OrderedEncodingBimap) { - PromPP::Primitives::SnugComposites::LabelSet::OrderedEncodingBimap lss; - RelabelConfigTest rct{.source_labels = std::vector{"__name__"}, - .separator = ";", - .regex = ".*(o).*", - .target_label = "replaced", - .replacement = "$1", - .action = 5}; // Replace - Relabel::StatelessRelabeler sr(std::vector{&rct}); - PromPP::Prometheus::Relabel::PerShardRelabeler prs(external_labels_, &sr, 2, 1); - make_hashdex(hx_, make_label_set({{"__name__", "booom"}, {"jab", "baj"}, {"job", "baj"}}), make_samples({{1712567046855, 0.1}})); - - prs.input_relabeling(lss, lss, cache_, hx_, o_, stats_, shards_inner_series_, relabeled_results_); - // shard id 1 + // Assert EXPECT_EQ(relabeled_results_[0]->size(), 0); EXPECT_EQ(relabeled_results_[1]->size(), 1); EXPECT_EQ(shards_inner_series_[0]->size(), 0); + EXPECT_TRUE(std::ranges::equal(std::vector{{.sample = Sample(1000, 0.1), .ls_id = 1}}, shards_inner_series_[1]->data())); + EXPECT_EQ((Stats{.samples_added = 1, .series_added = 1}), stats_); + ASSERT_EQ(1U, update_data.size()); + EXPECT_EQ((LabelViewSet{{"__name__", "booom"}, {"jab", "baj"}, {"job", "baj"}, {"replaced", "o"}}), lss_[update_data[0].relabeled_ls_id]); +} + +TEST_F(PerGoroutineRelabelerFixture, ReplaceToNewLS3) { + // Arrange + const RelabelConfig config{.separator = ";", .regex = ".*", .target_label = "replaced", .replacement = "blabla", .action = rReplace}; + const StatelessRelabeler stateless_relabeler(std::initializer_list{&config}); + hx_.emplace_back({{"__name__", "booom"}, {"jab", "baj"}, {"job", "baj"}}, {{1000, 0.1}}); + PerGoroutineRelabeler relabeler{kNumberOfShards, 1}; + RelabelerStateUpdate update_data; + + // Act + relabeler.input_relabeling(lss_, lss_, cache_, hx_, o_, stateless_relabeler, stats_, shards_inner_series_, relabeled_results_); + PerShardRelabeler::append_relabeler_series(lss_, shards_inner_series_[0], relabeled_results_[0], &update_data); + PerShardRelabeler::update_relabeler_state(cache_, &update_data, 1); + + // Assert + EXPECT_EQ(relabeled_results_[0]->size(), 1); + EXPECT_EQ(relabeled_results_[1]->size(), 0); + EXPECT_TRUE(std::ranges::equal(std::vector{{.sample = Sample(1000, 0.1), .ls_id = 1}}, shards_inner_series_[0]->data())); EXPECT_EQ(shards_inner_series_[1]->size(), 0); - EXPECT_EQ(stats_.samples_added, 1); - EXPECT_EQ(stats_.series_added, 1); - - PromPP::Prometheus::Relabel::RelabelerStateUpdate update_data{}; - prs.append_relabeler_series(lss, shards_inner_series_[1], relabeled_results_[1], &update_data); - EXPECT_EQ(shards_inner_series_[1]->size(), 1); + EXPECT_EQ((Stats{.samples_added = 1, .series_added = 1}), stats_); EXPECT_EQ(update_data.size(), 1); - - prs.update_relabeler_state(cache_, &update_data, 1); - - auto rlabels = lss[update_data[0].relabeled_ls_id]; - LabelViewSetForTest expected_labels = make_label_set({{"__name__", "booom"}, {"jab", "baj"}, {"job", "baj"}, {"replaced", "o"}}); - - EXPECT_EQ(rlabels, expected_labels); + EXPECT_EQ((LabelViewSet{{"__name__", "booom"}, {"jab", "baj"}, {"job", "baj"}, {"replaced", "blabla"}}), lss_[update_data[0].relabeled_ls_id]); } -TEST_F(TestPerShardRelabeler, InputRelabelingWithStalenans_Default) { - RelabelConfigTest rct{.source_labels = std::vector{"job"}, .regex = "abc", .action = 2}; // Keep - Relabel::StatelessRelabeler sr(std::vector{&rct}); - PromPP::Prometheus::Relabel::PerShardRelabeler prs(external_labels_, &sr, 2, 1); - make_hashdex(hx_, make_label_set({{"__name__", "value"}, {"job", "abc"}}), make_samples({{PromPP::Primitives::kNullTimestamp, 0.1}})); - PromPP::Prometheus::Relabel::StaleNaNsState state{}; - PromPP::Primitives::Timestamp def_timestamp{1712567046955}; +TEST_F(PerGoroutineRelabelerFixture, InputRelabelingWithStalenans_Default) { + // Arrange + const RelabelConfig config{.source_labels = {{"job"}}, .regex = "abc", .action = rKeep}; + const StatelessRelabeler stateless_relabeler(std::initializer_list{&config}); + hx_.emplace_back({{"__name__", "value"}, {"job", "abc"}}, {{kNullTimestamp, 0.1}}); + PerGoroutineRelabeler relabeler{kNumberOfShards, 1}; + StaleNaNsState state; + RelabelerStateUpdate update_data; - prs.input_relabeling_with_stalenans(lss_, lss_, cache_, hx_, o_, stats_, shards_inner_series_, relabeled_results_, state, def_timestamp); + // Act + relabeler.input_relabeling_with_stalenans(lss_, lss_, cache_, hx_, o_, stateless_relabeler, stats_, shards_inner_series_, relabeled_results_, state, 1000); + PerShardRelabeler::append_relabeler_series(lss_, shards_inner_series_[1], relabeled_results_[1], &update_data); + relabeler.input_relabeling_with_stalenans(lss_, lss_, cache_, HashdexTest{}, o_, stateless_relabeler, stats_, shards_inner_series_, relabeled_results_, state, + 2000); - // shard id 1 + // Assert EXPECT_EQ(relabeled_results_[0]->size(), 0); EXPECT_EQ(relabeled_results_[1]->size(), 0); EXPECT_EQ(shards_inner_series_[0]->size(), 0); - EXPECT_EQ(shards_inner_series_[1]->size(), 1); - EXPECT_EQ(shards_inner_series_[1]->data()[0].sample.timestamp(), def_timestamp); - EXPECT_EQ(stats_.samples_added, 1); - EXPECT_EQ(stats_.series_added, 1); - - PromPP::Prometheus::Relabel::RelabelerStateUpdate update_data{}; - prs.append_relabeler_series(lss_, shards_inner_series_[1], relabeled_results_[1], &update_data); - EXPECT_EQ(shards_inner_series_[1]->size(), 1); + EXPECT_EQ(shards_inner_series_[1]->size(), 2); + EXPECT_TRUE(std::ranges::equal(std::vector{{.sample = Sample(1000, 0.1), .ls_id = 0}, {.sample = Sample(2000, kStaleNan), .ls_id = 0}}, + shards_inner_series_[1]->data())); + EXPECT_EQ((Stats{.samples_added = 1, .series_added = 1}), stats_); EXPECT_EQ(update_data.size(), 0); - - vector_shards_inner_series_[1] = std::make_unique(); - HashdexTest empty_hx; - PromPP::Primitives::Timestamp stale_ts = 1712567047055; - prs.input_relabeling_with_stalenans(lss_, lss_, cache_, empty_hx, o_, stats_, shards_inner_series_, relabeled_results_, state, stale_ts); - EXPECT_EQ(shards_inner_series_[1]->size(), 1); - EXPECT_EQ(PromPP::Primitives::Sample(stale_ts, PromPP::Prometheus::kStaleNan), shards_inner_series_[1]->data()[0].sample); - EXPECT_EQ(stats_.samples_added, 1); - EXPECT_EQ(stats_.series_added, 1); } -TEST_F(TestPerShardRelabeler, InputRelabelingWithStalenans_DefaultHonorTimestamps) { - RelabelConfigTest rct{.source_labels = std::vector{"job"}, .regex = "abc", .action = 2}; // Keep - Relabel::StatelessRelabeler sr(std::vector{&rct}); - PromPP::Prometheus::Relabel::PerShardRelabeler prs(external_labels_, &sr, 2, 1); - make_hashdex(hx_, make_label_set({{"__name__", "value"}, {"job", "abc"}}), make_samples({{PromPP::Primitives::kNullTimestamp, 0.1}})); - PromPP::Prometheus::Relabel::StaleNaNsState state{}; - PromPP::Primitives::Timestamp def_timestamp{1712567046955}; +TEST_F(PerGoroutineRelabelerFixture, InputRelabelingWithStalenans_DefaultHonorTimestamps) { + // Arrange + const RelabelConfig config{.source_labels = {{"job"}}, .regex = "abc", .action = rKeep}; + const StatelessRelabeler stateless_relabeler(std::initializer_list{&config}); + hx_.emplace_back({{"__name__", "value"}, {"job", "abc"}}, {{kNullTimestamp, 0.1}}); + PerGoroutineRelabeler relabeler{kNumberOfShards, 1}; + StaleNaNsState state; + RelabelerStateUpdate update_data; o_.honor_timestamps = true; - prs.input_relabeling_with_stalenans(lss_, lss_, cache_, hx_, o_, stats_, shards_inner_series_, relabeled_results_, state, def_timestamp); + // Act + relabeler.input_relabeling_with_stalenans(lss_, lss_, cache_, hx_, o_, stateless_relabeler, stats_, shards_inner_series_, relabeled_results_, state, 1000); + PerShardRelabeler::append_relabeler_series(lss_, shards_inner_series_[1], relabeled_results_[1], &update_data); + relabeler.input_relabeling_with_stalenans(lss_, lss_, cache_, HashdexTest{}, o_, stateless_relabeler, stats_, shards_inner_series_, relabeled_results_, state, + 2000); - // shard id 1 + // Assert EXPECT_EQ(relabeled_results_[0]->size(), 0); EXPECT_EQ(relabeled_results_[1]->size(), 0); EXPECT_EQ(shards_inner_series_[0]->size(), 0); - EXPECT_EQ(shards_inner_series_[1]->size(), 1); - EXPECT_EQ(shards_inner_series_[1]->data()[0].sample.timestamp(), def_timestamp); - EXPECT_EQ(stats_.samples_added, 1); - EXPECT_EQ(stats_.series_added, 1); - - PromPP::Prometheus::Relabel::RelabelerStateUpdate update_data{}; - prs.append_relabeler_series(lss_, shards_inner_series_[1], relabeled_results_[1], &update_data); - EXPECT_EQ(shards_inner_series_[1]->size(), 1); + EXPECT_EQ(shards_inner_series_[1]->size(), 2); + EXPECT_TRUE(std::ranges::equal(std::vector{{.sample = Sample(1000, 0.1), .ls_id = 0}, {.sample = Sample(2000, kStaleNan), .ls_id = 0}}, + shards_inner_series_[1]->data())); + EXPECT_EQ((Stats{.samples_added = 1, .series_added = 1}), stats_); EXPECT_EQ(update_data.size(), 0); - - vector_shards_inner_series_[1] = std::make_unique(); - HashdexTest empty_hx; - PromPP::Primitives::Timestamp stale_ts = 1712567047055; - prs.input_relabeling_with_stalenans(lss_, lss_, cache_, empty_hx, o_, stats_, shards_inner_series_, relabeled_results_, state, stale_ts); - EXPECT_EQ(shards_inner_series_[1]->size(), 1); - EXPECT_EQ(PromPP::Primitives::Sample(stale_ts, PromPP::Prometheus::kStaleNan), shards_inner_series_[1]->data()[0].sample); - EXPECT_EQ(stats_.samples_added, 1); - EXPECT_EQ(stats_.series_added, 1); } -TEST_F(TestPerShardRelabeler, InputRelabelingWithStalenans_WithMetricTimestamp) { - RelabelConfigTest rct{.source_labels = std::vector{"job"}, .regex = "abc", .action = 2}; // Keep - Relabel::StatelessRelabeler sr(std::vector{&rct}); - PromPP::Prometheus::Relabel::PerShardRelabeler prs(external_labels_, &sr, 2, 1); - make_hashdex(hx_, make_label_set({{"__name__", "value"}, {"job", "abc"}}), make_samples({{1712567046855, 0.1}})); - PromPP::Prometheus::Relabel::StaleNaNsState state{}; - PromPP::Primitives::Timestamp def_timestamp{1712567046955}; +TEST_F(PerGoroutineRelabelerFixture, InputRelabelingWithStalenans_WithMetricTimestamp) { + // Arrange + const RelabelConfig config{.source_labels = {{"job"}}, .regex = "abc", .action = rKeep}; + StatelessRelabeler stateless_relabeler(std::initializer_list{&config}); + hx_.emplace_back({{"__name__", "value"}, {"job", "abc"}}, {{1712567046855, 0.1}}); + PerGoroutineRelabeler relabeler{kNumberOfShards, 1}; + StaleNaNsState state; + RelabelerStateUpdate update_data; - prs.input_relabeling_with_stalenans(lss_, lss_, cache_, hx_, o_, stats_, shards_inner_series_, relabeled_results_, state, def_timestamp); + // Act + relabeler.input_relabeling_with_stalenans(lss_, lss_, cache_, hx_, o_, stateless_relabeler, stats_, shards_inner_series_, relabeled_results_, state, 1000); + PerShardRelabeler::append_relabeler_series(lss_, shards_inner_series_[1], relabeled_results_[1], &update_data); + relabeler.input_relabeling_with_stalenans(lss_, lss_, cache_, HashdexTest{}, o_, stateless_relabeler, stats_, shards_inner_series_, relabeled_results_, state, + 2000); - // shard id 1 + // Assert EXPECT_EQ(relabeled_results_[0]->size(), 0); EXPECT_EQ(relabeled_results_[1]->size(), 0); EXPECT_EQ(shards_inner_series_[0]->size(), 0); - EXPECT_EQ(shards_inner_series_[1]->size(), 1); - EXPECT_EQ(shards_inner_series_[1]->data()[0].sample.timestamp(), def_timestamp); - EXPECT_EQ(stats_.samples_added, 1); - EXPECT_EQ(stats_.series_added, 1); - - PromPP::Prometheus::Relabel::RelabelerStateUpdate update_data{}; - prs.append_relabeler_series(lss_, shards_inner_series_[1], relabeled_results_[1], &update_data); - EXPECT_EQ(shards_inner_series_[1]->size(), 1); + EXPECT_EQ(shards_inner_series_[1]->size(), 2); + EXPECT_TRUE(std::ranges::equal(std::vector{{.sample = Sample(1000, 0.1), .ls_id = 0}, {.sample = Sample(2000, kStaleNan), .ls_id = 0}}, + shards_inner_series_[1]->data())); + EXPECT_EQ((Stats{.samples_added = 1, .series_added = 1}), stats_); EXPECT_EQ(update_data.size(), 0); - - vector_shards_inner_series_[1] = std::make_unique(); - HashdexTest empty_hx; - PromPP::Primitives::Timestamp stale_ts = 1712567047055; - prs.input_relabeling_with_stalenans(lss_, lss_, cache_, empty_hx, o_, stats_, shards_inner_series_, relabeled_results_, state, stale_ts); - EXPECT_EQ(shards_inner_series_[1]->size(), 1); - EXPECT_EQ(PromPP::Primitives::Sample(stale_ts, PromPP::Prometheus::kStaleNan), shards_inner_series_[1]->data()[0].sample); - EXPECT_EQ(stats_.samples_added, 1); - EXPECT_EQ(stats_.series_added, 1); } -TEST_F(TestPerShardRelabeler, InputRelabelingWithStalenans_HonorTimestamps) { - RelabelConfigTest rct{.source_labels = std::vector{"job"}, .regex = "abc", .action = 2}; // Keep - Relabel::StatelessRelabeler sr(std::vector{&rct}); - PromPP::Prometheus::Relabel::PerShardRelabeler prs(external_labels_, &sr, 2, 1); - PromPP::Primitives::Timestamp metric_timestamp{1712567046955}; - make_hashdex(hx_, make_label_set({{"__name__", "value"}, {"job", "abc"}}), make_samples({{metric_timestamp, 0.1}})); - PromPP::Prometheus::Relabel::StaleNaNsState state{}; - PromPP::Primitives::Timestamp def_timestamp{1712567046955}; +TEST_F(PerGoroutineRelabelerFixture, InputRelabelingWithStalenans_HonorTimestamps) { + // Arrange + const RelabelConfig config{.source_labels = {{"job"}}, .regex = "abc", .action = rKeep}; + const StatelessRelabeler stateless_relabeler(std::initializer_list{&config}); + hx_.emplace_back({{"__name__", "value"}, {"job", "abc"}}, {{1500, 0.1}}); + PerGoroutineRelabeler relabeler{kNumberOfShards, 1}; o_.honor_timestamps = true; + StaleNaNsState state; + RelabelerStateUpdate update_data; - prs.input_relabeling_with_stalenans(lss_, lss_, cache_, hx_, o_, stats_, shards_inner_series_, relabeled_results_, state, def_timestamp); + // Act + relabeler.input_relabeling_with_stalenans(lss_, lss_, cache_, hx_, o_, stateless_relabeler, stats_, shards_inner_series_, relabeled_results_, state, 1000); + PerShardRelabeler::append_relabeler_series(lss_, shards_inner_series_[1], relabeled_results_[1], &update_data); + relabeler.input_relabeling_with_stalenans(lss_, lss_, cache_, HashdexTest{}, o_, stateless_relabeler, stats_, shards_inner_series_, relabeled_results_, state, + 2000); - // shard id 1 + // Assert EXPECT_EQ(relabeled_results_[0]->size(), 0); EXPECT_EQ(relabeled_results_[1]->size(), 0); EXPECT_EQ(shards_inner_series_[0]->size(), 0); EXPECT_EQ(shards_inner_series_[1]->size(), 1); - EXPECT_EQ(shards_inner_series_[1]->data()[0].sample.timestamp(), metric_timestamp); - EXPECT_EQ(stats_.samples_added, 1); - EXPECT_EQ(stats_.series_added, 1); - - PromPP::Prometheus::Relabel::RelabelerStateUpdate update_data{}; - prs.append_relabeler_series(lss_, shards_inner_series_[1], relabeled_results_[1], &update_data); - EXPECT_EQ(shards_inner_series_[1]->size(), 1); + EXPECT_TRUE(std::ranges::equal(std::vector{{.sample = Sample(1500, 0.1), .ls_id = 0}}, shards_inner_series_[1]->data())); + EXPECT_EQ((Stats{.samples_added = 1, .series_added = 1}), stats_); EXPECT_EQ(update_data.size(), 0); - - vector_shards_inner_series_[1] = std::make_unique(); - HashdexTest empty_hx; - PromPP::Primitives::Timestamp stale_ts = 1712567047055; - prs.input_relabeling_with_stalenans(lss_, lss_, cache_, empty_hx, o_, stats_, shards_inner_series_, relabeled_results_, state, stale_ts); - EXPECT_EQ(shards_inner_series_[1]->size(), 0); } -TEST_F(TestPerShardRelabeler, InputRelabelingWithStalenans_HonorTimestampsAndTrackStaleness) { - RelabelConfigTest rct{.source_labels = std::vector{"job"}, .regex = "abc", .action = 2}; // Keep - Relabel::StatelessRelabeler sr(std::vector{&rct}); - PromPP::Prometheus::Relabel::PerShardRelabeler prs(external_labels_, &sr, 2, 1); - PromPP::Primitives::Timestamp metric_timestamp{1712567046955}; - make_hashdex(hx_, make_label_set({{"__name__", "value"}, {"job", "abc"}}), make_samples({{metric_timestamp, 0.1}})); - PromPP::Prometheus::Relabel::StaleNaNsState state{}; - PromPP::Primitives::Timestamp def_timestamp{1712567046955}; +TEST_F(PerGoroutineRelabelerFixture, InputRelabelingWithStalenans_HonorTimestampsAndTrackStaleness) { + // Arrange + const RelabelConfig config{.source_labels = {{"job"}}, .regex = "abc", .action = rKeep}; + const StatelessRelabeler stateless_relabeler(std::initializer_list{&config}); + hx_.emplace_back({{"__name__", "value"}, {"job", "abc"}}, {{1500, 0.1}}); + PerGoroutineRelabeler relabeler{kNumberOfShards, 1}; o_.honor_timestamps = true; o_.track_timestamps_staleness = true; + StaleNaNsState state; + RelabelerStateUpdate update_data; - prs.input_relabeling_with_stalenans(lss_, lss_, cache_, hx_, o_, stats_, shards_inner_series_, relabeled_results_, state, def_timestamp); + // Act + relabeler.input_relabeling_with_stalenans(lss_, lss_, cache_, hx_, o_, stateless_relabeler, stats_, shards_inner_series_, relabeled_results_, state, 1000); + PerShardRelabeler::append_relabeler_series(lss_, shards_inner_series_[1], relabeled_results_[1], &update_data); + relabeler.input_relabeling_with_stalenans(lss_, lss_, cache_, HashdexTest{}, o_, stateless_relabeler, stats_, shards_inner_series_, relabeled_results_, state, + 2000); - // shard id 1 + // Assert EXPECT_EQ(relabeled_results_[0]->size(), 0); EXPECT_EQ(relabeled_results_[1]->size(), 0); EXPECT_EQ(shards_inner_series_[0]->size(), 0); - EXPECT_EQ(shards_inner_series_[1]->size(), 1); - EXPECT_EQ(shards_inner_series_[1]->data()[0].sample.timestamp(), metric_timestamp); - EXPECT_EQ(stats_.samples_added, 1); - EXPECT_EQ(stats_.series_added, 1); - - PromPP::Prometheus::Relabel::RelabelerStateUpdate update_data{}; - prs.append_relabeler_series(lss_, shards_inner_series_[1], relabeled_results_[1], &update_data); - EXPECT_EQ(shards_inner_series_[1]->size(), 1); + EXPECT_EQ(shards_inner_series_[1]->size(), 2); + EXPECT_TRUE(std::ranges::equal(std::vector{{.sample = Sample(1500, 0.1), .ls_id = 0}, {.sample = Sample(2000, kStaleNan), .ls_id = 0}}, + shards_inner_series_[1]->data())); + EXPECT_EQ((Stats{.samples_added = 1, .series_added = 1}), stats_); EXPECT_EQ(update_data.size(), 0); - - vector_shards_inner_series_[1] = std::make_unique(); - HashdexTest empty_hx; - PromPP::Primitives::Timestamp stale_ts = 1712567047055; - prs.input_relabeling_with_stalenans(lss_, lss_, cache_, empty_hx, o_, stats_, shards_inner_series_, relabeled_results_, state, stale_ts); - EXPECT_EQ(shards_inner_series_[1]->size(), 1); - EXPECT_EQ(PromPP::Primitives::Sample(stale_ts, PromPP::Prometheus::kStaleNan), shards_inner_series_[1]->data()[0].sample); - EXPECT_EQ(stats_.samples_added, 1); - EXPECT_EQ(stats_.series_added, 1); } -TEST_F(TestPerShardRelabeler, TargetLabels_HappyPath) { - RelabelConfigTest rct{.source_labels = std::vector{"job"}, .regex = "abc", .action = 2}; // Keep - Relabel::StatelessRelabeler sr(std::vector{&rct}); - PromPP::Prometheus::Relabel::PerShardRelabeler prs(external_labels_, &sr, 2, 0); - make_hashdex(hx_, make_label_set({{"__name__", "booom"}, {"jab", "baj"}, {"job", "abc"}}), make_samples({{1712567046855, 0.1}})); - std::vector> list_target_labels{{"a_name", "target_a_value"}, {"z_name", "target_z_value"}}; - add_target_labels(list_target_labels); +TEST_F(PerGoroutineRelabelerFixture, TargetLabels_HappyPath) { + // Arrange + const RelabelConfig config{.source_labels = {{"job"}}, .regex = "abc", .action = rKeep}; + const StatelessRelabeler stateless_relabeler(std::initializer_list{&config}); + hx_.emplace_back({{"__name__", "booom"}, {"jab", "baj"}, {"job", "abc"}}, {{1000, 0.1}}); + add_target_labels(LabelViewSet{{"a_name", "target_a_value"}, {"z_name", "target_z_value"}}); + PerGoroutineRelabeler relabeler{kNumberOfShards, 0}; + RelabelerStateUpdate update_data; - prs.input_relabeling(lss_, lss_, cache_, hx_, o_, stats_, shards_inner_series_, relabeled_results_); - // shard id 1 + // Act + relabeler.input_relabeling(lss_, lss_, cache_, hx_, o_, stateless_relabeler, stats_, shards_inner_series_, relabeled_results_); + PerShardRelabeler::append_relabeler_series(lss_, shards_inner_series_[1], relabeled_results_[1], &update_data); + PerShardRelabeler::update_relabeler_state(cache_, &update_data, 1); + + // Assert EXPECT_EQ(relabeled_results_[0]->size(), 0); EXPECT_EQ(relabeled_results_[1]->size(), 1); EXPECT_EQ(shards_inner_series_[0]->size(), 0); - EXPECT_EQ(shards_inner_series_[1]->size(), 0); - EXPECT_EQ(stats_.samples_added, 1); - EXPECT_EQ(stats_.series_added, 1); - - PromPP::Prometheus::Relabel::RelabelerStateUpdate update_data{}; - prs.append_relabeler_series(lss_, shards_inner_series_[1], relabeled_results_[1], &update_data); - EXPECT_EQ(shards_inner_series_[1]->size(), 1); + EXPECT_TRUE(std::ranges::equal(std::vector{{.sample = Sample(1000, 0.1), .ls_id = 1}}, shards_inner_series_[1]->data())); + EXPECT_EQ((Stats{.samples_added = 1, .series_added = 1}), stats_); EXPECT_EQ(update_data.size(), 1); - - prs.update_relabeler_state(cache_, &update_data, 1); - - auto rlabels = lss_[update_data[0].relabeled_ls_id]; - LabelViewSetForTest expected_labels = - make_label_set({{"__name__", "booom"}, {"a_name", "target_a_value"}, {"jab", "baj"}, {"job", "abc"}, {"z_name", "target_z_value"}}); - - EXPECT_EQ(rlabels, expected_labels); + EXPECT_EQ((LabelViewSet{{"__name__", "booom"}, {"a_name", "target_a_value"}, {"jab", "baj"}, {"job", "abc"}, {"z_name", "target_z_value"}}), + lss_[update_data[0].relabeled_ls_id]); } -TEST_F(TestPerShardRelabeler, TargetLabels_ExportedLabel) { - RelabelConfigTest rct{.source_labels = std::vector{"job"}, .regex = "abc", .action = 2}; // Keep - Relabel::StatelessRelabeler sr(std::vector{&rct}); - PromPP::Prometheus::Relabel::PerShardRelabeler prs(external_labels_, &sr, 2, 0); - make_hashdex(hx_, make_label_set({{"__name__", "booom"}, {"jab", "baj"}, {"job", "abc"}}), make_samples({{1712567046855, 0.1}})); - std::vector> list_target_labels{{"jab", "target_a_value"}, {"z_name", "target_z_value"}}; - add_target_labels(list_target_labels); +TEST_F(PerGoroutineRelabelerFixture, TargetLabels_ExportedLabel) { + // Arrange + const RelabelConfig config{.source_labels = {{"job"}}, .regex = "abc", .action = rsKeep}; + const StatelessRelabeler stateless_relabeler(std::initializer_list{&config}); + hx_.emplace_back({{"__name__", "booom"}, {"jab", "baj"}, {"job", "abc"}}, {{1000, 0.1}}); + add_target_labels(LabelViewSet{{"jab", "target_a_value"}, {"z_name", "target_z_value"}}); + PerGoroutineRelabeler relabeler{kNumberOfShards, 0}; + RelabelerStateUpdate update_data; - prs.input_relabeling(lss_, lss_, cache_, hx_, o_, stats_, shards_inner_series_, relabeled_results_); - // shard id 1 + // Act + relabeler.input_relabeling(lss_, lss_, cache_, hx_, o_, stateless_relabeler, stats_, shards_inner_series_, relabeled_results_); + PerShardRelabeler::append_relabeler_series(lss_, shards_inner_series_[0], relabeled_results_[0], &update_data); + PerShardRelabeler::update_relabeler_state(cache_, &update_data, 1); + + // Assert EXPECT_EQ(relabeled_results_[0]->size(), 1); EXPECT_EQ(relabeled_results_[1]->size(), 0); - EXPECT_EQ(shards_inner_series_[0]->size(), 0); + EXPECT_TRUE(std::ranges::equal(std::vector{{.sample = Sample(1000, 0.1), .ls_id = 1}}, shards_inner_series_[0]->data())); EXPECT_EQ(shards_inner_series_[1]->size(), 0); - EXPECT_EQ(stats_.samples_added, 1); - EXPECT_EQ(stats_.series_added, 1); - - PromPP::Prometheus::Relabel::RelabelerStateUpdate update_data{}; - prs.append_relabeler_series(lss_, shards_inner_series_[0], relabeled_results_[0], &update_data); - EXPECT_EQ(shards_inner_series_[0]->size(), 1); + EXPECT_EQ((Stats{.samples_added = 1, .series_added = 1}), stats_); EXPECT_EQ(update_data.size(), 1); - - prs.update_relabeler_state(cache_, &update_data, 1); - - auto rlabels = lss_[update_data[0].relabeled_ls_id]; - LabelViewSetForTest expected_labels = - make_label_set({{"__name__", "booom"}, {"exported_jab", "baj"}, {"jab", "target_a_value"}, {"job", "abc"}, {"z_name", "target_z_value"}}); - - EXPECT_EQ(rlabels, expected_labels); + EXPECT_EQ((LabelViewSet{{"__name__", "booom"}, {"exported_jab", "baj"}, {"jab", "target_a_value"}, {"job", "abc"}, {"z_name", "target_z_value"}}), + lss_[update_data[0].relabeled_ls_id]); } -TEST_F(TestPerShardRelabeler, TargetLabels_ExportedLabel_Honor) { - RelabelConfigTest rct{.source_labels = std::vector{"job"}, .regex = "abc", .action = 2}; // Keep - Relabel::StatelessRelabeler sr(std::vector{&rct}); - PromPP::Prometheus::Relabel::PerShardRelabeler prs(external_labels_, &sr, 2, 0); - make_hashdex(hx_, make_label_set({{"__name__", "booom"}, {"jab", "baj"}, {"job", "abc"}}), make_samples({{1712567046855, 0.1}})); - std::vector> list_target_labels{{"jab", "target_a_value"}, {"z_name", "target_z_value"}}; - add_target_labels(list_target_labels); +TEST_F(PerGoroutineRelabelerFixture, TargetLabels_ExportedLabel_Honor) { + // Arrange + const RelabelConfig config{.source_labels = {{"job"}}, .regex = "abc", .action = rsKeep}; + const StatelessRelabeler stateless_relabeler(std::initializer_list{&config}); + hx_.emplace_back({{"__name__", "booom"}, {"jab", "baj"}, {"job", "abc"}}, {{1000, 0.1}}); + add_target_labels(LabelViewSet{{"jab", "target_a_value"}, {"z_name", "target_z_value"}}); + PerGoroutineRelabeler relabeler{kNumberOfShards, 0}; o_.honor_labels = true; + RelabelerStateUpdate update_data; - prs.input_relabeling(lss_, lss_, cache_, hx_, o_, stats_, shards_inner_series_, relabeled_results_); - // shard id 1 + // Act + relabeler.input_relabeling(lss_, lss_, cache_, hx_, o_, stateless_relabeler, stats_, shards_inner_series_, relabeled_results_); + PerShardRelabeler::append_relabeler_series(lss_, shards_inner_series_[0], relabeled_results_[0], &update_data); + PerShardRelabeler::update_relabeler_state(cache_, &update_data, 1); + + // Assert EXPECT_EQ(relabeled_results_[0]->size(), 1); EXPECT_EQ(relabeled_results_[1]->size(), 0); - EXPECT_EQ(shards_inner_series_[0]->size(), 0); + EXPECT_TRUE(std::ranges::equal(std::vector{{.sample = Sample(1000, 0.1), .ls_id = 1}}, shards_inner_series_[0]->data())); EXPECT_EQ(shards_inner_series_[1]->size(), 0); - EXPECT_EQ(stats_.samples_added, 1); - EXPECT_EQ(stats_.series_added, 1); - - PromPP::Prometheus::Relabel::RelabelerStateUpdate update_data{}; - prs.append_relabeler_series(lss_, shards_inner_series_[0], relabeled_results_[0], &update_data); - EXPECT_EQ(shards_inner_series_[0]->size(), 1); + EXPECT_EQ((Stats{.samples_added = 1, .series_added = 1}), stats_); EXPECT_EQ(update_data.size(), 1); + EXPECT_EQ((LabelViewSet{{"__name__", "booom"}, {"jab", "baj"}, {"job", "abc"}, {"z_name", "target_z_value"}}), lss_[update_data[0].relabeled_ls_id]); +} - prs.update_relabeler_state(cache_, &update_data, 1); +TEST_F(PerGoroutineRelabelerFixture, SampleLimitExceeded) { + // Arrange + const RelabelConfig config{.source_labels = {{"job"}}, .regex = "abc", .action = rKeep}; + const StatelessRelabeler stateless_relabeler(std::initializer_list{&config}); + hx_ = HashdexTest{ + {{{"__name__", "value"}, {"job", "abc"}}, {{1000, kStaleNan}}}, + {{{"__name__", "value"}, {"job", "abc"}}, {{2000, 0.1}}}, + {{{"__name__", "value"}, {"job", "abc"}}, {{3000, 0.1}}}, + }; + MetricLimits limits{.sample_limit = 1}; + PerGoroutineRelabeler relabeler{kNumberOfShards, 1}; + o_.metric_limits = &limits; - auto rlabels = lss_[update_data[0].relabeled_ls_id]; - LabelViewSetForTest expected_labels = make_label_set({{"__name__", "booom"}, {"jab", "baj"}, {"job", "abc"}, {"z_name", "target_z_value"}}); + // Act + relabeler.input_relabeling(lss_, lss_, cache_, hx_, o_, stateless_relabeler, stats_, shards_inner_series_, relabeled_results_); - EXPECT_EQ(rlabels, expected_labels); + // Assert + EXPECT_EQ(relabeled_results_[0]->size(), 0); + EXPECT_EQ(relabeled_results_[1]->size(), 0); + EXPECT_EQ(shards_inner_series_[0]->size(), 0); + EXPECT_TRUE(std::ranges::equal(std::vector{{.sample = Sample(1000, kStaleNan), .ls_id = 0}, {.sample = Sample(2000, 0.1), .ls_id = 0}}, + shards_inner_series_[1]->data())); + EXPECT_EQ((Stats{.samples_added = 2, .series_added = 1}), stats_); } -struct TestTargetLabels : public testing::Test { - // target_labels - std::vector> vector_target_labels_; +class TargetLabelsFixture : public testing::Test { + protected: + static constexpr uint16_t kNumberOfShards = 2; + + std::vector vector_target_labels_; - // Options PromPP::Prometheus::Relabel::RelabelerOptions o_; + std::vector rcts_; + std::vector vector_external_labels_; + SliceView external_labels_{}; - // for init PerShardRelabeler - PromPP::Primitives::SnugComposites::LabelSet::EncodingBimap lss_; - std::vector rcts_; - std::vector> vector_external_labels_; - PromPP::Primitives::Go::SliceView> external_labels_; + LabelsBuilder builder_; + + StatelessRelabeler stateless_relabeler_{rcts_}; + PerShardRelabeler relabeler_{external_labels_, &stateless_relabeler_, kNumberOfShards, 1}; void SetUp() final { o_.target_labels.reset_to(vector_target_labels_.data(), vector_target_labels_.size(), vector_target_labels_.size()); external_labels_.reset_to(vector_external_labels_.data(), vector_external_labels_.size(), vector_external_labels_.size()); } - void add_target_labels(std::vector>& list_target_labels) { + void add_target_labels(const LabelViewSet& list_target_labels) { vector_target_labels_.resize(list_target_labels.size()); for (size_t i = 0; i < vector_target_labels_.size(); i++) { vector_target_labels_[i].first.reset_to(list_target_labels[i].first.data(), list_target_labels[i].first.size()); @@ -1037,226 +600,242 @@ struct TestTargetLabels : public testing::Test { } }; -TEST_F(TestTargetLabels, ResolveConflictingExposedLabels_EmptyConflictingLabels) { - auto labels = make_label_set({{"c_name", "c_value"}}); - PromPP::Primitives::LabelsBuilderStateMap builder_state; - PromPP::Primitives::LabelsBuilder builder{builder_state}; - builder.reset(labels); - Relabel::StatelessRelabeler sr(rcts_); - PromPP::Prometheus::Relabel::PerShardRelabeler prs(external_labels_, &sr, 2, 1); - std::vector conflicting_exposed_labels{}; - LabelViewSetForTest expected_labels = make_label_set({{"c_name", "c_value"}}); +TEST_F(TargetLabelsFixture, ResolveConflictingExposedLabels_EmptyConflictingLabels) { + // Arrange + const LabelViewSet labels{{"c_name", "c_value"}}; + + builder_.reset(labels); - prs.resolve_conflicting_exposed_labels(builder, conflicting_exposed_labels); + std::vector