From 0c65d784434a37c1b0fb49b83f9ee877909a804e Mon Sep 17 00:00:00 2001 From: Ales Pour Date: Fri, 7 Jun 2024 12:46:08 +0200 Subject: [PATCH 01/16] feat(client): add initial support for exporting to Parquet --- cmd/influx_inspect/export/export.go | 40 +- cmd/influx_inspect/export/export_parquet.go | 349 ++++++++++++++++ cmd/influx_inspect/export/export_test.go | 2 +- cmd/influx_inspect/export/parquet/README.md | 2 + .../export/parquet/exporter/exporter.go | 168 ++++++++ .../export/parquet/exporter/meta.go | 79 ++++ .../parquet/index/measurement_schema.go | 21 + .../export/parquet/models/points.go | 389 ++++++++++++++++++ .../export/parquet/resultset/array.go | 91 ++++ .../export/parquet/resultset/result_set.go | 50 +++ .../export/parquet/table/allocator.go | 31 ++ .../export/parquet/table/column.go | 206 ++++++++++ .../export/parquet/table/resource_limits.go | 59 +++ .../export/parquet/table/table.go | 379 +++++++++++++++++ .../export/parquet/tsm1/encoding.go | 42 ++ go.mod | 62 +-- go.sum | 137 +++--- 17 files changed, 2024 insertions(+), 83 deletions(-) create mode 100644 cmd/influx_inspect/export/export_parquet.go create mode 100644 cmd/influx_inspect/export/parquet/README.md create mode 100644 cmd/influx_inspect/export/parquet/exporter/exporter.go create mode 100644 cmd/influx_inspect/export/parquet/exporter/meta.go create mode 100644 cmd/influx_inspect/export/parquet/index/measurement_schema.go create mode 100644 cmd/influx_inspect/export/parquet/models/points.go create mode 100644 cmd/influx_inspect/export/parquet/resultset/array.go create mode 100644 cmd/influx_inspect/export/parquet/resultset/result_set.go create mode 100644 cmd/influx_inspect/export/parquet/table/allocator.go create mode 100644 cmd/influx_inspect/export/parquet/table/column.go create mode 100644 cmd/influx_inspect/export/parquet/table/resource_limits.go create mode 100644 cmd/influx_inspect/export/parquet/table/table.go create mode 100644 cmd/influx_inspect/export/parquet/tsm1/encoding.go diff --git a/cmd/influx_inspect/export/export.go b/cmd/influx_inspect/export/export.go index 0906f02af9b..cab6cee03c0 100644 --- a/cmd/influx_inspect/export/export.go +++ b/cmd/influx_inspect/export/export.go @@ -33,14 +33,20 @@ type Command struct { out string database string retentionPolicy string + measurement string startTime int64 endTime int64 compress bool lponly bool + parquet bool + pqChunkSize int manifest map[string]struct{} tsmFiles map[string][]string walFiles map[string][]string + + writeValues func(io.Writer, []byte, string, []tsm1.Value) error + exportDone func(string) error } const stdoutMark = "-" @@ -71,14 +77,17 @@ func (cmd *Command) Run(args ...string) error { fs.StringVar(&cmd.out, "out", os.Getenv("HOME")+"/.influxdb/export", "'-' for standard out or the destination file to export to") fs.StringVar(&cmd.database, "database", "", "Optional: the database to export") fs.StringVar(&cmd.retentionPolicy, "retention", "", "Optional: the retention policy to export (requires -database)") + fs.StringVar(&cmd.measurement, "measurement", "", "Name of measurement to export") fs.StringVar(&start, "start", "", "Optional: the start time to export (RFC3339 format)") fs.StringVar(&end, "end", "", "Optional: the end time to export (RFC3339 format)") fs.BoolVar(&cmd.lponly, "lponly", false, "Only export line protocol") fs.BoolVar(&cmd.compress, "compress", false, "Compress the output") + fs.BoolVar(&cmd.parquet, "parquet", false, "Export to Parquet format (requires -database -retention -measurement)") + fs.IntVar(&cmd.pqChunkSize, "chunk-size", 100_000_000, "Size to partition Parquet files (in bytes)") fs.SetOutput(cmd.Stdout) fs.Usage = func() { - fmt.Fprintf(cmd.Stdout, "Exports TSM files into InfluxDB line protocol format.\n\n") + fmt.Fprintf(cmd.Stdout, "Exports TSM files into InfluxDB line protocol or Parquet format.\n\n") fmt.Fprintf(cmd.Stdout, "Usage: %s export [flags]\n\n", filepath.Base(os.Args[0])) fs.PrintDefaults() } @@ -112,6 +121,14 @@ func (cmd *Command) Run(args ...string) error { return err } + if cmd.parquet { + cmd.writeValues = cmd.writeValuesParquet + cmd.exportDone = cmd.exportDoneParquet + } else { + cmd.writeValues = cmd.writeValuesLp + cmd.exportDone = func(_ string) error { return nil } + } + return cmd.export() } @@ -122,6 +139,9 @@ func (cmd *Command) validate() error { if cmd.startTime != 0 && cmd.endTime != 0 && cmd.endTime < cmd.startTime { return fmt.Errorf("end time before start time") } + if cmd.parquet && (cmd.database == "" || cmd.retentionPolicy == "" || cmd.measurement == "") { + return fmt.Errorf("must specify database, renetion and measurement when exporting to Parquet") + } return nil } @@ -133,6 +153,9 @@ func (cmd *Command) export() error { return err } + if cmd.parquet { + return cmd.writeDML(io.Discard, io.Discard) + } return cmd.write() } @@ -331,6 +354,9 @@ func (cmd *Command) writeTsmFiles(mw io.Writer, w io.Writer, files []string) err if err := cmd.exportTSMFile(f, w); err != nil { return err } + if err := cmd.exportDone(f); err != nil { + return err + } } return nil @@ -368,11 +394,16 @@ func (cmd *Command) exportTSMFile(tsmFilePath string, w io.Writer) error { measurement, field := tsm1.SeriesAndFieldFromCompositeKey(key) field = escape.Bytes(field) + if cmd.measurement != "" && cmd.measurement != strings.Split(string(measurement), ",")[0] { + continue + } + if err := cmd.writeValues(w, measurement, string(field), values); err != nil { // An error from writeValues indicates an IO error, which should be returned. return err } } + return nil } @@ -436,6 +467,10 @@ func (cmd *Command) exportWALFile(walFilePath string, w io.Writer, warnDelete fu // measurements are stored escaped, field names are not field = escape.Bytes(field) + if cmd.measurement != "" && cmd.measurement != strings.Split(string(measurement), ",")[0] { + continue + } + if err := cmd.writeValues(w, measurement, string(field), values); err != nil { // An error from writeValues indicates an IO error, which should be returned. return err @@ -443,12 +478,13 @@ func (cmd *Command) exportWALFile(walFilePath string, w io.Writer, warnDelete fu } } } + return nil } // writeValues writes every value in values to w, using the given series key and field name. // If any call to w.Write fails, that error is returned. -func (cmd *Command) writeValues(w io.Writer, seriesKey []byte, field string, values []tsm1.Value) error { +func (cmd *Command) writeValuesLp(w io.Writer, seriesKey []byte, field string, values []tsm1.Value) error { buf := []byte(string(seriesKey) + " " + field + "=") prefixLen := len(buf) diff --git a/cmd/influx_inspect/export/export_parquet.go b/cmd/influx_inspect/export/export_parquet.go new file mode 100644 index 00000000000..10e6c275cfb --- /dev/null +++ b/cmd/influx_inspect/export/export_parquet.go @@ -0,0 +1,349 @@ +package export + +import ( + "context" + "fmt" + "golang.org/x/exp/maps" + "io" + "os" + "path/filepath" + "sync" + "time" + + "github.com/apache/arrow/go/v14/parquet" + "github.com/apache/arrow/go/v14/parquet/compress" + pqexport "github.com/influxdata/influxdb/cmd/influx_inspect/export/parquet/exporter" + "github.com/influxdata/influxdb/cmd/influx_inspect/export/parquet/index" + "github.com/influxdata/influxdb/cmd/influx_inspect/export/parquet/models" + "github.com/influxdata/influxdb/cmd/influx_inspect/export/parquet/resultset" + "github.com/influxdata/influxdb/cmd/influx_inspect/export/parquet/table" + tsdb "github.com/influxdata/influxdb/cmd/influx_inspect/export/parquet/tsm1" + "github.com/influxdata/influxdb/tsdb/engine/tsm1" + "go.uber.org/zap" +) + +// +// Export to Parquet file(s) is done per each TSM file. The files are apparently not sorted. +// Therefore, neither are output files. So for example `table-00001.parquet` may contain older data +// than `table-00000.parquet`. +// If the data should be sorted then +// 1. writeValuesParquet must collect multiple chunks of field values, and they should be sorted later +// 2. exporter must call exportDone after all files were processed +// + +// sequence is used for the suffix of generated parquet files +var sequence int + +// vc is the key:field:values collection of exported values +var vc map[string]map[string][]tsm1.Value + +func (cmd *Command) writeValuesParquet(_ io.Writer, seriesKey []byte, field string, values []tsm1.Value) error { + if vc == nil { + vc = make(map[string]map[string][]tsm1.Value) + } + + key := string(seriesKey) + fields, ok := vc[key] + if !ok { + fields = make(map[string][]tsm1.Value) + vc[key] = fields + } + fields[field] = values + + return nil +} + +func (cmd *Command) exportDoneParquet(path string) error { + defer func() { + vc = nil + }() + + var schema *index.MeasurementSchema + + for key, fields := range vc { + // since code from v2 exporter is used, we need v2 series key + seriesKeyEx := append([]byte(models.MeasurementTagKey+"="), []byte(key)...) + // get tags + tags := models.ParseTags(models.Escaped{B: seriesKeyEx}) + tagSet := make(map[string]struct{}, len(tags)) + for _, tag := range tags { + tagSet[string(tag.Key)] = struct{}{} + } + // get fields + fieldSet := make(map[index.MeasurementField]struct{}, len(fields)) + for field, values := range fields { + bt, err := tsdb.BlockTypeForType(values[0].Value()) + if err != nil { + return err + } + mf := index.MeasurementField{ + Name: field, + Type: bt, + } + fieldSet[mf] = struct{}{} + } + // populate schema + schema = &index.MeasurementSchema{ + TagSet: tagSet, + FieldSet: fieldSet, + } + // schema does not change in a table + break + } + + if err := cmd.writeToParquet(newResultSet(vc), schema); err != nil { + return err + } + + return nil +} + +func (cmd *Command) writeToParquet(rs resultset.ResultSet, schema *index.MeasurementSchema) error { + cfg := zap.NewDevelopmentConfig() + if cmd.usingStdOut() { + cfg.OutputPaths = []string{ + "stdout", + } + } + log, _ := cfg.Build() + + log.Info("Generating parquet", zap.Int("chunk_size", cmd.pqChunkSize)) + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + t, err := table.New(rs, schema) + if err != nil { + return err + } + + exporter := pqexport.New(t, + pqexport.WithParquetWriterProperties( + parquet.WithCompression(compress.Codecs.Snappy))) + + var wg sync.WaitGroup + + wg.Add(1) + go func() { + defer wg.Done() + + start := time.Now() + var ( + lastSeriesCount int64 + lastRowCount int64 + ) + for { + ticker := time.NewTicker(30 * time.Second) + select { + case <-ctx.Done(): + stats := t.Stats() + log.Info("Total Processed", zap.Int64("series_count", stats.SeriesCount), zap.Int64("rows", stats.RowCount), zap.Duration("build_time", time.Since(start))) + return + case <-ticker.C: + stats := t.Stats() + var ( + lastSeries int64 + lastRows int64 + ) + lastSeries, lastSeriesCount = stats.SeriesCount-lastSeriesCount, stats.SeriesCount + lastRows, lastRowCount = stats.RowCount-lastRowCount, stats.RowCount + log.Info("Processed", + zap.Int64("total_series", stats.SeriesCount), + zap.Int64("total_rows", stats.RowCount), + zap.Int64("last_series", lastSeries), + zap.Int64("last_rows", lastRows), + ) + } + } + }() + + log.Info("Schema info", + zap.Int("tag_count", len(schema.TagSet)), + zap.Int("field_count", len(schema.FieldSet))) + + var done bool + for !done && err == nil { + done, err = func() (done bool, err error) { + defer func() { sequence++ }() + + var parquetFile *os.File + { + fileName := filepath.Join(cmd.out, fmt.Sprintf("table-%05d.parquet", sequence)) + parquetFile, err = os.Create(fileName) + if err != nil { + return false, err + } + defer func() { + closeErr := parquetFile.Close() + if err == nil { + // we only care if the file fails to close when there was no + // previous error + err = closeErr + } + }() + } + + var metaFile *os.File + { + fileName := filepath.Join(cmd.out, fmt.Sprintf("table-%05d.meta.json", sequence)) + metaFile, err = os.Create(fileName) + if err != nil { + return false, err + } + defer func() { + closeErr := metaFile.Close() + if err == nil { + // we only care if the file fails to close when there was no + // previous error + err = closeErr + } + }() + } + + stats := t.Stats() + log.Info("Starting Parquet file.", + zap.Int64("total_series", stats.SeriesCount), + zap.Int64("total_rows", stats.RowCount), + zap.String("parquet", parquetFile.Name()), + zap.String("meta", metaFile.Name()), + ) + + defer func() { + stats := t.Stats() + log.Info("Closing Parquet file.", + zap.Int64("total_series", stats.SeriesCount), + zap.Int64("total_rows", stats.RowCount), + zap.String("parquet", parquetFile.Name()), + zap.String("meta", metaFile.Name()), + ) + }() + + return exporter.WriteChunk(ctx, parquetFile, metaFile, cmd.pqChunkSize) + }() + } + + cancel() + wg.Wait() + + if err != nil { + log.Error("Failed to write table.", zap.Error(err)) + } else { + log.Info("Finished") + } + + return nil +} + +// resultSet implements resultset.ResultSet over exported TSM data +type resultSet struct { + x map[string]map[string][]tsm1.Value + keys []string + keyIndex int + fields []string + fieldIndex int +} + +func newResultSet(x map[string]map[string][]tsm1.Value) resultset.ResultSet { + return &resultSet{ + x: x, + keys: maps.Keys(x), + } +} + +func (r *resultSet) Next(ctx context.Context) resultset.SeriesCursor { + if r.keyIndex >= len(r.keys) { + return nil + } + + if r.fields == nil { + r.fields = maps.Keys(r.x[r.keys[r.keyIndex]]) + } + + if r.fieldIndex >= len(r.fields) { + r.fields = nil + r.fieldIndex = 0 + r.keyIndex++ + return r.Next(ctx) + } + + defer func() { + r.fieldIndex++ + }() + + groupKey := r.keys[r.keyIndex] + fieldName := r.fields[r.fieldIndex] + // since code from v2 exporter is used, we need v2 series key + seriesKeyEx := append([]byte(groupKey), []byte(","+models.FieldKeyTagKey+"="+fieldName)...) + values := r.x[groupKey][fieldName] + + switch values[0].Value().(type) { + case int64: + return newExportedCursor[int64](seriesKeyEx, values) + case float64: + return newExportedCursor[float64](seriesKeyEx, values) + case string: + return newExportedCursor[string](seriesKeyEx, values) + case uint64: + return newExportedCursor[uint64](seriesKeyEx, values) + case bool: + return newExportedCursor[bool](seriesKeyEx, values) + } + + return nil +} + +func (r *resultSet) Close() { +} + +func (r *resultSet) Err() error { + return nil +} + +// exportedCursor implements resultset.TypedCursor for exported TSM data +type exportedCursor[T resultset.BlockType] struct { + seriesKey []byte + chunks []*resultset.TimeArray[T] + chunkIndex int +} + +func newExportedCursor[T resultset.BlockType](seriesKey []byte, values []tsm1.Value) *exportedCursor[T] { + c := &exportedCursor[T]{ + seriesKey: seriesKey, + chunks: []*resultset.TimeArray[T]{ + &resultset.TimeArray[T]{}, + }, + } + c.chunks[0].Timestamps = make([]int64, len(values)) + c.chunks[0].Values = make([]T, len(values)) + for i, value := range values { + c.chunks[0].Timestamps[i] = value.UnixNano() + c.chunks[0].Values[i] = value.(TsmValue).Value().(T) + } + return c +} + +func (c *exportedCursor[T]) SeriesKey() models.Escaped { + return models.Escaped{B: c.seriesKey} +} + +func (c *exportedCursor[T]) Close() { +} + +func (c *exportedCursor[T]) Err() error { + return nil +} + +func (c *exportedCursor[T]) Next() *resultset.TimeArray[T] { + if c.chunkIndex >= len(c.chunks) { + return nil + } + defer func() { + c.chunkIndex++ + }() + return c.chunks[c.chunkIndex] +} + +// TsmValue is helper for accessing value of tsm1.Value +type TsmValue interface { + Value() interface{} +} diff --git a/cmd/influx_inspect/export/export_test.go b/cmd/influx_inspect/export/export_test.go index e7252edb9f0..1ecb83f3c93 100644 --- a/cmd/influx_inspect/export/export_test.go +++ b/cmd/influx_inspect/export/export_test.go @@ -248,7 +248,7 @@ func makeCorpus(numSeries, numValuesPerSeries int, fn func(*rand.Rand) interface } k := fmt.Sprintf("m,t=%d", i) - corpus[tsm1.SeriesFieldKey(k, "x")] = vals + corpus[tsm1.SeriesFieldKey(k, "vc")] = vals } return corpus diff --git a/cmd/influx_inspect/export/parquet/README.md b/cmd/influx_inspect/export/parquet/README.md new file mode 100644 index 00000000000..c6c5e596317 --- /dev/null +++ b/cmd/influx_inspect/export/parquet/README.md @@ -0,0 +1,2 @@ +Contains Parquet exporter code ported from idpe. +Only the part required to make it work, with minimal modifications. \ No newline at end of file diff --git a/cmd/influx_inspect/export/parquet/exporter/exporter.go b/cmd/influx_inspect/export/parquet/exporter/exporter.go new file mode 100644 index 00000000000..00fe2c41908 --- /dev/null +++ b/cmd/influx_inspect/export/parquet/exporter/exporter.go @@ -0,0 +1,168 @@ +package exporter + +import ( + "context" + "io" + + "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v14/arrow/array" + "github.com/apache/arrow/go/v14/parquet" + "github.com/apache/arrow/go/v14/parquet/pqarrow" + "github.com/influxdata/influxdb/cmd/influx_inspect/export/parquet/models" + "github.com/influxdata/influxdb/cmd/influx_inspect/export/parquet/table" + "github.com/pkg/errors" +) + +type options struct { + writerProps []parquet.WriterProperty +} + +func (o *options) apply(fns ...OptionFn) { + for _, fn := range fns { + fn(o) + } +} + +type OptionFn func(*options) + +func WithParquetWriterProperties(opts ...parquet.WriterProperty) OptionFn { + return func(o *options) { + o.writerProps = opts + } +} + +type Exporter struct { + table *table.Table + schema *arrow.Schema + + writerProps *parquet.WriterProperties + arrowWriterProps pqarrow.ArrowWriterProperties +} + +func New(t *table.Table, fns ...OptionFn) *Exporter { + var opts options + opts.apply(fns...) + + columns := t.Columns() + fields := make([]arrow.Field, 0, len(columns)) + fields = append(fields, arrow.Field{ + Name: "time", + Type: &arrow.TimestampType{Unit: arrow.Nanosecond}, + Nullable: false, + }) + + // skip timestamp column + for _, col := range columns[1:] { + fields = append(fields, arrow.Field{ + Name: col.Name(), + Type: col.DataType(), + Nullable: true, + }) + } + schema := arrow.NewSchema(fields, nil) + + writerOptions := opts.writerProps + + // override subset of options to ensure consistency + writerOptions = append(writerOptions, + parquet.WithDictionaryDefault(false), + parquet.WithDataPageVersion(parquet.DataPageV2), + parquet.WithVersion(parquet.V2_LATEST), + ) + writerProps := parquet.NewWriterProperties(writerOptions...) + + return &Exporter{ + table: t, + schema: schema, + writerProps: writerProps, + // no reason to override these defaults + arrowWriterProps: pqarrow.NewArrowWriterProperties(pqarrow.WithStoreSchema()), + } +} + +func (e *Exporter) WriteChunk(ctx context.Context, parquet io.Writer, metaW io.Writer, maxSize int) (done bool, err error) { + if e.table.Err() != nil { + return false, e.table.Err() + } + + parquetWriter := writeCounter{Writer: parquet} + + var fw *pqarrow.FileWriter + fw, err = pqarrow.NewFileWriter(e.schema, &parquetWriter, e.writerProps, e.arrowWriterProps) + if err != nil { + return false, err + } + defer func() { + closeErr := fw.Close() + if err == nil { + // we only care if the file fails to close when there was no + // previous error + err = closeErr + } + }() + + var ( + firstSeriesKey models.Escaped + readInfo table.ReadInfo + firstTimestamp arrow.Timestamp + lastTimestamp arrow.Timestamp + seenFirstSeriesKey = false + ) + defer func() { + if err == nil { + err = writeMeta(Meta{ + FirstSeriesKey: firstSeriesKey, + FirstTimestamp: int64(firstTimestamp), + LastSeriesKey: readInfo.LastGroupKey, + LastTimestamp: int64(lastTimestamp), + }, metaW) + } + }() + + for parquetWriter.Count() < maxSize { + var cols []arrow.Array + cols, readInfo = e.table.Next(ctx) + if len(cols) == 0 { + return true, errors.Wrap(e.table.Err(), "reading table") + } + + if !seenFirstSeriesKey { + firstSeriesKey = readInfo.FirstGroupKey.Clone() + firstTimestamp = cols[0].(*array.Timestamp).Value(0) + seenFirstSeriesKey = true + } + + if cols[0].Len() > 0 { + lastTimestamp = cols[0].(*array.Timestamp).Value(cols[0].Len() - 1) + + rec := array.NewRecord(e.schema, cols, int64(cols[0].Len())) + err = fw.Write(rec) + rec.Release() + if err != nil { + return false, errors.Wrap(err, "writing Parquet") + } + } + + for _, col := range cols { + col.Release() + } + } + + return +} + +// writeCounter counts the number of bytes written to an io.Writer +type writeCounter struct { + io.Writer + count int +} + +func (c *writeCounter) Write(buf []byte) (n int, err error) { + defer func() { c.count += n }() + return c.Writer.Write(buf) +} + +// Count function return counted bytes +func (c *writeCounter) Count() int { + return c.count +} diff --git a/cmd/influx_inspect/export/parquet/exporter/meta.go b/cmd/influx_inspect/export/parquet/exporter/meta.go new file mode 100644 index 00000000000..3c3182b628b --- /dev/null +++ b/cmd/influx_inspect/export/parquet/exporter/meta.go @@ -0,0 +1,79 @@ +package exporter + +import ( + "bytes" + "encoding/base64" + "errors" + "io" + + "github.com/influxdata/influxdb/cmd/influx_inspect/export/parquet/models" + jsoniter "github.com/json-iterator/go" +) + +// Meta contains information about an exported Parquet file. +type Meta struct { + FirstSeriesKey models.Escaped + FirstTimestamp int64 + LastSeriesKey models.Escaped + LastTimestamp int64 +} + +// meta is an intermediate type to facilitate encoding and decoding +// series keys in JSON format. +type meta struct { + FirstSeriesKey string `json:"firstSeriesKeyEscaped"` + FirstTimestamp int64 `json:"firstTimestamp"` + LastSeriesKey string `json:"lastSeriesKeyEscaped"` + LastTimestamp int64 `json:"lastTimestamp"` +} + +func (m *Meta) UnmarshalJSON(data []byte) error { + var tmp meta + if err := jsoniter.Unmarshal(data, &tmp); err != nil { + return err + } + + firstSeriesKey, err := base64.StdEncoding.DecodeString(tmp.FirstSeriesKey) + if err != nil { + return errors.New("failed to decode firstSeriesKeyEscaped") + } + lastSeriesKey, err := base64.StdEncoding.DecodeString(tmp.LastSeriesKey) + if err != nil { + return errors.New("failed to decode lastSeriesKeyEscaped") + } + + m.FirstSeriesKey = models.MakeEscaped(firstSeriesKey) + m.FirstTimestamp = tmp.FirstTimestamp + m.LastSeriesKey = models.MakeEscaped(lastSeriesKey) + m.LastTimestamp = tmp.LastTimestamp + + return nil +} + +func (m Meta) MarshalJSON() ([]byte, error) { + var b bytes.Buffer + enc := jsoniter.NewEncoder(&b) + err := enc.Encode(meta{ + FirstSeriesKey: base64.StdEncoding.EncodeToString(m.FirstSeriesKey.B), + FirstTimestamp: m.FirstTimestamp, + LastSeriesKey: base64.StdEncoding.EncodeToString(m.LastSeriesKey.B), + LastTimestamp: m.LastTimestamp, + }) + if err != nil { + return nil, err + } + + return b.Bytes(), nil +} + +func writeMeta(m Meta, w io.Writer) error { + enc := jsoniter.NewEncoder(w) + return enc.Encode(m) +} + +// ReadMeta decodes a Meta instance from the reader. +func ReadMeta(r io.Reader) (Meta, error) { + dec := jsoniter.NewDecoder(r) + var m Meta + return m, dec.Decode(&m) +} diff --git a/cmd/influx_inspect/export/parquet/index/measurement_schema.go b/cmd/influx_inspect/export/parquet/index/measurement_schema.go new file mode 100644 index 00000000000..cb5a1a839f2 --- /dev/null +++ b/cmd/influx_inspect/export/parquet/index/measurement_schema.go @@ -0,0 +1,21 @@ +package index + +import ( + tsdb "github.com/influxdata/influxdb/cmd/influx_inspect/export/parquet/tsm1" +) + +type ( + MeasurementField struct { + Name string `json:"name"` + Type tsdb.BlockType `json:"type"` + } + + MeasurementSchema struct { + TagSet map[string]struct{} + FieldSet map[MeasurementField]struct{} + } + + MeasurementKey = string + + MeasurementSchemas map[MeasurementKey]*MeasurementSchema +) diff --git a/cmd/influx_inspect/export/parquet/models/points.go b/cmd/influx_inspect/export/parquet/models/points.go new file mode 100644 index 00000000000..9fffd67b02c --- /dev/null +++ b/cmd/influx_inspect/export/parquet/models/points.go @@ -0,0 +1,389 @@ +package models + +import ( + "bytes" + "errors" + "fmt" +) + +const ( + FieldKeyTagKey = "\xff" + MeasurementTagKey = "\x00" + + fieldKeyTagKeySep = ",\xff=" +) + +var ( + fieldKeyTagKeySepBytes = []byte(fieldKeyTagKeySep) +) + +type escapeSet struct { + k [1]byte + esc [2]byte +} + +// Many routines in this package make assumptions that k is 1 byte +// and esc is 2 bytes. This code ensures that if these invariants +// change, a compile-time error occurs and this entire package +// must be carefully evaluated for potential bugs. +var ( + _ [1]byte = escapeSet{}.k + _ [2]byte = escapeSet{}.esc +) + +var ( + measurementEscapeCodes = [...]escapeSet{ + {k: [1]byte{','}, esc: [2]byte{'\\', ','}}, + {k: [1]byte{' '}, esc: [2]byte{'\\', ' '}}, + } + + tagEscapeCodes = [...]escapeSet{ + {k: [1]byte{','}, esc: [2]byte{'\\', ','}}, + {k: [1]byte{' '}, esc: [2]byte{'\\', ' '}}, + {k: [1]byte{'='}, esc: [2]byte{'\\', '='}}, + } + + // ErrMeasurementTagExpected is returned by ParseMeasurement when parsing a + // series key where the first tag key is not a measurement. + ErrMeasurementTagExpected = errors.New("measurement tag expected") + + // ErrInvalidKey is returned by ParseMeasurement when parsing an empty + // or invalid series key. + ErrInvalidKey = errors.New("invalid key") +) + +// scanTo returns the end position in buf and the next consecutive block +// of bytes, starting from i and ending with stop byte, where stop byte +// has not been escaped. +// +// If there are leading spaces, they are skipped. +func scanTo(buf Escaped, i int, stop byte) (int, Escaped) { + start := i + for { + // reached the end of buf? + if i >= len(buf.B) { + break + } + + // Reached unescaped stop value? + if buf.B[i] == stop && (i == 0 || buf.B[i-1] != '\\') { + break + } + i++ + } + + return i, buf.Slice(start, i) +} + +func walkTags(buf Escaped, fn func(key, value []byte) bool) { + if len(buf.B) == 0 { + return + } + + pos, name := scanTo(buf, 0, ',') + + // it's an empty key, so there are no tags + if len(name.B) == 0 { + return + } + + hasEscape := bytes.IndexByte(buf.B, '\\') != -1 + i := pos + 1 + var key, value Escaped + for { + if i >= len(buf.B) { + break + } + i, key = scanTo(buf, i, '=') + i, value = scanTagValue(buf, i+1) + + if len(value.B) == 0 { + continue + } + + if hasEscape { + if !fn(UnescapeToken(key), UnescapeToken(value)) { + return + } + } else { + if !fn(key.B, value.B) { + return + } + } + + i++ + } +} + +func scanTagValue(buf Escaped, i int) (int, Escaped) { + start := i + for { + if i >= len(buf.B) { + break + } + + if buf.B[i] == ',' && buf.B[i-1] != '\\' { + break + } + i++ + } + if i > len(buf.B) { + return i, Escaped{} + } + return i, buf.Slice(start, i) +} + +// parseTags parses buf into the provided destination tags, returning destination +// Tags, which may have a different length and capacity. +func parseTags(buf Escaped, dst Tags) Tags { + if len(buf.B) == 0 { + return nil + } + + n := bytes.Count(buf.B, []byte(",")) + if cap(dst) < n { + dst = make(Tags, n) + } else { + dst = dst[:n] + } + + // Ensure existing behaviour when point has no tags and nil slice passed in. + if dst == nil { + dst = Tags{} + } + + // Series keys can contain escaped commas, therefore the number of commas + // in a series key only gives an estimation of the upper bound on the number + // of tags. + var i int + walkTags(buf, func(key, value []byte) bool { + dst[i].Key, dst[i].Value = key, value + i++ + return true + }) + return dst[:i] +} + +// Escaped represents an escaped portion of line-protocol syntax +// using an underlying byte slice. It's defined as a struct to avoid +// unintentional implicit conversion between []byte and Escaped. +// +// In general you should avoid using the B field unless necessary, +// as doing so risks breaking the typed assertion that the data is +// validly escaped syntax. +type Escaped struct{ B []byte } + +// MakeEscaped returns the equivalent of Escaped{B: b}. +// Only use this when you're sure that b is either already +// escaped or (in tests) when b has no characters that need escaping. +func MakeEscaped(b []byte) Escaped { + return Escaped{b} +} + +// S returns the EscapedString form of m. +func (e Escaped) S() EscapedString { + return EscapedString(e.B) +} + +// Clone makes a copy of e and its underlying byte slice. +func (e Escaped) Clone() Escaped { + return Escaped{B: cloneBytes(e.B)} +} + +// Slice returns a slice of the underlying data from i0 to i1. +func (e Escaped) Slice(i0, i1 int) Escaped { + return Escaped{e.B[i0:i1]} +} + +// SliceToEnd returns a slice of the underlying data from i0 to the end of the slice. +func (e Escaped) SliceToEnd(i0 int) Escaped { + return e.Slice(i0, len(e.B)) +} + +// EscapedString represents an escaped portion of line-protocol syntax. +// See Escaped for more information. +type EscapedString string + +// B returns the byte-slice form of e. +func (e EscapedString) B() Escaped { + return Escaped{[]byte(e)} +} + +func cloneBytes(b []byte) []byte { + return append([]byte(nil), b...) +} + +func UnescapeToken(in Escaped) []byte { + out := in.B + if bytes.IndexByte(out, '\\') == -1 { + return out + } + + for i := range tagEscapeCodes { + c := &tagEscapeCodes[i] + if bytes.IndexByte(out, c.k[0]) != -1 { + out = bytes.Replace(out, c.esc[:], c.k[:], -1) + } + } + return out +} + +func UnescapeMeasurement(inm Escaped) []byte { + in := inm.B + if bytes.IndexByte(in, '\\') == -1 { + return in + } + + for i := range measurementEscapeCodes { + c := &measurementEscapeCodes[i] + if bytes.IndexByte(in, c.k[0]) != -1 { + in = bytes.Replace(in, c.esc[:], c.k[:], -1) + } + } + return in +} + +// The following constants allow us to specify which state to move to +// next, when scanning sections of a Point. +const ( + tagKeyState = iota + tagValueState + fieldsState +) + +// scanMeasurement examines the measurement part of a Point, returning +// the next state to move to, and the current location in the buffer. +func scanMeasurement(bufe Escaped, i int) (int, int, error) { + buf := bufe.B + // Check first byte of measurement, anything except a comma is fine. + // It can't be a space, since whitespace is stripped prior to this + // function call. + if i >= len(buf) || buf[i] == ',' { + return -1, i, fmt.Errorf("missing measurement") + } + + for { + i++ + if i >= len(buf) { + // cpu + return -1, i, fmt.Errorf("missing fields") + } + + if buf[i-1] == '\\' { + // Skip character (it's escaped). + continue + } + + // Unescaped comma; move onto scanning the tags. + if buf[i] == ',' { + return tagKeyState, i + 1, nil + } + + // Unescaped space; move onto scanning the fields. + if buf[i] == ' ' { + // cpu value=1.0 + return fieldsState, i, nil + } + } +} + +func ParseTags(buf Escaped) Tags { + return parseTags(buf, nil) +} + +func ParseKeyBytesWithTags(buf Escaped, tags Tags) ([]byte, Tags) { + // Ignore the error because scanMeasurement returns "missing fields" which we ignore + // when just parsing a key + state, i, _ := scanMeasurement(buf, 0) + + var name Escaped + if state == tagKeyState { + tags = parseTags(buf, tags) + // scanMeasurement returns the location of the comma if there are tags, strip that off + name.B = buf.B[:i-1] + } else { + name.B = buf.B[:i] + } + return UnescapeMeasurement(name), tags +} + +// SeriesAndField parses splits the key at the last tag, which is the reserved +// \xff tag key. +func SeriesAndField(key Escaped) (series, field Escaped, found bool) { + sep := bytes.LastIndex(key.B, fieldKeyTagKeySepBytes) + if sep == -1 { + // No field??? + return key, Escaped{}, false + } + return key.Slice(0, sep), key.SliceToEnd(sep + len(fieldKeyTagKeySepBytes)), true +} + +// ParseMeasurement returns the value of the tag identified by MeasurementTagKey; otherwise, +// an error is returned. +// +// buf must be a normalized series key, such that the tags are +// lexicographically sorted and therefore the measurement tag is first. +func ParseMeasurement(buf Escaped) ([]byte, error) { + pos, name := scanTo(buf, 0, ',') + + // it's an empty key, so there are no tags + if len(name.B) == 0 { + return nil, ErrInvalidKey + } + + i := pos + 1 + var key, value Escaped + i, key = scanTo(buf, i, '=') + if key.S() != MeasurementTagKey { + return nil, ErrMeasurementTagExpected + } + + _, value = scanTagValue(buf, i+1) + if bytes.IndexByte(value.B, '\\') != -1 { + // hasEscape + return UnescapeToken(value), nil + } + return value.B, nil +} + +// Tag represents a single key/value tag pair. +type Tag struct { + Key []byte + Value []byte +} + +// NewTag returns a new Tag. +func NewTag(key, value []byte) Tag { + return Tag{ + Key: key, + Value: value, + } +} + +// Size returns the size of the key and value. +func (t Tag) Size() int { return len(t.Key) + len(t.Value) } + +// Clone returns a deep copy of Tag. +// +// Tags associated with a Point created by ParsePointsWithPrecision will hold references to the byte slice that was parsed. +// Use Clone to create a Tag with new byte slices that do not refer to the argument to ParsePointsWithPrecision. +func (t Tag) Clone() Tag { + return Tag{ + Key: cloneBytes(t.Key), + Value: cloneBytes(t.Value), + } +} + +// String returns the string representation of the tag. +func (t *Tag) String() string { + var buf bytes.Buffer + buf.WriteByte('{') + buf.WriteString(string(t.Key)) + buf.WriteByte(' ') + buf.WriteString(string(t.Value)) + buf.WriteByte('}') + return buf.String() +} + +// Tags represents a sorted list of tags. +type Tags []Tag diff --git a/cmd/influx_inspect/export/parquet/resultset/array.go b/cmd/influx_inspect/export/parquet/resultset/array.go new file mode 100644 index 00000000000..1b251194d00 --- /dev/null +++ b/cmd/influx_inspect/export/parquet/resultset/array.go @@ -0,0 +1,91 @@ +package resultset + +import ( + "slices" +) + +// BlockType is the set of Go data types that can be represented by TSM. +type BlockType interface { + int64 | float64 | string | uint64 | bool +} + +// TimeArray is a generic type that represents a block of TSM data as a +// collection of timestamps and values. The timestamps are expected +// to be in ascending order. +// +// # NOTE +// +// TimeArray data is materialised from TSM blocks, which are always +// sorted by timestamp in ascending order. +type TimeArray[T BlockType] struct { + Timestamps []int64 + Values []T +} + +// NewTimeArrayLen creates a new TimeArray with the specified length. +func NewTimeArrayLen[T BlockType](sz int) *TimeArray[T] { + return &TimeArray[T]{ + Timestamps: make([]int64, sz), + Values: make([]T, sz), + } +} + +// MinTime returns the timestamp of the first element. +func (a *TimeArray[T]) MinTime() int64 { + return a.Timestamps[0] +} + +// MaxTime returns the timestamp of the last element. +func (a *TimeArray[T]) MaxTime() int64 { + return a.Timestamps[len(a.Timestamps)-1] +} + +// Len returns the number of elements of the receiver. +func (a *TimeArray[T]) Len() int { + if a == nil { + return 0 + } + return len(a.Timestamps) +} + +// Clear sets the length of the receiver to zero. +func (a *TimeArray[T]) Clear() { + a.Timestamps = a.Timestamps[:0] + a.Values = a.Values[:0] +} + +// Exclude removes the subset of values in [min, max]. The values must +// be deduplicated and sorted before calling Exclude or the results are undefined. +func (a *TimeArray[T]) Exclude(min, max int64) { + if min > max { + // bad range + return + } + + if a.MaxTime() < min || a.MinTime() > max { + // the array is completely outside the range to be excluded + return + } + + rmin, _ := slices.BinarySearch(a.Timestamps, min) + rmax, found := slices.BinarySearch(a.Timestamps, max) + if rmax < a.Len() { + if found { + rmax++ + } + rest := a.Len() - rmax + if rest > 0 { + ts := a.Timestamps[:rmin+rest] + copy(ts[rmin:], a.Timestamps[rmax:]) + a.Timestamps = ts + + vs := a.Values[:rmin+rest] + copy(vs[rmin:], a.Values[rmax:]) + a.Values = vs + return + } + } + + a.Timestamps = a.Timestamps[:rmin] + a.Values = a.Values[:rmin] +} diff --git a/cmd/influx_inspect/export/parquet/resultset/result_set.go b/cmd/influx_inspect/export/parquet/resultset/result_set.go new file mode 100644 index 00000000000..2ed98872dd4 --- /dev/null +++ b/cmd/influx_inspect/export/parquet/resultset/result_set.go @@ -0,0 +1,50 @@ +package resultset + +import ( + "context" + + "github.com/influxdata/influxdb/cmd/influx_inspect/export/parquet/models" +) + +// ResultSet describes an time series iterator. +type ResultSet interface { + // Next advances the ResultSet and returns the next cursor. It returns nil + // when there are no more cursors. + Next(ctx context.Context) SeriesCursor + + // Close releases any resources allocated by the ResultSet. + Close() + + // Err returns the first error encountered by the ResultSet. + Err() error +} + +// SeriesCursor is an iterator for a single series key. +// +// A SeriesCursor is cast to a TypedCursor in order to +// enumerate the data for the SeriesKey. +type SeriesCursor interface { + // SeriesKey returns the series key for the receiver. + SeriesKey() models.Escaped + // Close releases any resources used by the receiver. + Close() + // Err returns the last error by the receiver. + Err() error +} + +// A TypedCursor provides access to the time series data for +// a SeriesCursor. +type TypedCursor[T BlockType] interface { + SeriesCursor + // Next returns the next block of data for the receiver. + // When TimeArray.Len equals 0, there is no more data. + Next() *TimeArray[T] +} + +type ( + FloatCursor TypedCursor[float64] + IntegerCursor TypedCursor[int64] + UnsignedCursor TypedCursor[uint64] + StringCursor TypedCursor[string] + BooleanCursor TypedCursor[bool] +) diff --git a/cmd/influx_inspect/export/parquet/table/allocator.go b/cmd/influx_inspect/export/parquet/table/allocator.go new file mode 100644 index 00000000000..ce06c24ac34 --- /dev/null +++ b/cmd/influx_inspect/export/parquet/table/allocator.go @@ -0,0 +1,31 @@ +package table + +import ( + "sync/atomic" + + "github.com/apache/arrow/go/v14/arrow/memory" +) + +// trackedAllocator is a memory.Allocator that tracks the total allocation size. +type trackedAllocator struct { + mem memory.GoAllocator + sz int64 +} + +// CurrentAlloc returns the current allocation size in bytes. +func (a *trackedAllocator) CurrentAlloc() int { return int(atomic.LoadInt64(&a.sz)) } + +func (a *trackedAllocator) Allocate(size int) []byte { + atomic.AddInt64(&a.sz, int64(size)) + return a.mem.Allocate(size) +} + +func (a *trackedAllocator) Reallocate(size int, b []byte) []byte { + atomic.AddInt64(&a.sz, int64(size-len(b))) + return a.mem.Reallocate(size, b) +} + +func (a *trackedAllocator) Free(b []byte) { + atomic.AddInt64(&a.sz, int64(len(b)*-1)) + a.mem.Free(b) +} diff --git a/cmd/influx_inspect/export/parquet/table/column.go b/cmd/influx_inspect/export/parquet/table/column.go new file mode 100644 index 00000000000..f0849302e79 --- /dev/null +++ b/cmd/influx_inspect/export/parquet/table/column.go @@ -0,0 +1,206 @@ +package table + +import ( + "fmt" + + "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v14/arrow/array" + "github.com/apache/arrow/go/v14/arrow/memory" + "github.com/influxdata/influxdb/cmd/influx_inspect/export/parquet/resultset" + "github.com/influxdata/influxdb/cmd/influx_inspect/export/parquet/tsm1" +) + +// SemanticType defines the possible semantic types of a column. +type SemanticType byte + +const ( + // SemanticTypeTimestamp indicates the column is a timestamp. + SemanticTypeTimestamp SemanticType = iota + // SemanticTypeTag indicates the column is a tag. + SemanticTypeTag + // SemanticTypeField indicates the column is a field. + SemanticTypeField +) + +// Column provides name and data type information about a Table column. +type Column interface { + // Name is the name of the receiver. + Name() string + + // DataType is the data type of the receiver. + DataType() arrow.DataType + + // SemanticType returns the semantic type of the receiver. + SemanticType() SemanticType +} + +// FieldBuilder defines the behavior for building an arrow.Array by +// consuming the data produced by a resultset.SeriesCursor. +type FieldBuilder interface { + Column + + // SetCursor specifies the source of time series data for the receiver. + // SetCursor returns an error if the resultset.TypedCursor does not match + // the expected DataType or Reset has not been called after a previous call + // to SetCursor. + SetCursor(c resultset.SeriesCursor) error + + // NewArray returns an array of the current values for the receiver. + NewArray() arrow.Array + + // PeekTimestamp returns the next timestamp of the receiver and a boolean + // indicating if the timestamp is valid. + PeekTimestamp() (int64, bool) + + // Append appends the value of the field, for the specified timestamp, + // to the array or null if there is no value. + Append(ts int64) + + // Err returns the error if the receiver failed to consume data from + // the resultset.SeriesCursor. + Err() error + + // Reset releases and resources from a previous call to SetCursor, and + // prepares the FieldBuilder for a new resultset.SeriesCursor. + Reset() +} + +// arrayBuilder describes the common behavior required to build an arrow.Array. +type arrayBuilder[T resultset.BlockType] interface { + AppendNull() + Append(T) + NewArray() arrow.Array + Type() arrow.DataType +} + +// typedField is a generic implementation of the FieldBuilder interface for +// all supported TSM block types. +type typedField[T resultset.BlockType] struct { + name string + cur resultset.TypedCursor[T] + bui arrayBuilder[T] + a *resultset.TimeArray[T] + i int + err error +} + +// newField constructs a new FieldBuilder with the specified column name and +// block type. +func newField(mem memory.Allocator, name string, bt tsm1.BlockType) FieldBuilder { + var col FieldBuilder + switch bt { + case tsm1.BlockFloat64: + col = &typedField[float64]{ + name: name, + bui: array.NewFloat64Builder(mem), + } + case tsm1.BlockInteger: + col = &typedField[int64]{ + name: name, + bui: array.NewInt64Builder(mem), + } + case tsm1.BlockUnsigned: + col = &typedField[uint64]{ + name: name, + bui: array.NewUint64Builder(mem), + } + case tsm1.BlockString: + col = &typedField[string]{ + name: name, + bui: array.NewStringBuilder(mem), + } + case tsm1.BlockBoolean: + col = &typedField[bool]{ + name: name, + bui: array.NewBooleanBuilder(mem), + } + default: + panic("unreachable") + } + return col +} + +func (c *typedField[T]) Name() string { return c.name } +func (c *typedField[T]) DataType() arrow.DataType { return c.bui.Type() } +func (c *typedField[T]) SemanticType() SemanticType { return SemanticTypeField } + +func (c *typedField[T]) SetCursor(cur resultset.SeriesCursor) error { + if c.err != nil { + return c.err + } + + tc, ok := cur.(resultset.TypedCursor[T]) + if !ok { + return fmt.Errorf("field %s: unexpected cursor type: %T", c.name, cur) + } + if c.cur != nil { + // Reset is expected + return fmt.Errorf("field %s: SetCursor called with previous cursor", c.name) + } + c.cur = tc + c.load() + return nil +} + +func (c *typedField[T]) NewArray() arrow.Array { + return c.bui.NewArray() +} + +func (c *typedField[T]) load() { + c.a = c.cur.Next() + if c.a == nil { + c.Reset() + } else { + c.i = 0 + } +} + +func (c *typedField[T]) PeekTimestamp() (int64, bool) { + if c.i >= c.a.Len() { + return 0, false + } + + return c.a.Timestamps[c.i], true +} + +func (c *typedField[T]) Append(ts int64) { + if c.i < c.a.Len() && c.a.Timestamps[c.i] == ts { + v := c.a.Values[c.i] + c.bui.Append(v) + c.i++ + if c.i >= len(c.a.Timestamps) { + c.load() + } + } else { + c.bui.AppendNull() + } +} + +func (c *typedField[T]) Err() error { return c.err } + +func (c *typedField[T]) Reset() { + if c.cur != nil { + c.cur.Close() + c.err = c.cur.Err() + c.cur = nil + } +} + +// tagColumn contains state for building an arrow.Array for a tag column. +type tagColumn struct { + name string + b *array.StringBuilder +} + +func (t *tagColumn) Name() string { return t.name } +func (t *tagColumn) DataType() arrow.DataType { return t.b.Type() } +func (t *tagColumn) SemanticType() SemanticType { return SemanticTypeTag } + +// timestampColumn contains state for building an arrow.Array for a timestamp column. +type timestampColumn struct { + b *array.TimestampBuilder +} + +func (t *timestampColumn) Name() string { return "time" } +func (t *timestampColumn) DataType() arrow.DataType { return t.b.Type() } +func (t *timestampColumn) SemanticType() SemanticType { return SemanticTypeTimestamp } diff --git a/cmd/influx_inspect/export/parquet/table/resource_limits.go b/cmd/influx_inspect/export/parquet/table/resource_limits.go new file mode 100644 index 00000000000..c0ed89027b9 --- /dev/null +++ b/cmd/influx_inspect/export/parquet/table/resource_limits.go @@ -0,0 +1,59 @@ +package table + +// resourceUsage tracks the resource usage of a table during a call to Table.Next. +type resourceUsage struct { + // rowCount is the number of rows read. + rowCount uint + // groupCount is the number of group keys read. + groupCount uint + // allocated is the number of bytes allocated. + allocated uint +} + +// ResourceLimit is a limit on the amount of resources a table can use. +type ResourceLimit interface { + // isBelow returns true if the resource usage is below the limit + // defined by the receiver. + isBelow(s *resourceUsage) bool +} + +// resourceLimitBytes is a limit on the number of bytes allocated. +type resourceLimitBytes uint + +func (r resourceLimitBytes) isBelow(s *resourceUsage) bool { + return s.allocated < uint(r) +} + +// resourceLimitRows is a limit on the number of rows read. +type resourceLimitRows uint + +func (r resourceLimitRows) isBelow(s *resourceUsage) bool { + return s.rowCount < uint(r) +} + +// resourceLimitGroups is a limit on the number of group keys read. +type resourceLimitGroups uint + +func (r resourceLimitGroups) isBelow(s *resourceUsage) bool { + return s.groupCount < uint(r) +} + +type resourceLimitNone struct{} + +var resourceLimitNoneSingleton ResourceLimit = resourceLimitNone{} + +func (r resourceLimitNone) isBelow(s *resourceUsage) bool { + return true +} + +// ResourceLimitBytes returns a ResourceLimit that limits the number of bytes allocated per call to Table.Next. +func ResourceLimitBytes(n uint) ResourceLimit { return resourceLimitBytes(n) } + +// ResourceLimitRows returns a ResourceLimit that limits the number of rows read per call to Table.Next. +func ResourceLimitRows(n uint) ResourceLimit { return resourceLimitRows(n) } + +// ResourceLimitGroups returns a ResourceLimit that limits the number of group keys read per call to Table.Next. +func ResourceLimitGroups(n uint) ResourceLimit { return resourceLimitGroups(n) } + +// ResourceLimitNone returns a ResourceLimit that does not limit the amount of resources used. +func ResourceLimitNone() ResourceLimit { return resourceLimitNoneSingleton } diff --git a/cmd/influx_inspect/export/parquet/table/table.go b/cmd/influx_inspect/export/parquet/table/table.go new file mode 100644 index 00000000000..c234eafbd13 --- /dev/null +++ b/cmd/influx_inspect/export/parquet/table/table.go @@ -0,0 +1,379 @@ +package table + +import ( + "context" + "fmt" + "math" + "slices" + "sort" + "strings" + "sync/atomic" + + "github.com/apache/arrow/go/v14/arrow" + "github.com/apache/arrow/go/v14/arrow/array" + "github.com/apache/arrow/go/v14/arrow/memory" + "github.com/influxdata/influxdb/cmd/influx_inspect/export/parquet/index" + "github.com/influxdata/influxdb/cmd/influx_inspect/export/parquet/models" + "github.com/influxdata/influxdb/cmd/influx_inspect/export/parquet/resultset" + "go.uber.org/multierr" + "golang.org/x/exp/maps" +) + +// ReadStatistics tracks statistics after each call to Table.Next. +type ReadStatistics struct { + // RowCount is the total number of rows read. + RowCount int64 + // SeriesCount is the total number of series read. + SeriesCount int64 +} + +type Table struct { + mem *trackedAllocator + rs resultset.ResultSet + err error + // columns is the list of Columns defined by the schema. + columns []Column + // lastGroupKey + lastGroupKey models.Escaped + // timestamps is the builder for the timestamp columns + timestamps timestampColumn + // tags are the builders for the tag columns. The slice is sorted by tagColumn.name. + tags []tagColumn + // fields are the builders for the fields. The slice is sorted by FieldBuilder.Name. + fields []FieldBuilder + // cursor holds the cursor for the next group key + cursor resultset.SeriesCursor + // limit controls the amount of data read for each call to Next. + limit ResourceLimit + // buf contains various scratch buffers + buf struct { + tags models.Tags + arrays []arrow.Array + fieldCursors []fieldCursor + } + stats ReadStatistics + // loadNextGroupKey is true if the last group key has been fully read. + loadNextGroupKey bool + // done is true when the result set has been fully read or an error has occurred. + done bool +} + +type options struct { + limit ResourceLimit + concurrency int +} + +// OptionFn defines the function signature for options to configure a new Table. +type OptionFn func(*options) + +// WithResourceLimit sets the resource limit for the Table when calling Table.Next. +func WithResourceLimit(limit ResourceLimit) OptionFn { + return func(o *options) { + o.limit = limit + } +} + +// WithConcurrency sets the number of goroutines to use when reading the result set. +func WithConcurrency(concurrency int) OptionFn { + return func(o *options) { + o.concurrency = concurrency + } +} + +// New returns a new Table for the specified resultset.ResultSet. +func New(rs resultset.ResultSet, schema *index.MeasurementSchema, opts ...OptionFn) (*Table, error) { + o := options{limit: ResourceLimitGroups(1)} + for _, opt := range opts { + opt(&o) + } + mem := &trackedAllocator{mem: memory.GoAllocator{}} + + tagNames := maps.Keys(schema.TagSet) + slices.Sort(tagNames) + + var tags []tagColumn + for _, name := range tagNames { + tags = append(tags, tagColumn{name: name, b: array.NewStringBuilder(mem)}) + } + + fieldTypes := maps.Keys(schema.FieldSet) + + // check for fields conflicts + seenFields := map[string]struct{}{} + for i := range fieldTypes { + f := &fieldTypes[i] + if _, ok := seenFields[f.Name]; ok { + return nil, fmt.Errorf("field conflict: field %s has multiple data types", f.Name) + } + } + + // fields are sorted in lexicographically ascending order + slices.SortFunc(fieldTypes, func(a, b index.MeasurementField) int { return strings.Compare(a.Name, b.Name) }) + + var fields []FieldBuilder + for i := range fieldTypes { + f := &fieldTypes[i] + fields = append(fields, newField(mem, f.Name, f.Type)) + } + + columnCount := 1 + // timestamp column + len(tags) + len(fields) + + res := &Table{ + mem: mem, + rs: rs, + columns: make([]Column, 0, columnCount), + timestamps: timestampColumn{b: array.NewTimestampBuilder(mem, &arrow.TimestampType{Unit: arrow.Nanosecond})}, + tags: tags, + fields: fields, + limit: o.limit, + buf: struct { + tags models.Tags + arrays []arrow.Array + fieldCursors []fieldCursor + }{ + arrays: make([]arrow.Array, columnCount), + fieldCursors: make([]fieldCursor, len(fields)), + }, + loadNextGroupKey: true, + } + + res.columns = append(res.columns, &res.timestamps) + for i := range tags { + res.columns = append(res.columns, &tags[i]) + } + for _, f := range fields { + res.columns = append(res.columns, f) + } + + return res, nil +} + +// Columns returns the list of columns for the table. +// Inspecting the Column.SemanticType will indicate the semantic types. +// +// The first element is always the SemanticTypeTimestamp column. +// The next 0-n are SemanticTypeTag, and the remaining are SemanticTypeField. +func (t *Table) Columns() []Column { return t.columns } + +// Err returns the last recorded error for the receiver. +func (t *Table) Err() error { return t.err } + +func (t *Table) findField(name []byte) (field FieldBuilder) { + slices.BinarySearchFunc(t.fields, name, func(c FieldBuilder, bytes []byte) int { + if c.Name() < string(bytes) { + return -1 + } else if c.Name() > string(bytes) { + return 1 + } else { + field = c + return 0 + } + }) + + return +} + +func (t *Table) loadNextKey(ctx context.Context) models.Escaped { + var next resultset.SeriesCursor + if t.cursor != nil { + next = t.cursor + t.cursor = nil + } else { + next = t.rs.Next(ctx) + } + + if next == nil { + t.rs.Close() + t.err = t.rs.Err() + t.done = true + return models.Escaped{} + } + + groupKey, fieldEsc, _ := models.SeriesAndField(next.SeriesKey()) + field := models.UnescapeToken(fieldEsc) + fieldCursors := t.buf.fieldCursors[:0] + if f := t.findField(field); f != nil { + fieldCursors = append(fieldCursors, fieldCursor{ + field: f, + cursor: next, + }) + } + + // find remaining fields for the current group key, + for { + cur := t.rs.Next(ctx) + if cur == nil { + break + } + + nextGroup, fieldEsc, _ := models.SeriesAndField(cur.SeriesKey()) + if string(groupKey.B) != string(nextGroup.B) { + // save this cursor, which is the next group key + t.cursor = cur + break + } + + field := models.UnescapeToken(fieldEsc) + if f := t.findField(field); f != nil { + fieldCursors = append(fieldCursors, fieldCursor{ + field: f, + cursor: cur, + }) + } + } + + for i := 0; i < len(fieldCursors); i++ { + // copy the fieldCursor + fc := fieldCursors[i] + // and ensure the fields of the struct are set to nil, so they can be garbage collected + fieldCursors[i] = fieldCursor{} + if err := fc.field.SetCursor(fc.cursor); err != nil { + t.err = err + return models.Escaped{} + } + } + + return groupKey +} + +// Stats returns the read statistics for the table, accumulated after +// each call to Next. +// It is safe to call Stats concurrently. +func (t *Table) Stats() ReadStatistics { + var stats ReadStatistics + stats.RowCount = atomic.LoadInt64(&t.stats.RowCount) + stats.SeriesCount = atomic.LoadInt64(&t.stats.SeriesCount) + return stats +} + +// ReadInfo contains additional information about the batch of data returned by Table.Next. +type ReadInfo struct { + // FirstGroupKey is the first group key read by the batch. + FirstGroupKey models.Escaped + // LastGroupKey is the last group key read by the batch. + LastGroupKey models.Escaped +} + +// Next returns the next batch of data and information about the batch. +func (t *Table) Next(ctx context.Context) (columns []arrow.Array, readInfo ReadInfo) { + if t.done { + return + } + + baseAllocated := t.mem.CurrentAlloc() + + var s resourceUsage + defer func() { + atomic.AddInt64(&t.stats.RowCount, int64(s.rowCount)) + readInfo.LastGroupKey = t.lastGroupKey + }() + + for t.limit.isBelow(&s) { + var groupKey models.Escaped + if t.loadNextGroupKey { + groupKey = t.loadNextKey(ctx) + if len(groupKey.B) == 0 { + // signals the end of the result set + if s.rowCount > 0 { + // return the remaining buffered data + break + } + return + } + atomic.AddInt64(&t.stats.SeriesCount, 1) + t.lastGroupKey = groupKey + t.loadNextGroupKey = false + } else { + // continue reading current group key + groupKey = t.lastGroupKey + } + + if len(readInfo.FirstGroupKey.B) == 0 { + readInfo.FirstGroupKey = groupKey + } + + baseRowCount := s.rowCount + // this loop reads the data for the current group key. + for t.limit.isBelow(&s) { + // find the next minimum timestamp from all the fields + var minTs int64 = math.MaxInt64 + for _, col := range t.fields { + if v, ok := col.PeekTimestamp(); ok && v < minTs { + minTs = v + } + } + + if minTs == math.MaxInt64 { + t.loadNextGroupKey = true + break + } + + t.timestamps.b.Append(arrow.Timestamp(minTs)) + + for _, col := range t.fields { + col.Append(minTs) + } + s.rowCount++ + s.allocated = uint(t.mem.CurrentAlloc() - baseAllocated) + } + + s.groupCount++ + + if t.loadNextGroupKey { + // Reset the fields for the next group key + for _, col := range t.fields { + col.Reset() + t.err = multierr.Append(t.err, col.Err()) + } + if t.err != nil { + t.done = true + return + } + } + + groupRowCount := int(s.rowCount - baseRowCount) + + _, t.buf.tags = models.ParseKeyBytesWithTags(groupKey, t.buf.tags) + tagSet := t.buf.tags[1:] // skip measurement, which is always the first tag key (\x00) + + // For each tag column, append the tag value for all rows of the current group; otherwise, + // append nulls if the tag does not exist in this group. + for i := range t.tags { + tc := &t.tags[i] + tc.b.Reserve(groupRowCount) + + // tagSet is sorted, so perform a binary search + idx := sort.Search(len(tagSet), func(i int) bool { + return string(tagSet[i].Key) >= tc.name + }) + if idx < len(tagSet) && string(tagSet[idx].Key) == tc.name { + value := tagSet[idx].Value + for j := 0; j < groupRowCount; j++ { + tc.b.BinaryBuilder.Append(value) + } + } else { + tc.b.BinaryBuilder.AppendNulls(groupRowCount) + } + } + } + + // build the final results + columns = t.buf.arrays[:0] + columns = append(columns, t.timestamps.b.NewArray()) + + for i := range t.tags { + columns = append(columns, t.tags[i].b.NewArray()) + } + + for _, b := range t.fields { + columns = append(columns, b.NewArray()) + } + + return +} + +type fieldCursor struct { + field FieldBuilder + cursor resultset.SeriesCursor +} diff --git a/cmd/influx_inspect/export/parquet/tsm1/encoding.go b/cmd/influx_inspect/export/parquet/tsm1/encoding.go new file mode 100644 index 00000000000..638b6b6b9b6 --- /dev/null +++ b/cmd/influx_inspect/export/parquet/tsm1/encoding.go @@ -0,0 +1,42 @@ +package tsm1 + +import "fmt" + +type BlockType byte + +const ( + // BlockFloat64 designates a block of float64 values. + BlockFloat64 = BlockType(0) + + // BlockInteger designates a block of int64 values. + BlockInteger = BlockType(1) + + // BlockBoolean designates a block of boolean values. + BlockBoolean = BlockType(2) + + // BlockString designates a block of string values. + BlockString = BlockType(3) + + // BlockUnsigned designates a block of uint64 values. + BlockUnsigned = BlockType(4) + + // BlockUndefined represents an undefined block type value. + BlockUndefined = BlockUnsigned + 1 +) + +func BlockTypeForType(v interface{}) (BlockType, error) { + switch v.(type) { + case int64: + return BlockInteger, nil + case uint64: + return BlockUnsigned, nil + case float64: + return BlockFloat64, nil + case bool: + return BlockBoolean, nil + case string: + return BlockString, nil + default: + return BlockUndefined, fmt.Errorf("unknown type: %T", v) + } +} diff --git a/go.mod b/go.mod index 5a140e93374..68740dc49cf 100644 --- a/go.mod +++ b/go.mod @@ -6,6 +6,7 @@ require ( collectd.org v0.3.0 github.com/BurntSushi/toml v0.3.1 github.com/apache/arrow/go/arrow v0.0.0-20211112161151-bc219186db40 + github.com/apache/arrow/go/v14 v14.0.2 github.com/benbjohnson/tmpl v1.0.0 github.com/bmizerany/pat v0.0.0-20170815010413-6226ea591a40 github.com/cespare/xxhash v1.1.0 @@ -22,10 +23,11 @@ require ( github.com/influxdata/pkg-config v0.2.11 github.com/influxdata/roaring v0.4.13-0.20180809181101-fc520f41fab6 github.com/influxdata/usage-client v0.0.0-20160829180054-6d3895376368 + github.com/json-iterator/go v1.1.11 github.com/jsternberg/zap-logfmt v1.2.0 github.com/jwilder/encoding v0.0.0-20170811194829-b4e1701a28ef github.com/klauspost/pgzip v1.0.2-0.20170402124221-0bf5dcad4ada - github.com/mattn/go-isatty v0.0.16 + github.com/mattn/go-isatty v0.0.19 github.com/mileusna/useragent v0.0.0-20190129205925-3e331f0949a5 github.com/opentracing/opentracing-go v1.2.0 github.com/peterh/liner v1.0.1-0.20180619022028-8c1271fcf47f @@ -37,7 +39,7 @@ require ( github.com/retailnext/hllpp v1.0.1-0.20180308014038-101a6d2f8b52 github.com/spf13/cast v1.3.0 github.com/spf13/cobra v0.0.3 - github.com/stretchr/testify v1.8.1 + github.com/stretchr/testify v1.8.4 github.com/tinylib/msgp v1.1.0 github.com/uber/jaeger-client-go v2.28.0+incompatible github.com/xlab/treeprint v0.0.0-20180616005107-d6fb6747feb6 @@ -51,18 +53,18 @@ require ( golang.org/x/text v0.14.0 golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba golang.org/x/tools v0.16.0 - google.golang.org/grpc v1.56.3 + google.golang.org/grpc v1.58.2 google.golang.org/protobuf v1.33.0 ) require ( - cloud.google.com/go v0.110.0 // indirect - cloud.google.com/go/bigquery v1.50.0 // indirect + cloud.google.com/go v0.110.8 // indirect + cloud.google.com/go/bigquery v1.55.0 // indirect cloud.google.com/go/bigtable v1.10.1 // indirect - cloud.google.com/go/compute v1.19.1 // indirect + cloud.google.com/go/compute v1.23.0 // indirect cloud.google.com/go/compute/metadata v0.2.3 // indirect - cloud.google.com/go/iam v0.13.0 // indirect - cloud.google.com/go/longrunning v0.4.1 // indirect + cloud.google.com/go/iam v1.1.2 // indirect + cloud.google.com/go/longrunning v0.5.1 // indirect github.com/Azure/azure-pipeline-go v0.2.3 // indirect github.com/Azure/azure-storage-blob-go v0.14.0 // indirect github.com/Azure/go-autorest v14.2.0+incompatible // indirect @@ -74,15 +76,16 @@ require ( github.com/Azure/go-autorest/logger v0.2.1 // indirect github.com/Azure/go-autorest/tracing v0.6.0 // indirect github.com/DATA-DOG/go-sqlmock v1.4.1 // indirect + github.com/JohnCGriffin/overflow v0.0.0-20211019200055-46fa312c352c // indirect github.com/Masterminds/semver v1.4.2 // indirect github.com/Masterminds/sprig v2.16.0+incompatible // indirect github.com/SAP/go-hdb v0.14.1 // indirect github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883 // indirect - github.com/andybalholm/brotli v1.0.4 // indirect + github.com/andybalholm/brotli v1.0.5 // indirect github.com/aokoli/goutils v1.0.1 // indirect - github.com/apache/arrow/go/v11 v11.0.0 // indirect + github.com/apache/arrow/go/v12 v12.0.0 // indirect github.com/apache/arrow/go/v7 v7.0.1 // indirect - github.com/apache/thrift v0.16.0 // indirect + github.com/apache/thrift v0.17.0 // indirect github.com/aws/aws-sdk-go v1.34.0 // indirect github.com/aws/aws-sdk-go-v2 v1.11.0 // indirect github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.0.0 // indirect @@ -102,24 +105,25 @@ require ( github.com/deepmap/oapi-codegen v1.6.0 // indirect github.com/denisenkom/go-mssqldb v0.10.0 // indirect github.com/dimchansky/utfbom v1.1.0 // indirect - github.com/dustin/go-humanize v1.0.0 // indirect + github.com/dustin/go-humanize v1.0.1 // indirect github.com/eclipse/paho.mqtt.golang v1.2.0 // indirect - github.com/fatih/color v1.13.0 // indirect + github.com/fatih/color v1.15.0 // indirect github.com/form3tech-oss/jwt-go v3.2.5+incompatible // indirect github.com/gabriel-vasile/mimetype v1.4.0 // indirect github.com/glycerine/go-unsnap-stream v0.0.0-20180323001048-9f0cb55181dd // indirect github.com/go-sql-driver/mysql v1.5.0 // indirect - github.com/goccy/go-json v0.9.11 // indirect + github.com/goccy/go-json v0.10.2 // indirect github.com/gofrs/uuid v3.3.0+incompatible // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe // indirect github.com/golang/geo v0.0.0-20190916061304-5b978397cfec // indirect - github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e // indirect + github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.4 // indirect - github.com/google/flatbuffers v22.9.30-0.20221019131441-5792623df42e+incompatible // indirect - github.com/google/uuid v1.3.0 // indirect - github.com/googleapis/enterprise-certificate-proxy v0.2.3 // indirect - github.com/googleapis/gax-go/v2 v2.7.1 // indirect + github.com/google/flatbuffers v23.5.26+incompatible // indirect + github.com/google/s2a-go v0.1.4 // indirect + github.com/google/uuid v1.3.1 // indirect + github.com/googleapis/enterprise-certificate-proxy v0.2.4 // indirect + github.com/googleapis/gax-go/v2 v2.12.0 // indirect github.com/grpc-ecosystem/grpc-gateway v1.16.0 // indirect github.com/huandu/xstrings v1.0.0 // indirect github.com/imdario/mergo v0.3.5 // indirect @@ -132,20 +136,22 @@ require ( github.com/influxdata/tdigest v0.0.2-0.20210216194612-fc98d27c9e8b // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/klauspost/asmfmt v1.3.2 // indirect - github.com/klauspost/compress v1.15.9 // indirect - github.com/klauspost/cpuid/v2 v2.0.9 // indirect + github.com/klauspost/compress v1.16.7 // indirect + github.com/klauspost/cpuid/v2 v2.2.5 // indirect github.com/klauspost/crc32 v0.0.0-20161016154125-cb6bfca970f6 // indirect github.com/lib/pq v1.0.0 // indirect - github.com/mattn/go-colorable v0.1.9 // indirect + github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-ieproxy v0.0.1 // indirect github.com/mattn/go-runewidth v0.0.3 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8 // indirect github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3 // indirect github.com/mitchellh/go-homedir v1.1.0 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.1 // indirect github.com/mschoch/smat v0.0.0-20160514031455-90eadee771ae // indirect github.com/philhofer/fwd v1.0.0 // indirect - github.com/pierrec/lz4/v4 v4.1.15 // indirect + github.com/pierrec/lz4/v4 v4.1.18 // indirect github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/prometheus/procfs v0.6.0 // indirect @@ -163,12 +169,14 @@ require ( go.uber.org/atomic v1.7.0 // indirect golang.org/x/mod v0.14.0 // indirect golang.org/x/net v0.23.0 // indirect - golang.org/x/oauth2 v0.7.0 // indirect + golang.org/x/oauth2 v0.10.0 // indirect golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect - gonum.org/v1/gonum v0.11.0 // indirect - google.golang.org/api v0.114.0 // indirect + gonum.org/v1/gonum v0.12.0 // indirect + google.golang.org/api v0.128.0 // indirect google.golang.org/appengine v1.6.7 // indirect - google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 // indirect + google.golang.org/genproto v0.0.0-20230920204549-e6e6cdab5c13 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20230913181813-007df8e322eb // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20231002182017-d307bd883b97 // indirect gopkg.in/yaml.v2 v2.3.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index cb6444b1bce..dfe9f0982d1 100644 --- a/go.sum +++ b/go.sum @@ -21,31 +21,31 @@ cloud.google.com/go v0.78.0/go.mod h1:QjdrLG0uq+YwhjoVOLsS1t7TW8fs36kLs4XO5R5ECH cloud.google.com/go v0.79.0/go.mod h1:3bzgcEeQlzbuEAYu4mrWhKqWjmpprinYgKJLgKHnbb8= cloud.google.com/go v0.81.0/go.mod h1:mk/AM35KwGk/Nm2YSeZbxXdrNK3KZOYHmLkOqC2V6E0= cloud.google.com/go v0.82.0/go.mod h1:vlKccHJGuFBFufnAnuB08dfEH9Y3H7dzDzRECFdC2TA= -cloud.google.com/go v0.110.0 h1:Zc8gqp3+a9/Eyph2KDmcGaPtbKRIoqq4YTlL4NMD0Ys= -cloud.google.com/go v0.110.0/go.mod h1:SJnCLqQ0FCFGSZMUNUf84MV3Aia54kn7pi8st7tMzaY= +cloud.google.com/go v0.110.8 h1:tyNdfIxjzaWctIiLYOTalaLKZ17SI44SKFW26QbOhME= +cloud.google.com/go v0.110.8/go.mod h1:Iz8AkXJf1qmxC3Oxoep8R1T36w8B92yU29PcBhHO5fk= cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= -cloud.google.com/go/bigquery v1.50.0 h1:RscMV6LbnAmhAzD893Lv9nXXy2WCaJmbxYPWDLbGqNQ= -cloud.google.com/go/bigquery v1.50.0/go.mod h1:YrleYEh2pSEbgTBZYMJ5SuSr0ML3ypjRB1zgf7pvQLU= +cloud.google.com/go/bigquery v1.55.0 h1:hs44Xxov3XLWQiCx2J8lK5U/ihLqnpm4RVVl5fdtLLI= +cloud.google.com/go/bigquery v1.55.0/go.mod h1:9Y5I3PN9kQWuid6183JFhOGOW3GcirA5LpsKCUn+2ec= cloud.google.com/go/bigtable v1.2.0/go.mod h1:JcVAOl45lrTmQfLj7T6TxyMzIN/3FGGcFm+2xVAli2o= cloud.google.com/go/bigtable v1.10.1 h1:QKcRHeAsraxIlrdCZ3LLobXKBvITqcOEnSbHG2rzL9g= cloud.google.com/go/bigtable v1.10.1/go.mod h1:cyHeKlx6dcZCO0oSQucYdauseD8kIENGuDOJPKMCVg8= -cloud.google.com/go/compute v1.19.1 h1:am86mquDUgjGNWxiGn+5PGLbmgiWXlE/yNWpIpNvuXY= -cloud.google.com/go/compute v1.19.1/go.mod h1:6ylj3a05WF8leseCdIf77NK0g1ey+nj5IKd5/kvShxE= +cloud.google.com/go/compute v1.23.0 h1:tP41Zoavr8ptEqaW6j+LQOnyBBhO7OkOMAGrgLopTwY= +cloud.google.com/go/compute v1.23.0/go.mod h1:4tCnrn48xsqlwSAiLf1HXMQk8CONslYbdiEZc9FEIbM= cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGBW5aJ7UnBMY= cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA= -cloud.google.com/go/datacatalog v1.13.0 h1:4H5IJiyUE0X6ShQBqgFFZvGGcrwGVndTwUSLP4c52gw= -cloud.google.com/go/datacatalog v1.13.0/go.mod h1:E4Rj9a5ZtAxcQJlEBTLgMTphfP11/lNaAshpoBgemX8= +cloud.google.com/go/datacatalog v1.17.1 h1:qGWrlYvWtK+8jD1jhwq5BsGoSr7S4/LOroV7LwXi00g= +cloud.google.com/go/datacatalog v1.17.1/go.mod h1:nCSYFHgtxh2MiEktWIz71s/X+7ds/UT9kp0PC7waCzE= cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= -cloud.google.com/go/iam v0.13.0 h1:+CmB+K0J/33d0zSQ9SlFWUeCCEn5XJA0ZMZ3pHE9u8k= -cloud.google.com/go/iam v0.13.0/go.mod h1:ljOg+rcNfzZ5d6f1nAUJ8ZIxOaZUVoS14bKCtaLZ/D0= -cloud.google.com/go/longrunning v0.4.1 h1:v+yFJOfKC3yZdY6ZUI933pIYdhyhV8S3NpWrXWmg7jM= -cloud.google.com/go/longrunning v0.4.1/go.mod h1:4iWDqhBZ70CvZ6BfETbvam3T8FMvLK+eFj0E6AaRQTo= +cloud.google.com/go/iam v1.1.2 h1:gacbrBdWcoVmGLozRuStX45YKvJtzIjJdAolzUs1sm4= +cloud.google.com/go/iam v1.1.2/go.mod h1:A5avdyVL2tCppe4unb0951eI9jreack+RJ0/d+KUZOU= +cloud.google.com/go/longrunning v0.5.1 h1:Fr7TXftcqTudoyRJa113hyaqlGdiBQkp0Gq7tErFDWI= +cloud.google.com/go/longrunning v0.5.1/go.mod h1:spvimkwdz6SPWKEt/XBij79E9fiTkHSQl/fRUUQJYJc= cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= @@ -55,8 +55,8 @@ cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0Zeo cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= -cloud.google.com/go/storage v1.29.0 h1:6weCgzRvMg7lzuUurI4697AqIRPU1SvzHhynwpW31jI= -cloud.google.com/go/storage v1.29.0/go.mod h1:4puEjyTKnku6gfKoTfNOU/W+a9JyuVNxjpS5GBrB8h4= +cloud.google.com/go/storage v1.30.1 h1:uOdMxAs8HExqBlnLtnQyP0YkvbiDpdGShGKtx6U/oNM= +cloud.google.com/go/storage v1.30.1/go.mod h1:NfxhC0UJE1aXSx7CIIbCf7y9HKT7BiccwkR7+P7gN8E= collectd.org v0.3.0 h1:iNBHGw1VvPJxH2B6RiFWFZ+vsjo1lCdRszBeOuwGi00= collectd.org v0.3.0/go.mod h1:A/8DzQBkF6abtvrT2j/AU/4tiBgJWYyh0y/oB/4MlWE= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= @@ -141,8 +141,8 @@ github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk5 github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883 h1:bvNMNQO63//z+xNgfBlViaCIJKLlCJ6/fmUseuG0wVQ= github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8= github.com/andybalholm/brotli v1.0.3/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= -github.com/andybalholm/brotli v1.0.4 h1:V7DdXeJtZscaqfNuAdSRuRFzuiKlHSC/Zh3zl9qY3JY= -github.com/andybalholm/brotli v1.0.4/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= +github.com/andybalholm/brotli v1.0.5 h1:8uQZIdzKmjc/iuPu7O2ioW48L81FgatrcpfFmiq/cCs= +github.com/andybalholm/brotli v1.0.5/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= github.com/antihax/optional v0.0.0-20180407024304-ca021399b1a6/go.mod h1:V8iCPQYkqmusNa815XgQio277wI47sdRh1dUOLdyC6Q= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= github.com/aokoli/goutils v1.0.1 h1:7fpzNGoJ3VA8qcrm++XEE1QUe0mIwNeLa02Nwq7RDkg= @@ -150,15 +150,17 @@ github.com/aokoli/goutils v1.0.1/go.mod h1:SijmP0QR8LtwsmDs8Yii5Z/S4trXFGFC2oO5g github.com/apache/arrow/go/arrow v0.0.0-20191024131854-af6fa24be0db/go.mod h1:VTxUBvSJ3s3eHAg65PNgrsn5BtqCRPdmyXh6rAfdxN0= github.com/apache/arrow/go/arrow v0.0.0-20211112161151-bc219186db40 h1:q4dksr6ICHXqG5hm0ZW5IHyeEJXoIJSOZeBLmWPNeIQ= github.com/apache/arrow/go/arrow v0.0.0-20211112161151-bc219186db40/go.mod h1:Q7yQnSMnLvcXlZ8RV+jwz/6y1rQTqbX6C82SndT52Zs= -github.com/apache/arrow/go/v11 v11.0.0 h1:hqauxvFQxww+0mEU/2XHG6LT7eZternCZq+A5Yly2uM= -github.com/apache/arrow/go/v11 v11.0.0/go.mod h1:Eg5OsL5H+e299f7u5ssuXsuHQVEGC4xei5aX110hRiI= +github.com/apache/arrow/go/v12 v12.0.0 h1:xtZE63VWl7qLdB0JObIXvvhGjoVNrQ9ciIHG2OK5cmc= +github.com/apache/arrow/go/v12 v12.0.0/go.mod h1:d+tV/eHZZ7Dz7RPrFKtPK02tpr+c9/PEd/zm8mDS9Vg= +github.com/apache/arrow/go/v14 v14.0.2 h1:N8OkaJEOfI3mEZt07BIkvo4sC6XDbL+48MBPWO5IONw= +github.com/apache/arrow/go/v14 v14.0.2/go.mod h1:u3fgh3EdgN/YQ8cVQRguVW3R+seMybFg8QBQ5LU+eBY= github.com/apache/arrow/go/v7 v7.0.1 h1:WpCfq+AQxvXaI6/KplHE27MPMFx5av0o5NbPCTAGfy4= github.com/apache/arrow/go/v7 v7.0.1/go.mod h1:JxDpochJbCVxqbX4G8i1jRqMrnTCQdf8pTccAfLD8Es= github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= github.com/apache/thrift v0.13.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= github.com/apache/thrift v0.15.0/go.mod h1:PHK3hniurgQaNMZYaCLEqXKsYK8upmhPbmdP2FXSqgU= -github.com/apache/thrift v0.16.0 h1:qEy6UW60iVOlUy+b9ZR0d5WzUWYGOo4HfopoyBaNmoY= -github.com/apache/thrift v0.16.0/go.mod h1:PHK3hniurgQaNMZYaCLEqXKsYK8upmhPbmdP2FXSqgU= +github.com/apache/thrift v0.17.0 h1:cMd2aj52n+8VoAtvSvLn4kDC3aZ6IAkBuqWQ2IDu7wo= +github.com/apache/thrift v0.17.0/go.mod h1:OLxhMRJxomX+1I/KUw03qoV3mMz16BwaKI+d4fPBx7Q= github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= github.com/armon/go-metrics v0.3.3/go.mod h1:4O98XIr/9W0sxpJ8UaYkvjk10Iff7SnFrb4QAOwNTFc= @@ -244,8 +246,11 @@ github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDk github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= +github.com/cncf/udpa/go v0.0.0-20210930031921-04548b0d99d4/go.mod h1:6pvJx4me5XPnfI9Z40ddWsdw2W/uZgQLFXToKeRcDiI= github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20210805033703-aa0b78936158/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8= github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI= github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= @@ -275,8 +280,9 @@ github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDD github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM= github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= -github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/eapache/go-resiliency v1.1.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs= github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU= github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I= @@ -298,8 +304,8 @@ github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7 github.com/evanphx/json-patch v4.2.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU= -github.com/fatih/color v1.13.0 h1:8LOYc1KYPPmyKMuN8QV2DNRWNbLo6LZ0iLs8+mlH53w= -github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= +github.com/fatih/color v1.15.0 h1:kOqh6YHBtK8aywxGerMG2Eq3H6Qgoqeo13Bk2Mv/nBs= +github.com/fatih/color v1.15.0/go.mod h1:0h5ZqXfHYED7Bhv2ZJamyIOUej9KtShiJESRwBDUSsw= github.com/fogleman/gg v1.2.1-0.20190220221249-0403632d5b90/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k= github.com/fogleman/gg v1.3.0/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k= github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= @@ -434,8 +440,8 @@ github.com/gobuffalo/packr/v2 v2.0.9/go.mod h1:emmyGweYTm6Kdper+iywB6YK5YzuKchGt github.com/gobuffalo/packr/v2 v2.2.0/go.mod h1:CaAwI0GPIAv+5wKLtv8Afwl+Cm78K/I/VCm/3ptBN+0= github.com/gobuffalo/syncx v0.0.0-20190224160051-33c29581e754/go.mod h1:HhnNqWY95UYwwW3uSASeV7vtgYkT2t16hJgV3AEPUpw= github.com/goccy/go-json v0.7.10/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= -github.com/goccy/go-json v0.9.11 h1:/pAaQDLHEoCq/5FFmSKBswWmK6H0e8g4159Kc/X/nqk= -github.com/goccy/go-json v0.9.11/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= +github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU= +github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= github.com/gofrs/uuid v3.3.0+incompatible h1:8K4tyRfvU1CYPgJsveYFQMhpFd/wXNM7iK6rR7UHz84= github.com/gofrs/uuid v3.3.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= github.com/gogo/googleapis v1.1.0/go.mod h1:gf4bu3Q80BeJ6H1S1vYPm8/ELATdvryBaNFGgqEef3s= @@ -458,8 +464,9 @@ github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfU github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e h1:1r7pUrabqp18hOBcwBwiTsbnFeTZHV9eER/QT5JVZxY= github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= @@ -501,8 +508,8 @@ github.com/google/btree v1.0.1 h1:gK4Kx5IaGY9CD5sPJ36FHiBJ6ZXl0kilRiiCj+jdYp4= github.com/google/btree v1.0.1/go.mod h1:xXMiIv4Fb/0kKde4SpL7qlzvu5cMJDRkFDxJfI9uaxA= github.com/google/flatbuffers v1.11.0/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= github.com/google/flatbuffers v2.0.0+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= -github.com/google/flatbuffers v22.9.30-0.20221019131441-5792623df42e+incompatible h1:Bqgl5d9t2UlT8pv9Oc/lkkI8yYk0jCwHkZKkHzbxEsc= -github.com/google/flatbuffers v22.9.30-0.20221019131441-5792623df42e+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= +github.com/google/flatbuffers v23.5.26+incompatible h1:M9dgRyhJemaM4Sw8+66GHBu8ioaQmyPLg1b8VwK5WJg= +github.com/google/flatbuffers v23.5.26+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= @@ -538,18 +545,21 @@ github.com/google/pprof v0.0.0-20210122040257-d980be63207e/go.mod h1:kpwsk12EmLe github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210506205249-923b5ab0fc1a/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= +github.com/google/s2a-go v0.1.4 h1:1kZ/sQM3srePvKs3tXAvQzo66XfcReoqFpIpIccE7Oc= +github.com/google/s2a-go v0.1.4/go.mod h1:Ej+mSEMGRnqRzjc7VtF+jdBwYG5fuJfiZ8ELkjEwM0A= github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/googleapis/enterprise-certificate-proxy v0.2.3 h1:yk9/cqRKtT9wXZSsRH9aurXEpJX+U6FLtpYTdC3R06k= -github.com/googleapis/enterprise-certificate-proxy v0.2.3/go.mod h1:AwSRAtLfXpU5Nm3pW+v7rGDHp09LsPtGY9MduiEsR9k= +github.com/google/uuid v1.3.1 h1:KjJaJ9iWZ3jOFZIf1Lqf4laDRCasjl0BCmnEGxkdLb4= +github.com/google/uuid v1.3.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/googleapis/enterprise-certificate-proxy v0.2.4 h1:uGy6JWR/uMIILU8wbf+OkstIrNiMjGpEIyhx8f6W7s4= +github.com/googleapis/enterprise-certificate-proxy v0.2.4/go.mod h1:AwSRAtLfXpU5Nm3pW+v7rGDHp09LsPtGY9MduiEsR9k= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= -github.com/googleapis/gax-go/v2 v2.7.1 h1:gF4c0zjUP2H/s/hEGyLA3I0fA2ZWjzYiONAD6cvPr8A= -github.com/googleapis/gax-go/v2 v2.7.1/go.mod h1:4orTrqY6hXxxaUL4LHIPl6lGo8vAE38/qKbhSAKP6QI= +github.com/googleapis/gax-go/v2 v2.12.0 h1:A+gCJKdRfqXkr+BIRGtZLibNXf0m1f9E4HG56etFpas= +github.com/googleapis/gax-go/v2 v2.12.0/go.mod h1:y+aIqrI5eb1YGMVJfuV3185Ts/D7qKpsEkdD5+I6QGU= github.com/googleapis/gnostic v0.0.0-20170729233727-0c5108395e2d/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY= github.com/googleapis/gnostic v0.4.0/go.mod h1:on+2t9HRStVgn95RSsFWFz+6Q0Snyqv1awfrALZdbtU= github.com/gophercloud/gophercloud v0.1.0/go.mod h1:vxM41WHh5uqHVBMZHzuwNOHh8XEoIEcSTewFxm1c5g8= @@ -664,6 +674,7 @@ github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/u github.com/json-iterator/go v1.1.8/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/json-iterator/go v1.1.11 h1:uVUAXhF2To8cbw/3xN3pxj6kk7TYKs98NIrTqPlMWAQ= github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= @@ -691,11 +702,12 @@ github.com/klauspost/compress v1.4.0/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0 github.com/klauspost/compress v1.9.5/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= github.com/klauspost/compress v1.13.1/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg= github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= -github.com/klauspost/compress v1.15.9 h1:wKRjX6JRtDdrE9qwa4b/Cip7ACOshUI4smpCQanqjSY= -github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU= +github.com/klauspost/compress v1.16.7 h1:2mk3MPGNzKyxErAw8YaohYh69+pa4sIQSC0fPGCFR9I= +github.com/klauspost/compress v1.16.7/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= github.com/klauspost/cpuid v0.0.0-20170728055534-ae7887de9fa5/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= -github.com/klauspost/cpuid/v2 v2.0.9 h1:lgaqFMSdTdQYdZ04uHyN2d/eKdOMyi2YLSvlQIBFYa4= github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= +github.com/klauspost/cpuid/v2 v2.2.5 h1:0E5MSMDEoAulmXNFquVs//DdoomxaoTY1kUhbc/qbZg= +github.com/klauspost/cpuid/v2 v2.2.5/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= github.com/klauspost/crc32 v0.0.0-20161016154125-cb6bfca970f6 h1:KAZ1BW2TCmT6PRihDPpocIy1QTtsAsrx6TneU/4+CMg= github.com/klauspost/crc32 v0.0.0-20161016154125-cb6bfca970f6/go.mod h1:+ZoRqAPRLkC4NPOvfYeR5KNOrY6TD+/sAC3HXPZgDYg= github.com/klauspost/pgzip v1.0.2-0.20170402124221-0bf5dcad4ada h1:3L+neHp83cTjegPdCiOxVOJtRIy7/8RldvMTsyPYH10= @@ -738,8 +750,8 @@ github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVc github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-colorable v0.1.7/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= -github.com/mattn/go-colorable v0.1.9 h1:sqDoxXbdeALODt0DAeJCVp38ps9ZogZEAXjus69YV3U= -github.com/mattn/go-colorable v0.1.9/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= +github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= +github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= github.com/mattn/go-ieproxy v0.0.1 h1:qiyop7gCflfhwCzGyeT0gro3sF9AIg9HU98JORTkqfI= github.com/mattn/go-ieproxy v0.0.1/go.mod h1:pYabZ6IHcRpFh7vIaLfK7rdcWgFEb3SFJ6/gNWuh88E= github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= @@ -749,9 +761,9 @@ github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2y github.com/mattn/go-isatty v0.0.10/go.mod h1:qgIWMr58cqv1PHHyhnkY9lrL7etaEgOFcMEpPG5Rm84= github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOAqxQCu2WE= github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= -github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= -github.com/mattn/go-isatty v0.0.16 h1:bq3VjFmv/sOjHtdEhmkEV4x1AJtvUvOJ2PFAZ5+peKQ= github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= +github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA= +github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= github.com/mattn/go-runewidth v0.0.3 h1:a+kO+98RDGEfo6asOGMmpodZq4FNtnGP54yps8BzLR4= github.com/mattn/go-runewidth v0.0.3/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= @@ -784,8 +796,10 @@ github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:F github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/mapstructure v1.2.2/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc= github.com/mschoch/smat v0.0.0-20160514031455-90eadee771ae h1:VeRdUYdCw49yizlSbMEn2SZ+gT+3IUKx8BqxyQdz+BY= @@ -847,8 +861,8 @@ github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi github.com/pierrec/lz4/v4 v4.1.8/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= github.com/pierrec/lz4/v4 v4.1.11/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= github.com/pierrec/lz4/v4 v4.1.12/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= -github.com/pierrec/lz4/v4 v4.1.15 h1:MO0/ucJhngq7299dKLwIMtgTfbkoSPF6AoMYDd8Q4q0= -github.com/pierrec/lz4/v4 v4.1.15/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= +github.com/pierrec/lz4/v4 v4.1.18 h1:xaKrnTkyoqfh1YItXl56+6KJNVYWlEEPuAQW9xsplYQ= +github.com/pierrec/lz4/v4 v4.1.18/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 h1:KoWmjvw+nsYOo29YJK9vDA65RGE3NrOnUtO7a+RF9HU= github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8/go.mod h1:HKlIX3XHQyzLZPlr7++PzdhaXEj94dEiJgZDTsxEqUI= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -972,8 +986,9 @@ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.2/go.mod h1:R6va5+xMeoiuVRoj+gSkQ7d3FALtqAAGI1FQKckRals= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= -github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= github.com/tinylib/msgp v1.0.2/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE= github.com/tinylib/msgp v1.1.0 h1:9fQd+ICuRIu/ue4vxJZu6/LzxN0HwMds2nq/0cFvxHU= @@ -1012,6 +1027,7 @@ github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= github.com/zeebo/assert v1.3.0 h1:g7C04CbJuIDKNPFHmsk4hwZDO5O+kntRxzaUoNXj+IQ= github.com/zeebo/assert v1.3.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0= github.com/zeebo/xxh3 v1.0.1/go.mod h1:8VHV24/3AZLn3b6Mlp/KuC33LWH687Wq6EnziEB+rsA= @@ -1086,6 +1102,7 @@ golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20211117183948-ae814b36b871/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.0.0-20220314234659-1baeb1ce4c0b/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.21.0 h1:X31++rzVUdKhX5sWmSOFZxx8UW/ldWx55cbf08iNAMA= golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs= golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -1144,6 +1161,7 @@ golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.5.1/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro= golang.org/x/mod v0.6.0-dev.0.20211013180041-c96bc1413d57/go.mod h1:3p9vT2HGsQu2K1YbXdKPJLVgG5VJdoTa1poYQBtP1AY= golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3/go.mod h1:3p9vT2HGsQu2K1YbXdKPJLVgG5VJdoTa1poYQBtP1AY= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.14.0 h1:dGoOF9QVLYng8IHTm7BAyWqCqSheQ5pYWGhzW00YJr0= golang.org/x/mod v0.14.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/net v0.0.0-20170114055629-f2499483f923/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -1205,6 +1223,7 @@ golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qx golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211118161319-6a13c67c3ce4/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.23.0 h1:7EYJ93RZ9vYSZAIb2x3lnuvqO5zneoD6IvWjuhfxjTs= golang.org/x/net v0.23.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -1220,8 +1239,8 @@ golang.org/x/oauth2 v0.0.0-20210220000619-9bb904979d93/go.mod h1:KelEdhl1UZF7XfJ golang.org/x/oauth2 v0.0.0-20210313182246-cd4f82c27b84/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210427180440-81ed05c6b58c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.7.0 h1:qe6s0zUXlPX80/dITx3440hWZ7GwMwgDDyrSGTPJG/g= -golang.org/x/oauth2 v0.7.0/go.mod h1:hPLQkd9LyjfXTiRohC/41GhcFqxisoUQ99sCUOHO9x4= +golang.org/x/oauth2 v0.10.0 h1:zHCpF2Khkwy4mMB4bv0U37YtJdTGW8jI0glAApi0Kh8= +golang.org/x/oauth2 v0.10.0/go.mod h1:kTpgurOux7LqtuxjuyZa4Gj2gdezIt/jQtGnNFfypQI= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -1234,6 +1253,7 @@ golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.5.0 h1:60k92dhOjHxJkrqnwsfl8KuaHbn/5dl0lUPUklKo3qE= golang.org/x/sync v0.5.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20170830134202-bb24a47a89ea/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -1322,7 +1342,11 @@ golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20211117180635-dee7805ff2e1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4= golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= @@ -1340,6 +1364,7 @@ golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -1428,6 +1453,7 @@ golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.4/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.8-0.20211029000441-d6a9af8af023/go.mod h1:nABZi5QlRsZVlzPpHl034qft6wpY4eDcsTt5AaioBiU= golang.org/x/tools v0.1.10/go.mod h1:Uh6Zz+xoGYZom868N8YTex3t7RhtHDBrE8Gzo9bV56E= +golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.16.0 h1:GO788SKMRunPIBCXiQyo2AaexLstOrVhuAL5YwsckQM= golang.org/x/tools v0.16.0/go.mod h1:kYVVN6I1mBNoB1OX+noeBjbRk4IUEPa7JJ+TJMEooJ0= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -1442,8 +1468,8 @@ gonum.org/v1/gonum v0.0.0-20181121035319-3f7ecaa7e8ca/go.mod h1:Y+Yx5eoAFn32cQvJ gonum.org/v1/gonum v0.6.0/go.mod h1:9mxDZsDKxgMAuccQkewq682L+0eCu4dCN2yonUJTCLU= gonum.org/v1/gonum v0.8.2/go.mod h1:oe/vMfY3deqTw+1EZJhuvEW2iwGF1bW9wwu7XCu0+v0= gonum.org/v1/gonum v0.9.3/go.mod h1:TZumC3NeyVQskjXqmyWt4S3bINhy7B4eYwW69EbyX+0= -gonum.org/v1/gonum v0.11.0 h1:f1IJhK4Km5tBJmaiJXtk/PkL4cdVX6J+tGiM187uT5E= -gonum.org/v1/gonum v0.11.0/go.mod h1:fSG4YDCxxUZQJ7rKsQrj0gMOg00Il0Z96/qMA4bVQhA= +gonum.org/v1/gonum v0.12.0 h1:xKuo6hzt+gMav00meVPUlXwSdoEJP46BR+wdxQEFK2o= +gonum.org/v1/gonum v0.12.0/go.mod h1:73TDxJfAAHeA8Mk9mf8NlIppyhQNo5GLTcYeqgo2lvY= gonum.org/v1/netlib v0.0.0-20181029234149-ec6d1f5cefe6/go.mod h1:wa6Ws7BG/ESfp6dHfk7C6KdzKA7wR7u/rKwOGE66zvw= gonum.org/v1/netlib v0.0.0-20190313105609-8cb42192e0e0/go.mod h1:wa6Ws7BG/ESfp6dHfk7C6KdzKA7wR7u/rKwOGE66zvw= gonum.org/v1/plot v0.0.0-20190515093506-e2840ee46a6b/go.mod h1:Wt8AAjI+ypCyYX3nZBvf6cAIx93T+c/OS2HFAYskSZc= @@ -1472,8 +1498,8 @@ google.golang.org/api v0.41.0/go.mod h1:RkxM5lITDfTzmyKFPt+wGrCJbVfniCr2ool8kTBz google.golang.org/api v0.43.0/go.mod h1:nQsDGjRXMo4lvh5hP0TKqF244gqhGcr/YSIykhUk/94= google.golang.org/api v0.46.0/go.mod h1:ceL4oozhkAiTID8XMmJBsIxID/9wMXJVVFXPg4ylg3I= google.golang.org/api v0.47.0/go.mod h1:Wbvgpq1HddcWVtzsVLyfLp8lDg6AA241LmgIL59tHXo= -google.golang.org/api v0.114.0 h1:1xQPji6cO2E2vLiI+C/XiFAnsn1WV3mjaEwGLhi3grE= -google.golang.org/api v0.114.0/go.mod h1:ifYI2ZsFK6/uGddGfAD5BMxlnkBqCmqHSDUVi45N5Yg= +google.golang.org/api v0.128.0 h1:RjPESny5CnQRn9V6siglged+DZCgfu9l6mO9dkX9VOg= +google.golang.org/api v0.128.0/go.mod h1:Y611qgqaE92On/7g65MQgxYul3c0rEB894kniWLY750= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.2.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= @@ -1533,8 +1559,12 @@ google.golang.org/genproto v0.0.0-20210513213006-bf773b8c8384/go.mod h1:P3QM42oQ google.golang.org/genproto v0.0.0-20210517163617-5e0236093d7a/go.mod h1:P3QM42oQyzQSnHPnZ/vqoCdDmzH28fzWByN9asMeM8A= google.golang.org/genproto v0.0.0-20210601144548-a796c710e9b6/go.mod h1:P3QM42oQyzQSnHPnZ/vqoCdDmzH28fzWByN9asMeM8A= google.golang.org/genproto v0.0.0-20210630183607-d20f26d13c79/go.mod h1:yiaVoXHpRzHGyxV3o4DktVWY4mSUErTKaeEOq6C3t3U= -google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 h1:KpwkzHKEF7B9Zxg18WzOa7djJ+Ha5DzthMyZYQfEn2A= -google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1/go.mod h1:nKE/iIaLqn2bQwXBg8f1g2Ylh6r5MN5CmZvuzZCgsCU= +google.golang.org/genproto v0.0.0-20230920204549-e6e6cdab5c13 h1:vlzZttNJGVqTsRFU9AmdnrcO1Znh8Ew9kCD//yjigk0= +google.golang.org/genproto v0.0.0-20230920204549-e6e6cdab5c13/go.mod h1:CCviP9RmpZ1mxVr8MUjCnSiY09IbAXZxhLE6EhHIdPU= +google.golang.org/genproto/googleapis/api v0.0.0-20230913181813-007df8e322eb h1:lK0oleSc7IQsUxO3U5TjL9DWlsxpEBemh+zpB7IqhWI= +google.golang.org/genproto/googleapis/api v0.0.0-20230913181813-007df8e322eb/go.mod h1:KjSP20unUpOx5kyQUFa7k4OJg0qeJ7DEZflGDu2p6Bk= +google.golang.org/genproto/googleapis/rpc v0.0.0-20231002182017-d307bd883b97 h1:6GQBEOdGkX6MMTLT9V+TjtIRZCw9VPD5Z+yHY9wMgS0= +google.golang.org/genproto/googleapis/rpc v0.0.0-20231002182017-d307bd883b97/go.mod h1:v7nGkzlmW8P3n/bKmWBn2WpBjpOEx8Q6gMueudAmKfY= google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.0/go.mod h1:chYK+tFQF0nDUGJgXMSgLCQk3phJEuONr2DCgLDdAQM= @@ -1566,8 +1596,9 @@ google.golang.org/grpc v1.37.1/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQ google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= google.golang.org/grpc v1.39.0/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnDzfrE= google.golang.org/grpc v1.41.0/go.mod h1:U3l9uK9J0sini8mHphKoXyaqDA/8VyGnDee1zzIUK6k= -google.golang.org/grpc v1.56.3 h1:8I4C0Yq1EjstUzUJzpcRVbuYA2mODtEmpWiQoN/b2nc= -google.golang.org/grpc v1.56.3/go.mod h1:I9bI3vqKfayGqPUAwGdOSu7kt6oIJLixfffKrpXqQ9s= +google.golang.org/grpc v1.45.0/go.mod h1:lN7owxKUQEqMfSyQikvvk5tf/6zMPsrK+ONuO11+0rQ= +google.golang.org/grpc v1.58.2 h1:SXUpjxeVF3FKrTYQI4f4KvbGD5u2xccdYdurwowix5I= +google.golang.org/grpc v1.58.2/go.mod h1:tgX3ZQDlNJGU96V6yHh1T/JeoBQ2TXdr43YbYSsCJk0= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= From 733e1bbb768275b1b4533a9bac3f708fc9293d4f Mon Sep 17 00:00:00 2001 From: Ales Pour Date: Fri, 7 Jun 2024 12:52:04 +0200 Subject: [PATCH 02/16] style: import order --- cmd/influx_inspect/export/export_parquet.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/influx_inspect/export/export_parquet.go b/cmd/influx_inspect/export/export_parquet.go index 10e6c275cfb..2c284a1f4ca 100644 --- a/cmd/influx_inspect/export/export_parquet.go +++ b/cmd/influx_inspect/export/export_parquet.go @@ -3,7 +3,6 @@ package export import ( "context" "fmt" - "golang.org/x/exp/maps" "io" "os" "path/filepath" @@ -20,6 +19,7 @@ import ( tsdb "github.com/influxdata/influxdb/cmd/influx_inspect/export/parquet/tsm1" "github.com/influxdata/influxdb/tsdb/engine/tsm1" "go.uber.org/zap" + "golang.org/x/exp/maps" ) // From 3b5896cad7265b7623f011ec607a3c5c9752edf8 Mon Sep 17 00:00:00 2001 From: Ales Pour Date: Fri, 7 Jun 2024 17:54:01 +0200 Subject: [PATCH 03/16] fix: unused input parameter --- cmd/influx_inspect/export/export_parquet.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/cmd/influx_inspect/export/export_parquet.go b/cmd/influx_inspect/export/export_parquet.go index 2c284a1f4ca..b37ab7f616d 100644 --- a/cmd/influx_inspect/export/export_parquet.go +++ b/cmd/influx_inspect/export/export_parquet.go @@ -49,11 +49,10 @@ func (cmd *Command) writeValuesParquet(_ io.Writer, seriesKey []byte, field stri vc[key] = fields } fields[field] = values - return nil } -func (cmd *Command) exportDoneParquet(path string) error { +func (cmd *Command) exportDoneParquet(_ string) error { defer func() { vc = nil }() From 92f08e2e154274533f6bd61be63d15b16035e63b Mon Sep 17 00:00:00 2001 From: Ales Pour Date: Fri, 7 Jun 2024 17:55:58 +0200 Subject: [PATCH 04/16] test: add influx_inspect test --- .../export/export_parquet_test.go | 80 +++++++++++++++++++ cmd/influx_inspect/export/export_test.go | 5 +- 2 files changed, 84 insertions(+), 1 deletion(-) create mode 100644 cmd/influx_inspect/export/export_parquet_test.go diff --git a/cmd/influx_inspect/export/export_parquet_test.go b/cmd/influx_inspect/export/export_parquet_test.go new file mode 100644 index 00000000000..3f62d7743d5 --- /dev/null +++ b/cmd/influx_inspect/export/export_parquet_test.go @@ -0,0 +1,80 @@ +package export + +import ( + "github.com/influxdata/influxdb/tsdb/engine/tsm1" + "io" + "os" + "testing" +) + +var ( + myCorpus = corpus{ + tsm1.SeriesFieldKey("mym,tag=abc", "f"): []tsm1.Value{ + tsm1.NewValue(1, 1.5), + tsm1.NewValue(2, 3.0), + }, + tsm1.SeriesFieldKey("mym,tag=abc", "i"): []tsm1.Value{ + tsm1.NewValue(1, int64(15)), + tsm1.NewValue(2, int64(30)), + }, + tsm1.SeriesFieldKey("mym,tag=abc", "b"): []tsm1.Value{ + tsm1.NewValue(1, true), + tsm1.NewValue(2, false), + }, + tsm1.SeriesFieldKey("mym,tag=abc", "s"): []tsm1.Value{ + tsm1.NewValue(1, "1k"), + tsm1.NewValue(2, "2k"), + }, + tsm1.SeriesFieldKey("mym,tag=abc", "u"): []tsm1.Value{ + tsm1.NewValue(1, uint64(45)), + tsm1.NewValue(2, uint64(60)), + }, + tsm1.SeriesFieldKey("mym,tag=xyz", "f"): []tsm1.Value{ + tsm1.NewValue(1000, 1.5), + tsm1.NewValue(2000, 3.0), + }, + tsm1.SeriesFieldKey("mym,tag=xyz", "i"): []tsm1.Value{ + tsm1.NewValue(1000, int64(15)), + tsm1.NewValue(2000, int64(30)), + }, + tsm1.SeriesFieldKey("mym,tag=xyz", "b"): []tsm1.Value{ + tsm1.NewValue(1000, true), + tsm1.NewValue(2000, false), + }, + tsm1.SeriesFieldKey("mym,tag=xyz", "s"): []tsm1.Value{ + tsm1.NewValue(1000, "1k"), + tsm1.NewValue(2000, "2k"), + }, + tsm1.SeriesFieldKey("mym,tag=xyz", "u"): []tsm1.Value{ + tsm1.NewValue(1000, uint64(45)), + tsm1.NewValue(2000, uint64(60)), + }, + tsm1.SeriesFieldKey("alien,a=b", "level"): []tsm1.Value{ + tsm1.NewValue(1000000, "error"), + }, + } +) + +func TestExportParquet(t *testing.T) { + for _, c := range []struct { + corpus corpus + }{ + {corpus: myCorpus}, + } { + tsmFile := writeCorpusToTSMFile(c.corpus) + defer os.Remove(tsmFile.Name()) + outdir := t.TempDir() + + cmd := newCommand() + cmd.out = outdir + cmd.writeValues = cmd.writeValuesParquet + cmd.exportDone = cmd.exportDoneParquet + cmd.measurement = "mym" + cmd.parquet = true + cmd.pqChunkSize = 100_000_000 + + if err := cmd.writeTsmFiles(io.Discard, io.Discard, []string{tsmFile.Name()}); err != nil { + t.Fatal(err) + } + } +} diff --git a/cmd/influx_inspect/export/export_test.go b/cmd/influx_inspect/export/export_test.go index 1ecb83f3c93..8f16fc9df3a 100644 --- a/cmd/influx_inspect/export/export_test.go +++ b/cmd/influx_inspect/export/export_test.go @@ -226,12 +226,15 @@ func BenchmarkExportWALStrings_100s_250vps(b *testing.B) { // newCommand returns a command that discards its output and that accepts all timestamps. func newCommand() *Command { - return &Command{ + cmd := &Command{ Stderr: io.Discard, Stdout: io.Discard, startTime: math.MinInt64, endTime: math.MaxInt64, } + cmd.writeValues = cmd.writeValuesLp + cmd.exportDone = func(_ string) error { return nil } + return cmd } // makeCorpus returns a new corpus filled with values generated by fn. From 2544c280e2fa3f609d17fdc9c2da41add74f8349 Mon Sep 17 00:00:00 2001 From: Ales Pour Date: Fri, 7 Jun 2024 18:07:45 +0200 Subject: [PATCH 05/16] style: go fmt --- cmd/influx_inspect/export/export_parquet_test.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cmd/influx_inspect/export/export_parquet_test.go b/cmd/influx_inspect/export/export_parquet_test.go index 3f62d7743d5..4fb76c97c9e 100644 --- a/cmd/influx_inspect/export/export_parquet_test.go +++ b/cmd/influx_inspect/export/export_parquet_test.go @@ -1,10 +1,11 @@ package export import ( - "github.com/influxdata/influxdb/tsdb/engine/tsm1" "io" "os" "testing" + + "github.com/influxdata/influxdb/tsdb/engine/tsm1" ) var ( From f04fc04b710eaabd4a10b8f772f273f88058cd9f Mon Sep 17 00:00:00 2001 From: Ales Pour Date: Mon, 10 Jun 2024 11:49:08 +0200 Subject: [PATCH 06/16] fix: extend Parquet options values checks --- cmd/influx_inspect/export/export.go | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/cmd/influx_inspect/export/export.go b/cmd/influx_inspect/export/export.go index cab6cee03c0..3cdb23b9a9d 100644 --- a/cmd/influx_inspect/export/export.go +++ b/cmd/influx_inspect/export/export.go @@ -74,7 +74,7 @@ func (cmd *Command) Run(args ...string) error { fs := flag.NewFlagSet("export", flag.ExitOnError) fs.StringVar(&cmd.dataDir, "datadir", os.Getenv("HOME")+"/.influxdb/data", "Data storage path") fs.StringVar(&cmd.walDir, "waldir", os.Getenv("HOME")+"/.influxdb/wal", "WAL storage path") - fs.StringVar(&cmd.out, "out", os.Getenv("HOME")+"/.influxdb/export", "'-' for standard out or the destination file to export to") + fs.StringVar(&cmd.out, "out", os.Getenv("HOME")+"/.influxdb/export", "'-' for standard out or the destination file to export to (line protocol) | directory to write Parquet files") fs.StringVar(&cmd.database, "database", "", "Optional: the database to export") fs.StringVar(&cmd.retentionPolicy, "retention", "", "Optional: the retention policy to export (requires -database)") fs.StringVar(&cmd.measurement, "measurement", "", "Name of measurement to export") @@ -139,8 +139,16 @@ func (cmd *Command) validate() error { if cmd.startTime != 0 && cmd.endTime != 0 && cmd.endTime < cmd.startTime { return fmt.Errorf("end time before start time") } - if cmd.parquet && (cmd.database == "" || cmd.retentionPolicy == "" || cmd.measurement == "") { - return fmt.Errorf("must specify database, renetion and measurement when exporting to Parquet") + if cmd.parquet { + if cmd.database == "" || cmd.retentionPolicy == "" || cmd.measurement == "" { + return fmt.Errorf("must specify database, retention and measurement when exporting to Parquet") + } + if cmd.out == "-" { + return fmt.Errorf("-out must point to a folder for Parquet files") + } + if cmd.pqChunkSize < 1_000_000 { + return fmt.Errorf("minimum Parquet partition size is 1000000 bytes") + } } return nil } From 031aae71918d3b2d5b201b5b3d2b09889c8599cf Mon Sep 17 00:00:00 2001 From: Ales Pour Date: Thu, 13 Jun 2024 09:41:30 +0200 Subject: [PATCH 07/16] chore: update arrow to v16 --- cmd/influx_inspect/export/export_parquet.go | 4 +- .../export/parquet/exporter/exporter.go | 8 +- .../export/parquet/table/allocator.go | 2 +- .../export/parquet/table/column.go | 6 +- .../export/parquet/table/table.go | 6 +- go.mod | 76 ++++---- go.sum | 171 ++++++++++-------- 7 files changed, 152 insertions(+), 121 deletions(-) diff --git a/cmd/influx_inspect/export/export_parquet.go b/cmd/influx_inspect/export/export_parquet.go index b37ab7f616d..61b39ee7a7d 100644 --- a/cmd/influx_inspect/export/export_parquet.go +++ b/cmd/influx_inspect/export/export_parquet.go @@ -9,8 +9,8 @@ import ( "sync" "time" - "github.com/apache/arrow/go/v14/parquet" - "github.com/apache/arrow/go/v14/parquet/compress" + "github.com/apache/arrow/go/v16/parquet" + "github.com/apache/arrow/go/v16/parquet/compress" pqexport "github.com/influxdata/influxdb/cmd/influx_inspect/export/parquet/exporter" "github.com/influxdata/influxdb/cmd/influx_inspect/export/parquet/index" "github.com/influxdata/influxdb/cmd/influx_inspect/export/parquet/models" diff --git a/cmd/influx_inspect/export/parquet/exporter/exporter.go b/cmd/influx_inspect/export/parquet/exporter/exporter.go index 00fe2c41908..9ccacebd652 100644 --- a/cmd/influx_inspect/export/parquet/exporter/exporter.go +++ b/cmd/influx_inspect/export/parquet/exporter/exporter.go @@ -4,10 +4,10 @@ import ( "context" "io" - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/arrow/array" - "github.com/apache/arrow/go/v14/parquet" - "github.com/apache/arrow/go/v14/parquet/pqarrow" + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/array" + "github.com/apache/arrow/go/v16/parquet" + "github.com/apache/arrow/go/v16/parquet/pqarrow" "github.com/influxdata/influxdb/cmd/influx_inspect/export/parquet/models" "github.com/influxdata/influxdb/cmd/influx_inspect/export/parquet/table" "github.com/pkg/errors" diff --git a/cmd/influx_inspect/export/parquet/table/allocator.go b/cmd/influx_inspect/export/parquet/table/allocator.go index ce06c24ac34..6a664d09a7e 100644 --- a/cmd/influx_inspect/export/parquet/table/allocator.go +++ b/cmd/influx_inspect/export/parquet/table/allocator.go @@ -3,7 +3,7 @@ package table import ( "sync/atomic" - "github.com/apache/arrow/go/v14/arrow/memory" + "github.com/apache/arrow/go/v16/arrow/memory" ) // trackedAllocator is a memory.Allocator that tracks the total allocation size. diff --git a/cmd/influx_inspect/export/parquet/table/column.go b/cmd/influx_inspect/export/parquet/table/column.go index f0849302e79..be86c406b66 100644 --- a/cmd/influx_inspect/export/parquet/table/column.go +++ b/cmd/influx_inspect/export/parquet/table/column.go @@ -3,9 +3,9 @@ package table import ( "fmt" - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/arrow/array" - "github.com/apache/arrow/go/v14/arrow/memory" + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/array" + "github.com/apache/arrow/go/v16/arrow/memory" "github.com/influxdata/influxdb/cmd/influx_inspect/export/parquet/resultset" "github.com/influxdata/influxdb/cmd/influx_inspect/export/parquet/tsm1" ) diff --git a/cmd/influx_inspect/export/parquet/table/table.go b/cmd/influx_inspect/export/parquet/table/table.go index c234eafbd13..94c161b5fbc 100644 --- a/cmd/influx_inspect/export/parquet/table/table.go +++ b/cmd/influx_inspect/export/parquet/table/table.go @@ -9,9 +9,9 @@ import ( "strings" "sync/atomic" - "github.com/apache/arrow/go/v14/arrow" - "github.com/apache/arrow/go/v14/arrow/array" - "github.com/apache/arrow/go/v14/arrow/memory" + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/array" + "github.com/apache/arrow/go/v16/arrow/memory" "github.com/influxdata/influxdb/cmd/influx_inspect/export/parquet/index" "github.com/influxdata/influxdb/cmd/influx_inspect/export/parquet/models" "github.com/influxdata/influxdb/cmd/influx_inspect/export/parquet/resultset" diff --git a/go.mod b/go.mod index 68740dc49cf..f5da89e2223 100644 --- a/go.mod +++ b/go.mod @@ -6,7 +6,7 @@ require ( collectd.org v0.3.0 github.com/BurntSushi/toml v0.3.1 github.com/apache/arrow/go/arrow v0.0.0-20211112161151-bc219186db40 - github.com/apache/arrow/go/v14 v14.0.2 + github.com/apache/arrow/go/v16 v16.1.0 github.com/benbjohnson/tmpl v1.0.0 github.com/bmizerany/pat v0.0.0-20170815010413-6226ea591a40 github.com/cespare/xxhash v1.1.0 @@ -16,14 +16,14 @@ require ( github.com/golang-jwt/jwt v3.2.1+incompatible github.com/golang/mock v1.5.0 github.com/golang/snappy v0.0.4 - github.com/google/go-cmp v0.5.9 + github.com/google/go-cmp v0.6.0 github.com/influxdata/flux v0.194.5 github.com/influxdata/httprouter v1.3.1-0.20191122104820-ee83e2772f69 github.com/influxdata/influxql v1.2.0 github.com/influxdata/pkg-config v0.2.11 github.com/influxdata/roaring v0.4.13-0.20180809181101-fc520f41fab6 github.com/influxdata/usage-client v0.0.0-20160829180054-6d3895376368 - github.com/json-iterator/go v1.1.11 + github.com/json-iterator/go v1.1.12 github.com/jsternberg/zap-logfmt v1.2.0 github.com/jwilder/encoding v0.0.0-20170811194829-b4e1701a28ef github.com/klauspost/pgzip v1.0.2-0.20170402124221-0bf5dcad4ada @@ -39,32 +39,32 @@ require ( github.com/retailnext/hllpp v1.0.1-0.20180308014038-101a6d2f8b52 github.com/spf13/cast v1.3.0 github.com/spf13/cobra v0.0.3 - github.com/stretchr/testify v1.8.4 + github.com/stretchr/testify v1.9.0 github.com/tinylib/msgp v1.1.0 github.com/uber/jaeger-client-go v2.28.0+incompatible github.com/xlab/treeprint v0.0.0-20180616005107-d6fb6747feb6 go.uber.org/multierr v1.6.0 go.uber.org/zap v1.16.0 golang.org/x/crypto v0.21.0 - golang.org/x/exp v0.0.0-20231214170342-aacd6d4b4611 - golang.org/x/sync v0.5.0 + golang.org/x/exp v0.0.0-20240222234643-814bf88cf225 + golang.org/x/sync v0.6.0 golang.org/x/sys v0.18.0 golang.org/x/term v0.18.0 golang.org/x/text v0.14.0 - golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba - golang.org/x/tools v0.16.0 - google.golang.org/grpc v1.58.2 + golang.org/x/time v0.5.0 + golang.org/x/tools v0.19.0 + google.golang.org/grpc v1.62.1 google.golang.org/protobuf v1.33.0 ) require ( - cloud.google.com/go v0.110.8 // indirect - cloud.google.com/go/bigquery v1.55.0 // indirect + cloud.google.com/go v0.112.0 // indirect + cloud.google.com/go/bigquery v1.58.0 // indirect cloud.google.com/go/bigtable v1.10.1 // indirect - cloud.google.com/go/compute v1.23.0 // indirect + cloud.google.com/go/compute v1.23.3 // indirect cloud.google.com/go/compute/metadata v0.2.3 // indirect - cloud.google.com/go/iam v1.1.2 // indirect - cloud.google.com/go/longrunning v0.5.1 // indirect + cloud.google.com/go/iam v1.1.5 // indirect + cloud.google.com/go/longrunning v0.5.4 // indirect github.com/Azure/azure-pipeline-go v0.2.3 // indirect github.com/Azure/azure-storage-blob-go v0.14.0 // indirect github.com/Azure/go-autorest v14.2.0+incompatible // indirect @@ -81,11 +81,11 @@ require ( github.com/Masterminds/sprig v2.16.0+incompatible // indirect github.com/SAP/go-hdb v0.14.1 // indirect github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883 // indirect - github.com/andybalholm/brotli v1.0.5 // indirect + github.com/andybalholm/brotli v1.1.0 // indirect github.com/aokoli/goutils v1.0.1 // indirect - github.com/apache/arrow/go/v12 v12.0.0 // indirect + github.com/apache/arrow/go/v12 v12.0.1 // indirect github.com/apache/arrow/go/v7 v7.0.1 // indirect - github.com/apache/thrift v0.17.0 // indirect + github.com/apache/thrift v0.19.0 // indirect github.com/aws/aws-sdk-go v1.34.0 // indirect github.com/aws/aws-sdk-go-v2 v1.11.0 // indirect github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.0.0 // indirect @@ -108,9 +108,12 @@ require ( github.com/dustin/go-humanize v1.0.1 // indirect github.com/eclipse/paho.mqtt.golang v1.2.0 // indirect github.com/fatih/color v1.15.0 // indirect + github.com/felixge/httpsnoop v1.0.4 // indirect github.com/form3tech-oss/jwt-go v3.2.5+incompatible // indirect github.com/gabriel-vasile/mimetype v1.4.0 // indirect github.com/glycerine/go-unsnap-stream v0.0.0-20180323001048-9f0cb55181dd // indirect + github.com/go-logr/logr v1.3.0 // indirect + github.com/go-logr/stdr v1.2.2 // indirect github.com/go-sql-driver/mysql v1.5.0 // indirect github.com/goccy/go-json v0.10.2 // indirect github.com/gofrs/uuid v3.3.0+incompatible // indirect @@ -119,10 +122,10 @@ require ( github.com/golang/geo v0.0.0-20190916061304-5b978397cfec // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.4 // indirect - github.com/google/flatbuffers v23.5.26+incompatible // indirect - github.com/google/s2a-go v0.1.4 // indirect - github.com/google/uuid v1.3.1 // indirect - github.com/googleapis/enterprise-certificate-proxy v0.2.4 // indirect + github.com/google/flatbuffers v24.3.25+incompatible // indirect + github.com/google/s2a-go v0.1.7 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/googleapis/enterprise-certificate-proxy v0.3.2 // indirect github.com/googleapis/gax-go/v2 v2.12.0 // indirect github.com/grpc-ecosystem/grpc-gateway v1.16.0 // indirect github.com/huandu/xstrings v1.0.0 // indirect @@ -136,8 +139,8 @@ require ( github.com/influxdata/tdigest v0.0.2-0.20210216194612-fc98d27c9e8b // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/klauspost/asmfmt v1.3.2 // indirect - github.com/klauspost/compress v1.16.7 // indirect - github.com/klauspost/cpuid/v2 v2.2.5 // indirect + github.com/klauspost/compress v1.17.7 // indirect + github.com/klauspost/cpuid/v2 v2.2.7 // indirect github.com/klauspost/crc32 v0.0.0-20161016154125-cb6bfca970f6 // indirect github.com/lib/pq v1.0.0 // indirect github.com/mattn/go-colorable v0.1.13 // indirect @@ -148,10 +151,10 @@ require ( github.com/minio/c2goasm v0.0.0-20190812172519-36a3d3bbc4f3 // indirect github.com/mitchellh/go-homedir v1.1.0 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect - github.com/modern-go/reflect2 v1.0.1 // indirect + github.com/modern-go/reflect2 v1.0.2 // indirect github.com/mschoch/smat v0.0.0-20160514031455-90eadee771ae // indirect github.com/philhofer/fwd v1.0.0 // indirect - github.com/pierrec/lz4/v4 v4.1.18 // indirect + github.com/pierrec/lz4/v4 v4.1.21 // indirect github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/prometheus/procfs v0.6.0 // indirect @@ -166,17 +169,22 @@ require ( github.com/willf/bitset v1.1.9 // indirect github.com/zeebo/xxh3 v1.0.2 // indirect go.opencensus.io v0.24.0 // indirect + go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.46.1 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.46.1 // indirect + go.opentelemetry.io/otel v1.21.0 // indirect + go.opentelemetry.io/otel/metric v1.21.0 // indirect + go.opentelemetry.io/otel/trace v1.21.0 // indirect go.uber.org/atomic v1.7.0 // indirect - golang.org/x/mod v0.14.0 // indirect + golang.org/x/mod v0.16.0 // indirect golang.org/x/net v0.23.0 // indirect - golang.org/x/oauth2 v0.10.0 // indirect - golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect - gonum.org/v1/gonum v0.12.0 // indirect - google.golang.org/api v0.128.0 // indirect - google.golang.org/appengine v1.6.7 // indirect - google.golang.org/genproto v0.0.0-20230920204549-e6e6cdab5c13 // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20230913181813-007df8e322eb // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20231002182017-d307bd883b97 // indirect + golang.org/x/oauth2 v0.16.0 // indirect + golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 // indirect + gonum.org/v1/gonum v0.15.0 // indirect + google.golang.org/api v0.155.0 // indirect + google.golang.org/appengine v1.6.8 // indirect + google.golang.org/genproto v0.0.0-20240123012728-ef4313101c80 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20240123012728-ef4313101c80 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240123012728-ef4313101c80 // indirect gopkg.in/yaml.v2 v2.3.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index dfe9f0982d1..d1ba9133b8b 100644 --- a/go.sum +++ b/go.sum @@ -21,31 +21,31 @@ cloud.google.com/go v0.78.0/go.mod h1:QjdrLG0uq+YwhjoVOLsS1t7TW8fs36kLs4XO5R5ECH cloud.google.com/go v0.79.0/go.mod h1:3bzgcEeQlzbuEAYu4mrWhKqWjmpprinYgKJLgKHnbb8= cloud.google.com/go v0.81.0/go.mod h1:mk/AM35KwGk/Nm2YSeZbxXdrNK3KZOYHmLkOqC2V6E0= cloud.google.com/go v0.82.0/go.mod h1:vlKccHJGuFBFufnAnuB08dfEH9Y3H7dzDzRECFdC2TA= -cloud.google.com/go v0.110.8 h1:tyNdfIxjzaWctIiLYOTalaLKZ17SI44SKFW26QbOhME= -cloud.google.com/go v0.110.8/go.mod h1:Iz8AkXJf1qmxC3Oxoep8R1T36w8B92yU29PcBhHO5fk= +cloud.google.com/go v0.112.0 h1:tpFCD7hpHFlQ8yPwT3x+QeXqc2T6+n6T+hmABHfDUSM= +cloud.google.com/go v0.112.0/go.mod h1:3jEEVwZ/MHU4djK5t5RHuKOA/GbLddgTdVubX1qnPD4= cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= -cloud.google.com/go/bigquery v1.55.0 h1:hs44Xxov3XLWQiCx2J8lK5U/ihLqnpm4RVVl5fdtLLI= -cloud.google.com/go/bigquery v1.55.0/go.mod h1:9Y5I3PN9kQWuid6183JFhOGOW3GcirA5LpsKCUn+2ec= +cloud.google.com/go/bigquery v1.58.0 h1:drSd9RcPVLJP2iFMimvOB9SCSIrcl+9HD4II03Oy7A0= +cloud.google.com/go/bigquery v1.58.0/go.mod h1:0eh4mWNY0KrBTjUzLjoYImapGORq9gEPT7MWjCy9lik= cloud.google.com/go/bigtable v1.2.0/go.mod h1:JcVAOl45lrTmQfLj7T6TxyMzIN/3FGGcFm+2xVAli2o= cloud.google.com/go/bigtable v1.10.1 h1:QKcRHeAsraxIlrdCZ3LLobXKBvITqcOEnSbHG2rzL9g= cloud.google.com/go/bigtable v1.10.1/go.mod h1:cyHeKlx6dcZCO0oSQucYdauseD8kIENGuDOJPKMCVg8= -cloud.google.com/go/compute v1.23.0 h1:tP41Zoavr8ptEqaW6j+LQOnyBBhO7OkOMAGrgLopTwY= -cloud.google.com/go/compute v1.23.0/go.mod h1:4tCnrn48xsqlwSAiLf1HXMQk8CONslYbdiEZc9FEIbM= +cloud.google.com/go/compute v1.23.3 h1:6sVlXXBmbd7jNX0Ipq0trII3e4n1/MsADLK6a+aiVlk= +cloud.google.com/go/compute v1.23.3/go.mod h1:VCgBUoMnIVIR0CscqQiPJLAG25E3ZRZMzcFZeQ+h8CI= cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGBW5aJ7UnBMY= cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA= -cloud.google.com/go/datacatalog v1.17.1 h1:qGWrlYvWtK+8jD1jhwq5BsGoSr7S4/LOroV7LwXi00g= -cloud.google.com/go/datacatalog v1.17.1/go.mod h1:nCSYFHgtxh2MiEktWIz71s/X+7ds/UT9kp0PC7waCzE= +cloud.google.com/go/datacatalog v1.19.2 h1:BV5sB7fPc8ccv/obwtHwQtCdLMAgI4KyaQWfkh8/mWg= +cloud.google.com/go/datacatalog v1.19.2/go.mod h1:2YbODwmhpLM4lOFe3PuEhHK9EyTzQJ5AXgIy7EDKTEE= cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= -cloud.google.com/go/iam v1.1.2 h1:gacbrBdWcoVmGLozRuStX45YKvJtzIjJdAolzUs1sm4= -cloud.google.com/go/iam v1.1.2/go.mod h1:A5avdyVL2tCppe4unb0951eI9jreack+RJ0/d+KUZOU= -cloud.google.com/go/longrunning v0.5.1 h1:Fr7TXftcqTudoyRJa113hyaqlGdiBQkp0Gq7tErFDWI= -cloud.google.com/go/longrunning v0.5.1/go.mod h1:spvimkwdz6SPWKEt/XBij79E9fiTkHSQl/fRUUQJYJc= +cloud.google.com/go/iam v1.1.5 h1:1jTsCu4bcsNsE4iiqNT5SHwrDRCfRmIaaaVFhRveTJI= +cloud.google.com/go/iam v1.1.5/go.mod h1:rB6P/Ic3mykPbFio+vo7403drjlgvoWfYpJhMXEbzv8= +cloud.google.com/go/longrunning v0.5.4 h1:w8xEcbZodnA2BbW6sVirkkoC+1gP8wS57EUUgGS0GVg= +cloud.google.com/go/longrunning v0.5.4/go.mod h1:zqNVncI0BOP8ST6XQD1+VcvuShMmq7+xFSzOL++V0dI= cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= @@ -55,8 +55,8 @@ cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0Zeo cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= -cloud.google.com/go/storage v1.30.1 h1:uOdMxAs8HExqBlnLtnQyP0YkvbiDpdGShGKtx6U/oNM= -cloud.google.com/go/storage v1.30.1/go.mod h1:NfxhC0UJE1aXSx7CIIbCf7y9HKT7BiccwkR7+P7gN8E= +cloud.google.com/go/storage v1.36.0 h1:P0mOkAcaJxhCTvAkMhxMfrTKiNcub4YmmPBtlhAyTr8= +cloud.google.com/go/storage v1.36.0/go.mod h1:M6M/3V/D3KpzMTJyPOR/HU6n2Si5QdaXYEsng2xgOs8= collectd.org v0.3.0 h1:iNBHGw1VvPJxH2B6RiFWFZ+vsjo1lCdRszBeOuwGi00= collectd.org v0.3.0/go.mod h1:A/8DzQBkF6abtvrT2j/AU/4tiBgJWYyh0y/oB/4MlWE= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= @@ -141,8 +141,8 @@ github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk5 github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883 h1:bvNMNQO63//z+xNgfBlViaCIJKLlCJ6/fmUseuG0wVQ= github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8= github.com/andybalholm/brotli v1.0.3/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= -github.com/andybalholm/brotli v1.0.5 h1:8uQZIdzKmjc/iuPu7O2ioW48L81FgatrcpfFmiq/cCs= -github.com/andybalholm/brotli v1.0.5/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= +github.com/andybalholm/brotli v1.1.0 h1:eLKJA0d02Lf0mVpIDgYnqXcUn0GqVmEFny3VuID1U3M= +github.com/andybalholm/brotli v1.1.0/go.mod h1:sms7XGricyQI9K10gOSf56VKKWS4oLer58Q+mhRPtnY= github.com/antihax/optional v0.0.0-20180407024304-ca021399b1a6/go.mod h1:V8iCPQYkqmusNa815XgQio277wI47sdRh1dUOLdyC6Q= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= github.com/aokoli/goutils v1.0.1 h1:7fpzNGoJ3VA8qcrm++XEE1QUe0mIwNeLa02Nwq7RDkg= @@ -150,17 +150,17 @@ github.com/aokoli/goutils v1.0.1/go.mod h1:SijmP0QR8LtwsmDs8Yii5Z/S4trXFGFC2oO5g github.com/apache/arrow/go/arrow v0.0.0-20191024131854-af6fa24be0db/go.mod h1:VTxUBvSJ3s3eHAg65PNgrsn5BtqCRPdmyXh6rAfdxN0= github.com/apache/arrow/go/arrow v0.0.0-20211112161151-bc219186db40 h1:q4dksr6ICHXqG5hm0ZW5IHyeEJXoIJSOZeBLmWPNeIQ= github.com/apache/arrow/go/arrow v0.0.0-20211112161151-bc219186db40/go.mod h1:Q7yQnSMnLvcXlZ8RV+jwz/6y1rQTqbX6C82SndT52Zs= -github.com/apache/arrow/go/v12 v12.0.0 h1:xtZE63VWl7qLdB0JObIXvvhGjoVNrQ9ciIHG2OK5cmc= -github.com/apache/arrow/go/v12 v12.0.0/go.mod h1:d+tV/eHZZ7Dz7RPrFKtPK02tpr+c9/PEd/zm8mDS9Vg= -github.com/apache/arrow/go/v14 v14.0.2 h1:N8OkaJEOfI3mEZt07BIkvo4sC6XDbL+48MBPWO5IONw= -github.com/apache/arrow/go/v14 v14.0.2/go.mod h1:u3fgh3EdgN/YQ8cVQRguVW3R+seMybFg8QBQ5LU+eBY= +github.com/apache/arrow/go/v12 v12.0.1 h1:JsR2+hzYYjgSUkBSaahpqCetqZMr76djX80fF/DiJbg= +github.com/apache/arrow/go/v12 v12.0.1/go.mod h1:weuTY7JvTG/HDPtMQxEUp7pU73vkLWMLpY67QwZ/WWw= +github.com/apache/arrow/go/v16 v16.1.0 h1:dwgfOya6s03CzH9JrjCBx6bkVb4yPD4ma3haj9p7FXI= +github.com/apache/arrow/go/v16 v16.1.0/go.mod h1:9wnc9mn6vEDTRIm4+27pEjQpRKuTvBaessPoEXQzxWA= github.com/apache/arrow/go/v7 v7.0.1 h1:WpCfq+AQxvXaI6/KplHE27MPMFx5av0o5NbPCTAGfy4= github.com/apache/arrow/go/v7 v7.0.1/go.mod h1:JxDpochJbCVxqbX4G8i1jRqMrnTCQdf8pTccAfLD8Es= github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= github.com/apache/thrift v0.13.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= github.com/apache/thrift v0.15.0/go.mod h1:PHK3hniurgQaNMZYaCLEqXKsYK8upmhPbmdP2FXSqgU= -github.com/apache/thrift v0.17.0 h1:cMd2aj52n+8VoAtvSvLn4kDC3aZ6IAkBuqWQ2IDu7wo= -github.com/apache/thrift v0.17.0/go.mod h1:OLxhMRJxomX+1I/KUw03qoV3mMz16BwaKI+d4fPBx7Q= +github.com/apache/thrift v0.19.0 h1:sOqkWPzMj7w6XaYbJQG7m4sGqVolaW/0D28Ln7yPzMk= +github.com/apache/thrift v0.19.0/go.mod h1:SUALL216IiaOw2Oy+5Vs9lboJ/t9g40C+G07Dc0QC1I= github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= github.com/armon/go-metrics v0.3.3/go.mod h1:4O98XIr/9W0sxpJ8UaYkvjk10Iff7SnFrb4QAOwNTFc= @@ -246,11 +246,10 @@ github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDk github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= -github.com/cncf/udpa/go v0.0.0-20210930031921-04548b0d99d4/go.mod h1:6pvJx4me5XPnfI9Z40ddWsdw2W/uZgQLFXToKeRcDiI= github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20210805033703-aa0b78936158/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= -github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= -github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20231128003011-0fa0005c9caa h1:jQCWAUqqlij9Pgj2i/PB79y4KOPYVyFYdROxgaCwdTQ= +github.com/cncf/xds/go v0.0.0-20231128003011-0fa0005c9caa/go.mod h1:x/1Gn8zydmfq8dk6e9PdstVsDgu9RuyIIJqAaF//0IM= github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8= github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI= github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= @@ -301,11 +300,15 @@ github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.m github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.mod h1:hliV/p42l8fGbc6Y9bQ70uLwIvmJyVE5k4iMKlh8wCQ= github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go.mod h1:AFq3mo9L8Lqqiid3OhADV3RfLJnjiw63cSpi+fDTRC0= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/envoyproxy/protoc-gen-validate v1.0.4 h1:gVPz/FMfvh57HdSJQyvBtF00j8JU4zdyUgIUNhlgg0A= +github.com/envoyproxy/protoc-gen-validate v1.0.4/go.mod h1:qys6tmnRsYrQqIhm2bvKZH4Blx/1gTIZ2UKVY1M+Yew= github.com/evanphx/json-patch v4.2.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU= github.com/fatih/color v1.15.0 h1:kOqh6YHBtK8aywxGerMG2Eq3H6Qgoqeo13Bk2Mv/nBs= github.com/fatih/color v1.15.0/go.mod h1:0h5ZqXfHYED7Bhv2ZJamyIOUej9KtShiJESRwBDUSsw= +github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= +github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/fogleman/gg v1.2.1-0.20190220221249-0403632d5b90/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k= github.com/fogleman/gg v1.3.0/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k= github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= @@ -351,6 +354,11 @@ github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= github.com/go-logr/logr v0.4.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.3.0 h1:2y3SDp0ZXuc6/cjLSZ+Q3ir+QB9T/iG5yYRXqsagWSY= +github.com/go-logr/logr v1.3.0/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-openapi/analysis v0.0.0-20180825180245-b006789cd277/go.mod h1:k70tL6pCuVxPJOHXQ+wIac1FUrvNkHolPie/cLEU6hI= github.com/go-openapi/analysis v0.17.0/go.mod h1:IowGgpVeD0vNm45So8nr+IcQ3pxVtpRoBWb8PVZO0ik= github.com/go-openapi/analysis v0.18.0/go.mod h1:IowGgpVeD0vNm45So8nr+IcQ3pxVtpRoBWb8PVZO0ik= @@ -508,8 +516,8 @@ github.com/google/btree v1.0.1 h1:gK4Kx5IaGY9CD5sPJ36FHiBJ6ZXl0kilRiiCj+jdYp4= github.com/google/btree v1.0.1/go.mod h1:xXMiIv4Fb/0kKde4SpL7qlzvu5cMJDRkFDxJfI9uaxA= github.com/google/flatbuffers v1.11.0/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= github.com/google/flatbuffers v2.0.0+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= -github.com/google/flatbuffers v23.5.26+incompatible h1:M9dgRyhJemaM4Sw8+66GHBu8ioaQmyPLg1b8VwK5WJg= -github.com/google/flatbuffers v23.5.26+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= +github.com/google/flatbuffers v24.3.25+incompatible h1:CX395cjN9Kke9mmalRoL3d81AtFUxJM+yDthflgJGkI= +github.com/google/flatbuffers v24.3.25+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= @@ -522,8 +530,8 @@ github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= -github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPgecwXBIDzw5no= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= @@ -545,17 +553,17 @@ github.com/google/pprof v0.0.0-20210122040257-d980be63207e/go.mod h1:kpwsk12EmLe github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210506205249-923b5ab0fc1a/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= -github.com/google/s2a-go v0.1.4 h1:1kZ/sQM3srePvKs3tXAvQzo66XfcReoqFpIpIccE7Oc= -github.com/google/s2a-go v0.1.4/go.mod h1:Ej+mSEMGRnqRzjc7VtF+jdBwYG5fuJfiZ8ELkjEwM0A= +github.com/google/s2a-go v0.1.7 h1:60BLSyTrOV4/haCDW4zb1guZItoSq8foHCXrAnjBo/o= +github.com/google/s2a-go v0.1.7/go.mod h1:50CgR4k1jNlWBu4UfS4AcfhVe1r6pdZPygJ3R8F0Qdw= github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.3.1 h1:KjJaJ9iWZ3jOFZIf1Lqf4laDRCasjl0BCmnEGxkdLb4= -github.com/google/uuid v1.3.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/googleapis/enterprise-certificate-proxy v0.2.4 h1:uGy6JWR/uMIILU8wbf+OkstIrNiMjGpEIyhx8f6W7s4= -github.com/googleapis/enterprise-certificate-proxy v0.2.4/go.mod h1:AwSRAtLfXpU5Nm3pW+v7rGDHp09LsPtGY9MduiEsR9k= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/googleapis/enterprise-certificate-proxy v0.3.2 h1:Vie5ybvEvT75RniqhfFxPRy3Bf7vr3h0cechB90XaQs= +github.com/googleapis/enterprise-certificate-proxy v0.3.2/go.mod h1:VLSiSSBs/ksPL8kq3OBOQ6WRI2QnaFynd1DCjZ62+V0= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/googleapis/gax-go/v2 v2.12.0 h1:A+gCJKdRfqXkr+BIRGtZLibNXf0m1f9E4HG56etFpas= @@ -674,8 +682,9 @@ github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/u github.com/json-iterator/go v1.1.8/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= -github.com/json-iterator/go v1.1.11 h1:uVUAXhF2To8cbw/3xN3pxj6kk7TYKs98NIrTqPlMWAQ= github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= github.com/jsternberg/zap-logfmt v1.0.0/go.mod h1:uvPs/4X51zdkcm5jXl5SYoN+4RK21K8mysFmDaM/h+o= @@ -702,12 +711,12 @@ github.com/klauspost/compress v1.4.0/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0 github.com/klauspost/compress v1.9.5/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= github.com/klauspost/compress v1.13.1/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg= github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= -github.com/klauspost/compress v1.16.7 h1:2mk3MPGNzKyxErAw8YaohYh69+pa4sIQSC0fPGCFR9I= -github.com/klauspost/compress v1.16.7/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= +github.com/klauspost/compress v1.17.7 h1:ehO88t2UGzQK66LMdE8tibEd1ErmzZjNEqWkjLAKQQg= +github.com/klauspost/compress v1.17.7/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= github.com/klauspost/cpuid v0.0.0-20170728055534-ae7887de9fa5/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= -github.com/klauspost/cpuid/v2 v2.2.5 h1:0E5MSMDEoAulmXNFquVs//DdoomxaoTY1kUhbc/qbZg= -github.com/klauspost/cpuid/v2 v2.2.5/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= +github.com/klauspost/cpuid/v2 v2.2.7 h1:ZWSB3igEs+d0qvnxR/ZBzXVmxkgt8DdzP6m9pfuVLDM= +github.com/klauspost/cpuid/v2 v2.2.7/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= github.com/klauspost/crc32 v0.0.0-20161016154125-cb6bfca970f6 h1:KAZ1BW2TCmT6PRihDPpocIy1QTtsAsrx6TneU/4+CMg= github.com/klauspost/crc32 v0.0.0-20161016154125-cb6bfca970f6/go.mod h1:+ZoRqAPRLkC4NPOvfYeR5KNOrY6TD+/sAC3HXPZgDYg= github.com/klauspost/pgzip v1.0.2-0.20170402124221-0bf5dcad4ada h1:3L+neHp83cTjegPdCiOxVOJtRIy7/8RldvMTsyPYH10= @@ -799,8 +808,9 @@ github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= -github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc= github.com/mschoch/smat v0.0.0-20160514031455-90eadee771ae h1:VeRdUYdCw49yizlSbMEn2SZ+gT+3IUKx8BqxyQdz+BY= github.com/mschoch/smat v0.0.0-20160514031455-90eadee771ae/go.mod h1:qAyveg+e4CE+eKJXWVjKXM4ck2QobLqTDytGJbLLhJg= @@ -861,8 +871,8 @@ github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi github.com/pierrec/lz4/v4 v4.1.8/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= github.com/pierrec/lz4/v4 v4.1.11/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= github.com/pierrec/lz4/v4 v4.1.12/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= -github.com/pierrec/lz4/v4 v4.1.18 h1:xaKrnTkyoqfh1YItXl56+6KJNVYWlEEPuAQW9xsplYQ= -github.com/pierrec/lz4/v4 v4.1.18/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= +github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ= +github.com/pierrec/lz4/v4 v4.1.21/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 h1:KoWmjvw+nsYOo29YJK9vDA65RGE3NrOnUtO7a+RF9HU= github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8/go.mod h1:HKlIX3XHQyzLZPlr7++PzdhaXEj94dEiJgZDTsxEqUI= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -974,8 +984,9 @@ github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+ github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= -github.com/stretchr/objx v0.5.0 h1:1zr/of2m5FGMsad5YfcqgdqdWrIhu+EBEJRhR1U7z/c= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= +github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/testify v1.2.0/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= @@ -987,8 +998,8 @@ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.2/go.mod h1:R6va5+xMeoiuVRoj+gSkQ7d3FALtqAAGI1FQKckRals= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= -github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= github.com/tinylib/msgp v1.0.2/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE= github.com/tinylib/msgp v1.1.0 h1:9fQd+ICuRIu/ue4vxJZu6/LzxN0HwMds2nq/0cFvxHU= @@ -1051,11 +1062,23 @@ go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.46.1 h1:SpGay3w+nEwMpfVnbqOLH5gY52/foP8RE8UzTZ1pdSE= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.46.1/go.mod h1:4UoMYEZOC0yN/sPGH76KPkkU7zgiEWYWL9vwmbnTJPE= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.46.1 h1:aFJWCqJMNjENlcleuuOkGAPH82y0yULBScfXcIEdS24= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.46.1/go.mod h1:sEGXWArGqc3tVa+ekntsN65DmVbVeW+7lTKTjZF3/Fo= go.opentelemetry.io/otel v0.20.0/go.mod h1:Y3ugLH2oa81t5QO+Lty+zXf8zC9L26ax4Nzoxm/dooo= +go.opentelemetry.io/otel v1.21.0 h1:hzLeKBZEL7Okw2mGzZ0cc4k/A7Fta0uoPgaJCr8fsFc= +go.opentelemetry.io/otel v1.21.0/go.mod h1:QZzNPQPm1zLX4gZK4cMi+71eaorMSGT3A4znnUvNNEo= go.opentelemetry.io/otel/metric v0.20.0/go.mod h1:598I5tYlH1vzBjn+BTuhzTCSb/9debfNp6R3s7Pr1eU= +go.opentelemetry.io/otel/metric v1.21.0 h1:tlYWfeo+Bocx5kLEloTjbcDwBuELRrIFxwdQ36PlJu4= +go.opentelemetry.io/otel/metric v1.21.0/go.mod h1:o1p3CA8nNHW8j5yuQLdc1eeqEaPfzug24uvsyIEJRWM= go.opentelemetry.io/otel/oteltest v0.20.0/go.mod h1:L7bgKf9ZB7qCwT9Up7i9/pn0PWIa9FqQ2IQ8LoxiGnw= go.opentelemetry.io/otel/sdk v0.20.0/go.mod h1:g/IcepuwNsoiX5Byy2nNV0ySUF1em498m7hBWC279Yc= +go.opentelemetry.io/otel/sdk v1.21.0 h1:FTt8qirL1EysG6sTQRZ5TokkU8d0ugCj8htOgThZXQ8= +go.opentelemetry.io/otel/sdk v1.21.0/go.mod h1:Nna6Yv7PWTdgJHVRD9hIYywQBRx7pbox6nwBnZIxl/E= go.opentelemetry.io/otel/trace v0.20.0/go.mod h1:6GjCW8zgDjwGHGa6GkyeB8+/5vjT16gUEi0Nf1iBdgw= +go.opentelemetry.io/otel/trace v1.21.0 h1:WD9i5gzvoUPuXIXH24ZNBudiarZDKuekPqi/E8fpfLc= +go.opentelemetry.io/otel/trace v1.21.0/go.mod h1:LGbsEB0f9LGjN+OZaQQ26sohbOmiMR+BaslueVtS/qQ= go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= @@ -1102,7 +1125,6 @@ golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20211117183948-ae814b36b871/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.0.0-20220314234659-1baeb1ce4c0b/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.21.0 h1:X31++rzVUdKhX5sWmSOFZxx8UW/ldWx55cbf08iNAMA= golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs= golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -1121,8 +1143,8 @@ golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u0 golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= golang.org/x/exp v0.0.0-20211216164055-b2b84827b756/go.mod h1:b9TAUYHmRtqA6klRHApnXMnj+OyLce4yF5cZCUbk2ps= -golang.org/x/exp v0.0.0-20231214170342-aacd6d4b4611 h1:qCEDpW1G+vcj3Y7Fy52pEM1AWm3abj8WimGYejI3SC4= -golang.org/x/exp v0.0.0-20231214170342-aacd6d4b4611/go.mod h1:iRJReGqOEeBhDZGkGbynYwcHlctCvnjTYIamk7uXpHI= +golang.org/x/exp v0.0.0-20240222234643-814bf88cf225 h1:LfspQV/FYTatPTr/3HzIcmiUFH7PGP+OQ6mgDYo3yuQ= +golang.org/x/exp v0.0.0-20240222234643-814bf88cf225/go.mod h1:CxmFvTBINI24O/j8iY7H1xHzx2i4OsyguNBmN/uPtqc= golang.org/x/image v0.0.0-20180708004352-c73c2afc3b81/go.mod h1:ux5Hcp/YLpHSI86hEcLt0YII63i6oz57MZXIpbrjZUs= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= @@ -1162,8 +1184,8 @@ golang.org/x/mod v0.5.1/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro= golang.org/x/mod v0.6.0-dev.0.20211013180041-c96bc1413d57/go.mod h1:3p9vT2HGsQu2K1YbXdKPJLVgG5VJdoTa1poYQBtP1AY= golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3/go.mod h1:3p9vT2HGsQu2K1YbXdKPJLVgG5VJdoTa1poYQBtP1AY= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= -golang.org/x/mod v0.14.0 h1:dGoOF9QVLYng8IHTm7BAyWqCqSheQ5pYWGhzW00YJr0= -golang.org/x/mod v0.14.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/mod v0.16.0 h1:QX4fJ0Rr5cPQCF7O9lh9Se4pmwfwskqZfq5moyldzic= +golang.org/x/mod v0.16.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/net v0.0.0-20170114055629-f2499483f923/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -1239,8 +1261,8 @@ golang.org/x/oauth2 v0.0.0-20210220000619-9bb904979d93/go.mod h1:KelEdhl1UZF7XfJ golang.org/x/oauth2 v0.0.0-20210313182246-cd4f82c27b84/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210427180440-81ed05c6b58c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.10.0 h1:zHCpF2Khkwy4mMB4bv0U37YtJdTGW8jI0glAApi0Kh8= -golang.org/x/oauth2 v0.10.0/go.mod h1:kTpgurOux7LqtuxjuyZa4Gj2gdezIt/jQtGnNFfypQI= +golang.org/x/oauth2 v0.16.0 h1:aDkGMBSYxElaoP81NpoUoz2oo2R2wHdZpGToUxfyQrQ= +golang.org/x/oauth2 v0.16.0/go.mod h1:hqZ+0LWXsiVoZpeld6jVt06P3adbS2Uu911W1SsJv2o= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -1254,8 +1276,8 @@ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.5.0 h1:60k92dhOjHxJkrqnwsfl8KuaHbn/5dl0lUPUklKo3qE= -golang.org/x/sync v0.5.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.6.0 h1:5BMeUDZ7vkXGfEr1x9B4bRcTH4lpkTkpdh0T/J+qjbQ= +golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20170830134202-bb24a47a89ea/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -1373,8 +1395,9 @@ golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxb golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20200416051211-89c76fbcd5d1/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20201208040808-7e3f01d25324/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba h1:O8mE0/t419eoIwhTFpKVkHiTs/Igowgfkj25AcZrtiE= golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= +golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180525024113-a5b4c53f6e8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -1454,22 +1477,22 @@ golang.org/x/tools v0.1.4/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.8-0.20211029000441-d6a9af8af023/go.mod h1:nABZi5QlRsZVlzPpHl034qft6wpY4eDcsTt5AaioBiU= golang.org/x/tools v0.1.10/go.mod h1:Uh6Zz+xoGYZom868N8YTex3t7RhtHDBrE8Gzo9bV56E= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= -golang.org/x/tools v0.16.0 h1:GO788SKMRunPIBCXiQyo2AaexLstOrVhuAL5YwsckQM= -golang.org/x/tools v0.16.0/go.mod h1:kYVVN6I1mBNoB1OX+noeBjbRk4IUEPa7JJ+TJMEooJ0= +golang.org/x/tools v0.19.0 h1:tfGCXNR1OsFG+sVdLAitlpjAvD/I6dHDKnYrpEZUHkw= +golang.org/x/tools v0.19.0/go.mod h1:qoJWxmGSIBmAeriMx19ogtrEPrGtDbPK634QFIcLAhc= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20220411194840-2f41105eb62f/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 h1:H2TDz8ibqkAF6YGhCdN3jS9O0/s90v0rJh3X/OLHEUk= -golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= +golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 h1:+cNy6SZtPcJQH3LJVLOSmiC7MMxXNOb3PU/VUEz+EhU= +golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028/go.mod h1:NDW/Ps6MPRej6fsCIbMTohpP40sJ/P/vI1MoTEGwX90= gonum.org/v1/gonum v0.0.0-20180816165407-929014505bf4/go.mod h1:Y+Yx5eoAFn32cQvJDxZx5Dpnq+c3wtXuadVZAcxbbBo= gonum.org/v1/gonum v0.0.0-20181121035319-3f7ecaa7e8ca/go.mod h1:Y+Yx5eoAFn32cQvJDxZx5Dpnq+c3wtXuadVZAcxbbBo= gonum.org/v1/gonum v0.6.0/go.mod h1:9mxDZsDKxgMAuccQkewq682L+0eCu4dCN2yonUJTCLU= gonum.org/v1/gonum v0.8.2/go.mod h1:oe/vMfY3deqTw+1EZJhuvEW2iwGF1bW9wwu7XCu0+v0= gonum.org/v1/gonum v0.9.3/go.mod h1:TZumC3NeyVQskjXqmyWt4S3bINhy7B4eYwW69EbyX+0= -gonum.org/v1/gonum v0.12.0 h1:xKuo6hzt+gMav00meVPUlXwSdoEJP46BR+wdxQEFK2o= -gonum.org/v1/gonum v0.12.0/go.mod h1:73TDxJfAAHeA8Mk9mf8NlIppyhQNo5GLTcYeqgo2lvY= +gonum.org/v1/gonum v0.15.0 h1:2lYxjRbTYyxkJxlhC+LvJIx3SsANPdRybu1tGj9/OrQ= +gonum.org/v1/gonum v0.15.0/go.mod h1:xzZVBJBtS+Mz4q0Yl2LJTk+OxOg4jiXZ7qBoM0uISGo= gonum.org/v1/netlib v0.0.0-20181029234149-ec6d1f5cefe6/go.mod h1:wa6Ws7BG/ESfp6dHfk7C6KdzKA7wR7u/rKwOGE66zvw= gonum.org/v1/netlib v0.0.0-20190313105609-8cb42192e0e0/go.mod h1:wa6Ws7BG/ESfp6dHfk7C6KdzKA7wR7u/rKwOGE66zvw= gonum.org/v1/plot v0.0.0-20190515093506-e2840ee46a6b/go.mod h1:Wt8AAjI+ypCyYX3nZBvf6cAIx93T+c/OS2HFAYskSZc= @@ -1498,8 +1521,8 @@ google.golang.org/api v0.41.0/go.mod h1:RkxM5lITDfTzmyKFPt+wGrCJbVfniCr2ool8kTBz google.golang.org/api v0.43.0/go.mod h1:nQsDGjRXMo4lvh5hP0TKqF244gqhGcr/YSIykhUk/94= google.golang.org/api v0.46.0/go.mod h1:ceL4oozhkAiTID8XMmJBsIxID/9wMXJVVFXPg4ylg3I= google.golang.org/api v0.47.0/go.mod h1:Wbvgpq1HddcWVtzsVLyfLp8lDg6AA241LmgIL59tHXo= -google.golang.org/api v0.128.0 h1:RjPESny5CnQRn9V6siglged+DZCgfu9l6mO9dkX9VOg= -google.golang.org/api v0.128.0/go.mod h1:Y611qgqaE92On/7g65MQgxYul3c0rEB894kniWLY750= +google.golang.org/api v0.155.0 h1:vBmGhCYs0djJttDNynWo44zosHlPvHmA0XiN2zP2DtA= +google.golang.org/api v0.155.0/go.mod h1:GI5qK5f40kCpHfPn6+YzGAByIKWv8ujFnmoWm7Igduk= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.2.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= @@ -1507,8 +1530,9 @@ google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7 google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= -google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/appengine v1.6.8 h1:IhEN5q69dyKagZPYMSdIjS2HqprW324FRQZJcGqPAsM= +google.golang.org/appengine v1.6.8/go.mod h1:1jJ3jBArFh5pcgW8gCtRJnepW8FzD1V44FJffLiz/Ds= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= @@ -1559,12 +1583,12 @@ google.golang.org/genproto v0.0.0-20210513213006-bf773b8c8384/go.mod h1:P3QM42oQ google.golang.org/genproto v0.0.0-20210517163617-5e0236093d7a/go.mod h1:P3QM42oQyzQSnHPnZ/vqoCdDmzH28fzWByN9asMeM8A= google.golang.org/genproto v0.0.0-20210601144548-a796c710e9b6/go.mod h1:P3QM42oQyzQSnHPnZ/vqoCdDmzH28fzWByN9asMeM8A= google.golang.org/genproto v0.0.0-20210630183607-d20f26d13c79/go.mod h1:yiaVoXHpRzHGyxV3o4DktVWY4mSUErTKaeEOq6C3t3U= -google.golang.org/genproto v0.0.0-20230920204549-e6e6cdab5c13 h1:vlzZttNJGVqTsRFU9AmdnrcO1Znh8Ew9kCD//yjigk0= -google.golang.org/genproto v0.0.0-20230920204549-e6e6cdab5c13/go.mod h1:CCviP9RmpZ1mxVr8MUjCnSiY09IbAXZxhLE6EhHIdPU= -google.golang.org/genproto/googleapis/api v0.0.0-20230913181813-007df8e322eb h1:lK0oleSc7IQsUxO3U5TjL9DWlsxpEBemh+zpB7IqhWI= -google.golang.org/genproto/googleapis/api v0.0.0-20230913181813-007df8e322eb/go.mod h1:KjSP20unUpOx5kyQUFa7k4OJg0qeJ7DEZflGDu2p6Bk= -google.golang.org/genproto/googleapis/rpc v0.0.0-20231002182017-d307bd883b97 h1:6GQBEOdGkX6MMTLT9V+TjtIRZCw9VPD5Z+yHY9wMgS0= -google.golang.org/genproto/googleapis/rpc v0.0.0-20231002182017-d307bd883b97/go.mod h1:v7nGkzlmW8P3n/bKmWBn2WpBjpOEx8Q6gMueudAmKfY= +google.golang.org/genproto v0.0.0-20240123012728-ef4313101c80 h1:KAeGQVN3M9nD0/bQXnr/ClcEMJ968gUXJQ9pwfSynuQ= +google.golang.org/genproto v0.0.0-20240123012728-ef4313101c80/go.mod h1:cc8bqMqtv9gMOr0zHg2Vzff5ULhhL2IXP4sbcn32Dro= +google.golang.org/genproto/googleapis/api v0.0.0-20240123012728-ef4313101c80 h1:Lj5rbfG876hIAYFjqiJnPHfhXbv+nzTWfm04Fg/XSVU= +google.golang.org/genproto/googleapis/api v0.0.0-20240123012728-ef4313101c80/go.mod h1:4jWUdICTdgc3Ibxmr8nAJiiLHwQBY0UI0XZcEMaFKaA= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240123012728-ef4313101c80 h1:AjyfHzEPEFp/NpvfN5g+KDla3EMojjhRVZc1i7cj+oM= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240123012728-ef4313101c80/go.mod h1:PAREbraiVEVGVdTZsVWjSbbTtSyGbAgIIvni8a8CD5s= google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.0/go.mod h1:chYK+tFQF0nDUGJgXMSgLCQk3phJEuONr2DCgLDdAQM= @@ -1596,9 +1620,8 @@ google.golang.org/grpc v1.37.1/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQ google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= google.golang.org/grpc v1.39.0/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnDzfrE= google.golang.org/grpc v1.41.0/go.mod h1:U3l9uK9J0sini8mHphKoXyaqDA/8VyGnDee1zzIUK6k= -google.golang.org/grpc v1.45.0/go.mod h1:lN7owxKUQEqMfSyQikvvk5tf/6zMPsrK+ONuO11+0rQ= -google.golang.org/grpc v1.58.2 h1:SXUpjxeVF3FKrTYQI4f4KvbGD5u2xccdYdurwowix5I= -google.golang.org/grpc v1.58.2/go.mod h1:tgX3ZQDlNJGU96V6yHh1T/JeoBQ2TXdr43YbYSsCJk0= +google.golang.org/grpc v1.62.1 h1:B4n+nfKzOICUXMgyrNd19h/I9oH0L1pizfk1d4zSgTk= +google.golang.org/grpc v1.62.1/go.mod h1:IWTG0VlJLCh1SkC58F7np9ka9mx/WNkjl4PGJaiq+QE= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= From 50f45115176619354f61a08b262bf10c3b4ab9b0 Mon Sep 17 00:00:00 2001 From: Ales Pour Date: Thu, 13 Jun 2024 10:00:07 +0200 Subject: [PATCH 08/16] docs: update with new influx_inspect options and sample command --- cmd/influx_inspect/README.md | 30 +++++++++++++++++++++++++++--- 1 file changed, 27 insertions(+), 3 deletions(-) diff --git a/cmd/influx_inspect/README.md b/cmd/influx_inspect/README.md index bcdf4185ada..bd812ea6c90 100644 --- a/cmd/influx_inspect/README.md +++ b/cmd/influx_inspect/README.md @@ -49,17 +49,26 @@ WAL storage path. `default` = "$HOME/.influxdb/wal" #### `-out` string -Destination file to export to +Destination file to export to. +In case of export to Parquet, destination should be existing directory. `default` = "$HOME/.influxdb/export" -#### `-database` string (optional) +#### `-database` string Database to export. +Mandatory for export to Parquet, optional otherwise (default). `default` = "" -#### `-retention` string (optional) +#### `-retention` string Retention policy to export. +Mandatory for export to Parquet, optional otherwise (default). + +`default` = "" + +#### `-measurement` string +Name of the measurement to export. +Mandatory for export to Parquet, optional otherwise (default). `default` = "" @@ -74,6 +83,16 @@ Compress the output. `default` = false +#### `-parquet` bool (optional) +Export to Parquet. + +`default` = false + +#### `-chunk-size` int (optional) +Size to partition Parquet files, in bytes. + +`default` = 100000000 + #### Sample Commands Export entire database and compress output: @@ -86,6 +105,11 @@ Export specific retention policy: influx_inspect export --database mydb --retention autogen ``` +Export specific measurement to Parquet: +``` +influx_inspect export --database mydb --retention autogen --measurement cpu --parquet +``` + ##### Sample Data This is a sample of what the output will look like. From db4bb2aaa970ca682e626a28b561ef877a56ffba Mon Sep 17 00:00:00 2001 From: Ales Pour Date: Wed, 24 Jul 2024 10:30:49 +0200 Subject: [PATCH 09/16] refactor: use constants instead of literal values --- cmd/influx_inspect/export/export.go | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/cmd/influx_inspect/export/export.go b/cmd/influx_inspect/export/export.go index 3cdb23b9a9d..c9b5b12561e 100644 --- a/cmd/influx_inspect/export/export.go +++ b/cmd/influx_inspect/export/export.go @@ -22,6 +22,11 @@ import ( "github.com/influxdata/influxql" ) +const ( + DefaultParquetPartitionSize = 100_000_000 + MinParquetPartitionSize = 1_000_000 +) + // Command represents the program execution for "influx_inspect export". type Command struct { // Standard input/output, overridden for testing. @@ -83,7 +88,7 @@ func (cmd *Command) Run(args ...string) error { fs.BoolVar(&cmd.lponly, "lponly", false, "Only export line protocol") fs.BoolVar(&cmd.compress, "compress", false, "Compress the output") fs.BoolVar(&cmd.parquet, "parquet", false, "Export to Parquet format (requires -database -retention -measurement)") - fs.IntVar(&cmd.pqChunkSize, "chunk-size", 100_000_000, "Size to partition Parquet files (in bytes)") + fs.IntVar(&cmd.pqChunkSize, "chunk-size", DefaultParquetPartitionSize, "Size to partition Parquet files (in bytes)") fs.SetOutput(cmd.Stdout) fs.Usage = func() { @@ -146,8 +151,8 @@ func (cmd *Command) validate() error { if cmd.out == "-" { return fmt.Errorf("-out must point to a folder for Parquet files") } - if cmd.pqChunkSize < 1_000_000 { - return fmt.Errorf("minimum Parquet partition size is 1000000 bytes") + if cmd.pqChunkSize < MinParquetPartitionSize { + return fmt.Errorf("minimum Parquet partition size is %d bytes", MinParquetPartitionSize) } } return nil From 05c1e50779b53e5046b15622ff21f21130e79f25 Mon Sep 17 00:00:00 2001 From: Ales Pour Date: Wed, 24 Jul 2024 11:46:11 +0200 Subject: [PATCH 10/16] fix: use errors.Capture to handle errors in defer calls --- cmd/influx_inspect/export/export_parquet.go | 29 +++++++-------------- 1 file changed, 10 insertions(+), 19 deletions(-) diff --git a/cmd/influx_inspect/export/export_parquet.go b/cmd/influx_inspect/export/export_parquet.go index 61b39ee7a7d..eb1647465c9 100644 --- a/cmd/influx_inspect/export/export_parquet.go +++ b/cmd/influx_inspect/export/export_parquet.go @@ -17,6 +17,7 @@ import ( "github.com/influxdata/influxdb/cmd/influx_inspect/export/parquet/resultset" "github.com/influxdata/influxdb/cmd/influx_inspect/export/parquet/table" tsdb "github.com/influxdata/influxdb/cmd/influx_inspect/export/parquet/tsm1" + "github.com/influxdata/influxdb/pkg/errors" "github.com/influxdata/influxdb/tsdb/engine/tsm1" "go.uber.org/zap" "golang.org/x/exp/maps" @@ -104,7 +105,11 @@ func (cmd *Command) writeToParquet(rs resultset.ResultSet, schema *index.Measure "stdout", } } - log, _ := cfg.Build() + + log, err := cfg.Build() + if err != nil { + return fmt.Errorf("failed creating logger instance: %w", err) + } log.Info("Generating parquet", zap.Int("chunk_size", cmd.pqChunkSize)) @@ -170,16 +175,9 @@ func (cmd *Command) writeToParquet(rs resultset.ResultSet, schema *index.Measure fileName := filepath.Join(cmd.out, fmt.Sprintf("table-%05d.parquet", sequence)) parquetFile, err = os.Create(fileName) if err != nil { - return false, err + return false, fmt.Errorf("failed creating parquet file %s for export: %w", fileName, err) } - defer func() { - closeErr := parquetFile.Close() - if err == nil { - // we only care if the file fails to close when there was no - // previous error - err = closeErr - } - }() + defer errors.Capture(&err, parquetFile.Close) } var metaFile *os.File @@ -187,16 +185,9 @@ func (cmd *Command) writeToParquet(rs resultset.ResultSet, schema *index.Measure fileName := filepath.Join(cmd.out, fmt.Sprintf("table-%05d.meta.json", sequence)) metaFile, err = os.Create(fileName) if err != nil { - return false, err + return false, fmt.Errorf("failed creating parquet meta file %s: %w", fileName, err) } - defer func() { - closeErr := metaFile.Close() - if err == nil { - // we only care if the file fails to close when there was no - // previous error - err = closeErr - } - }() + defer errors.Capture(&err, metaFile.Close) } stats := t.Stats() From 64829118395030875971b0a7159dce6852671508 Mon Sep 17 00:00:00 2001 From: Ales Pour Date: Wed, 24 Jul 2024 11:46:57 +0200 Subject: [PATCH 11/16] fix: use errors.Capture to handle errors in defer calls --- cmd/influx_inspect/export/parquet/exporter/exporter.go | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/cmd/influx_inspect/export/parquet/exporter/exporter.go b/cmd/influx_inspect/export/parquet/exporter/exporter.go index 9ccacebd652..1336f91b25e 100644 --- a/cmd/influx_inspect/export/parquet/exporter/exporter.go +++ b/cmd/influx_inspect/export/parquet/exporter/exporter.go @@ -10,6 +10,7 @@ import ( "github.com/apache/arrow/go/v16/parquet/pqarrow" "github.com/influxdata/influxdb/cmd/influx_inspect/export/parquet/models" "github.com/influxdata/influxdb/cmd/influx_inspect/export/parquet/table" + errors2 "github.com/influxdata/influxdb/pkg/errors" "github.com/pkg/errors" ) @@ -92,14 +93,7 @@ func (e *Exporter) WriteChunk(ctx context.Context, parquet io.Writer, metaW io.W if err != nil { return false, err } - defer func() { - closeErr := fw.Close() - if err == nil { - // we only care if the file fails to close when there was no - // previous error - err = closeErr - } - }() + defer errors2.Capture(&err, fw.Close) var ( firstSeriesKey models.Escaped From c1e50bb02dee82094b6c9b2a3a0df4214ddee487 Mon Sep 17 00:00:00 2001 From: Ales Pour Date: Wed, 24 Jul 2024 11:47:36 +0200 Subject: [PATCH 12/16] fix: wrap original error --- cmd/influx_inspect/export/parquet/exporter/meta.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/cmd/influx_inspect/export/parquet/exporter/meta.go b/cmd/influx_inspect/export/parquet/exporter/meta.go index 3c3182b628b..997a49d392a 100644 --- a/cmd/influx_inspect/export/parquet/exporter/meta.go +++ b/cmd/influx_inspect/export/parquet/exporter/meta.go @@ -3,7 +3,7 @@ package exporter import ( "bytes" "encoding/base64" - "errors" + "fmt" "io" "github.com/influxdata/influxdb/cmd/influx_inspect/export/parquet/models" @@ -35,11 +35,11 @@ func (m *Meta) UnmarshalJSON(data []byte) error { firstSeriesKey, err := base64.StdEncoding.DecodeString(tmp.FirstSeriesKey) if err != nil { - return errors.New("failed to decode firstSeriesKeyEscaped") + return fmt.Errorf("failed to decode firstSeriesKeyEscaped: %w", err) } lastSeriesKey, err := base64.StdEncoding.DecodeString(tmp.LastSeriesKey) if err != nil { - return errors.New("failed to decode lastSeriesKeyEscaped") + return fmt.Errorf("failed to decode lastSeriesKeyEscaped: %w", err) } m.FirstSeriesKey = models.MakeEscaped(firstSeriesKey) @@ -50,7 +50,7 @@ func (m *Meta) UnmarshalJSON(data []byte) error { return nil } -func (m Meta) MarshalJSON() ([]byte, error) { +func (m *Meta) MarshalJSON() ([]byte, error) { var b bytes.Buffer enc := jsoniter.NewEncoder(&b) err := enc.Encode(meta{ From b48a1ca6fd9be88b6132eab91648710fc3f2c820 Mon Sep 17 00:00:00 2001 From: Ales Pour Date: Wed, 24 Jul 2024 12:11:45 +0200 Subject: [PATCH 13/16] fix: print buffer inside error message --- cmd/influx_inspect/export/parquet/models/points.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/influx_inspect/export/parquet/models/points.go b/cmd/influx_inspect/export/parquet/models/points.go index 9fffd67b02c..2867b4b11d8 100644 --- a/cmd/influx_inspect/export/parquet/models/points.go +++ b/cmd/influx_inspect/export/parquet/models/points.go @@ -259,14 +259,14 @@ func scanMeasurement(bufe Escaped, i int) (int, int, error) { // It can't be a space, since whitespace is stripped prior to this // function call. if i >= len(buf) || buf[i] == ',' { - return -1, i, fmt.Errorf("missing measurement") + return -1, i, fmt.Errorf("missing measurement in: %q", buf) } for { i++ if i >= len(buf) { // cpu - return -1, i, fmt.Errorf("missing fields") + return -1, i, fmt.Errorf("missing fields in: %d", buf) } if buf[i-1] == '\\' { From 54906a5d99eac76b9be862dc1469ac801070adf8 Mon Sep 17 00:00:00 2001 From: Ales Pour Date: Wed, 24 Jul 2024 14:24:47 +0200 Subject: [PATCH 14/16] fix: print buffer inside error message --- cmd/influx_inspect/export/parquet/models/points.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/influx_inspect/export/parquet/models/points.go b/cmd/influx_inspect/export/parquet/models/points.go index 2867b4b11d8..033b7def4fe 100644 --- a/cmd/influx_inspect/export/parquet/models/points.go +++ b/cmd/influx_inspect/export/parquet/models/points.go @@ -259,14 +259,14 @@ func scanMeasurement(bufe Escaped, i int) (int, int, error) { // It can't be a space, since whitespace is stripped prior to this // function call. if i >= len(buf) || buf[i] == ',' { - return -1, i, fmt.Errorf("missing measurement in: %q", buf) + return -1, i, fmt.Errorf("missing measurement in: %q", string(buf)) } for { i++ if i >= len(buf) { // cpu - return -1, i, fmt.Errorf("missing fields in: %d", buf) + return -1, i, fmt.Errorf("missing fields in: %q", string(buf)) } if buf[i-1] == '\\' { From 5b5d6ee724e5255426d59820c07f90e26647064a Mon Sep 17 00:00:00 2001 From: Ales Pour Date: Wed, 24 Jul 2024 14:26:18 +0200 Subject: [PATCH 15/16] fix: check for empty input --- cmd/influx_inspect/export/export_parquet.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/cmd/influx_inspect/export/export_parquet.go b/cmd/influx_inspect/export/export_parquet.go index eb1647465c9..9eebb03f27c 100644 --- a/cmd/influx_inspect/export/export_parquet.go +++ b/cmd/influx_inspect/export/export_parquet.go @@ -54,6 +54,10 @@ func (cmd *Command) writeValuesParquet(_ io.Writer, seriesKey []byte, field stri } func (cmd *Command) exportDoneParquet(_ string) error { + if len(vc) == 0 { + return nil + } + defer func() { vc = nil }() @@ -87,7 +91,7 @@ func (cmd *Command) exportDoneParquet(_ string) error { TagSet: tagSet, FieldSet: fieldSet, } - // schema does not change in a table + // schema does not change in a table in one tsm file break } From b09c14e2a379963b37f2820a1c34d768fc1e8635 Mon Sep 17 00:00:00 2001 From: Ales Pour Date: Wed, 24 Jul 2024 14:27:11 +0200 Subject: [PATCH 16/16] test: assert and cleanaup properly --- .../export/export_parquet_test.go | 31 +++++++++++-------- 1 file changed, 18 insertions(+), 13 deletions(-) diff --git a/cmd/influx_inspect/export/export_parquet_test.go b/cmd/influx_inspect/export/export_parquet_test.go index 4fb76c97c9e..ae9c4df222e 100644 --- a/cmd/influx_inspect/export/export_parquet_test.go +++ b/cmd/influx_inspect/export/export_parquet_test.go @@ -5,6 +5,7 @@ import ( "os" "testing" + "github.com/influxdata/influxdb/pkg/testing/assert" "github.com/influxdata/influxdb/tsdb/engine/tsm1" ) @@ -62,20 +63,24 @@ func TestExportParquet(t *testing.T) { }{ {corpus: myCorpus}, } { - tsmFile := writeCorpusToTSMFile(c.corpus) - defer os.Remove(tsmFile.Name()) - outdir := t.TempDir() + func() { + tsmFile := writeCorpusToTSMFile(c.corpus) + defer func() { + assert.NoError(t, os.Remove(tsmFile.Name())) + }() + out := t.TempDir() - cmd := newCommand() - cmd.out = outdir - cmd.writeValues = cmd.writeValuesParquet - cmd.exportDone = cmd.exportDoneParquet - cmd.measurement = "mym" - cmd.parquet = true - cmd.pqChunkSize = 100_000_000 + cmd := newCommand() + cmd.out = out + cmd.writeValues = cmd.writeValuesParquet + cmd.exportDone = cmd.exportDoneParquet + cmd.measurement = "mym" + cmd.parquet = true + cmd.pqChunkSize = 100_000_000 - if err := cmd.writeTsmFiles(io.Discard, io.Discard, []string{tsmFile.Name()}); err != nil { - t.Fatal(err) - } + t.Logf("cmd.writeTsmFiles: []{%s}", tsmFile.Name()) + err := cmd.writeTsmFiles(io.Discard, io.Discard, []string{tsmFile.Name()}) + assert.NoError(t, err) + }() } }