From 46aef0b95b2f55755fb6f4d6ad937a18b29d13d1 Mon Sep 17 00:00:00 2001 From: Sven Rebhan Date: Wed, 18 Sep 2024 12:41:48 +0200 Subject: [PATCH 01/14] feat(influx_tools): Add export to parquet files --- cmd/influx_tools/main.go | 6 + cmd/influx_tools/parquet/batcher.go | 236 ++++++++++++ cmd/influx_tools/parquet/command.go | 101 +++++ cmd/influx_tools/parquet/converters.go | 121 ++++++ cmd/influx_tools/parquet/exporter.go | 488 +++++++++++++++++++++++++ cmd/influx_tools/parquet/schema.go | 273 ++++++++++++++ go.mod | 83 +++-- go.sum | 185 ++++++---- 8 files changed, 1390 insertions(+), 103 deletions(-) create mode 100644 cmd/influx_tools/parquet/batcher.go create mode 100644 cmd/influx_tools/parquet/command.go create mode 100644 cmd/influx_tools/parquet/converters.go create mode 100644 cmd/influx_tools/parquet/exporter.go create mode 100644 cmd/influx_tools/parquet/schema.go diff --git a/cmd/influx_tools/main.go b/cmd/influx_tools/main.go index 7d50ce80e23..28a2e89aa79 100644 --- a/cmd/influx_tools/main.go +++ b/cmd/influx_tools/main.go @@ -15,6 +15,7 @@ import ( geninit "github.com/influxdata/influxdb/cmd/influx_tools/generate/init" "github.com/influxdata/influxdb/cmd/influx_tools/help" "github.com/influxdata/influxdb/cmd/influx_tools/importer" + "github.com/influxdata/influxdb/cmd/influx_tools/parquet" "github.com/influxdata/influxdb/cmd/influx_tools/server" "github.com/influxdata/influxdb/cmd/influxd/run" "github.com/influxdata/influxdb/services/meta" @@ -67,6 +68,11 @@ func (m *Main) Run(args ...string) error { if err := c.Run(args); err != nil { return fmt.Errorf("export failed: %s", err) } + case "export-parquet": + c := parquet.NewCommand(&ossServer{logger: zap.NewNop()}) + if err := c.Run(args); err != nil { + return fmt.Errorf("export failed: %s", err) + } case "import": c := importer.NewCommand(&ossServer{logger: zap.NewNop()}) if err := c.Run(args); err != nil { diff --git a/cmd/influx_tools/parquet/batcher.go b/cmd/influx_tools/parquet/batcher.go new file mode 100644 index 00000000000..43c8af25fd3 --- /dev/null +++ b/cmd/influx_tools/parquet/batcher.go @@ -0,0 +1,236 @@ +package parquet + +import ( + "context" + "fmt" + "os" + "sort" + + "github.com/influxdata/influxdb/models" + "github.com/influxdata/influxdb/tsdb" + "github.com/influxdata/influxql" +) + +type row struct { + timestamp int64 + tags map[string]string + fields map[string]interface{} +} + +type batcher struct { + measurement []byte + shard *tsdb.Shard + + converter map[string]func(interface{}) (interface{}, error) + nameMapping map[string]string + + series []seriesEntry + start int64 +} + +func newBatcher( + ctx context.Context, + shard *tsdb.Shard, + measurement string, + series []seriesEntry, + converter map[string]func(interface{}) (interface{}, error), + nameMapping map[string]string, +) (*batcher, error) { + seriesCursor, err := shard.CreateSeriesCursor( + ctx, + tsdb.SeriesCursorRequest{}, + influxql.MustParseExpr("_name = '"+measurement+"'"), + ) + if err != nil { + return nil, fmt.Errorf("getting series cursor failed: %w", err) + } + defer seriesCursor.Close() + + return &batcher{ + measurement: []byte(measurement), + shard: shard, + series: series, + converter: converter, + nameMapping: nameMapping, + start: models.MinNanoTime, + }, nil +} + +func (b *batcher) reset() { + b.start = models.MinNanoTime +} + +func (b *batcher) next(ctx context.Context) ([]row, error) { + // Iterate over the series and fields and accumulate the data row-wise + iter, err := b.shard.CreateCursorIterator(ctx) + if err != nil { + return nil, fmt.Errorf("getting cursor iterator for %q failed: %w", string(b.measurement), err) + } + + data := make(map[string]map[int64]row) + end := models.MaxNanoTime + for _, s := range b.series { + data[s.key] = make(map[int64]row) + tags := make(map[string]string, len(s.tags)) + for _, t := range s.tags { + tags[string(t.Key)] = string(t.Value) + } + for field := range s.fields { + cursor, err := iter.Next(ctx, + &tsdb.CursorRequest{ + Name: b.measurement, + Tags: s.tags, + Field: field, + Ascending: true, + StartTime: b.start, + EndTime: models.MaxNanoTime, + }, + ) + if err != nil { + return nil, fmt.Errorf("getting cursor for %s-%s failed: %w", s.key, field, err) + } + if cursor == nil { + continue + } + + // Prepare mappings + fname := field + if n, found := b.nameMapping[field]; found { + fname = n + } + converter := identity + if c, found := b.converter[field]; found { + converter = c + } + fieldEnd := models.MaxNanoTime + switch c := cursor.(type) { + case tsdb.IntegerArrayCursor: + values := c.Next() + for i, t := range values.Timestamps { + v, err := converter(values.Values[i]) + if err != nil { + fmt.Fprintf(os.Stderr, "converting %v of field %q failed: %v", values.Values[i], field, err) + continue + } + + if _, found := data[s.key][t]; !found { + data[s.key][t] = row{ + timestamp: t, + tags: tags, + fields: make(map[string]interface{}), + } + } + + data[s.key][t].fields[fname] = v + fieldEnd = t + } + case tsdb.FloatArrayCursor: + values := c.Next() + for i, t := range values.Timestamps { + v, err := converter(values.Values[i]) + if err != nil { + fmt.Fprintf(os.Stderr, "converting %v of field %q failed: %v", values.Values[i], field, err) + continue + } + + if _, found := data[s.key][t]; !found { + data[s.key][t] = row{ + timestamp: t, + tags: tags, + fields: make(map[string]interface{}), + } + } + + data[s.key][t].fields[fname] = v + fieldEnd = t + } + case tsdb.UnsignedArrayCursor: + values := c.Next() + for i, t := range values.Timestamps { + v, err := converter(values.Values[i]) + if err != nil { + fmt.Fprintf(os.Stderr, "converting %v of field %q failed: %v", values.Values[i], field, err) + continue + } + + if _, found := data[s.key][t]; !found { + data[s.key][t] = row{ + timestamp: t, + tags: tags, + fields: make(map[string]interface{}), + } + } + + data[s.key][t].fields[fname] = v + fieldEnd = t + } + case tsdb.BooleanArrayCursor: + values := c.Next() + for i, t := range values.Timestamps { + v, err := converter(values.Values[i]) + if err != nil { + fmt.Fprintf(os.Stderr, "converting %v of field %q failed: %v", values.Values[i], field, err) + continue + } + + if _, found := data[s.key][t]; !found { + data[s.key][t] = row{ + timestamp: t, + tags: tags, + fields: make(map[string]interface{}), + } + } + + data[s.key][t].fields[fname] = v + fieldEnd = t + } + case tsdb.StringArrayCursor: + values := c.Next() + for i, t := range values.Timestamps { + v, err := converter(values.Values[i]) + if err != nil { + fmt.Fprintf(os.Stderr, "converting %v of field %q failed: %v", values.Values[i], field, err) + continue + } + + if _, found := data[s.key][t]; !found { + data[s.key][t] = row{ + timestamp: t, + tags: tags, + fields: make(map[string]interface{}), + } + } + + data[s.key][t].fields[fname] = v + fieldEnd = t + } + default: + cursor.Close() + panic(fmt.Errorf("unexpected type %T", cursor)) + } + cursor.Close() + end = min(end, fieldEnd) + } + } + if len(data) == 0 { + return nil, nil + } + + // Extract the rows ordered by timestamp + var rows []row + for _, tmap := range data { + for _, r := range tmap { + rows = append(rows, r) + } + } + sort.Slice(rows, func(i, j int) bool { return rows[i].timestamp < rows[j].timestamp }) + + // Only include rows that are before the end-timestamp to avoid duplicate + // or incomplete entries due to not iterating through all data + n := sort.Search(len(rows), func(i int) bool { return rows[i].timestamp > end }) + + // Remember the earliest datum to use this for the next batch excluding the entry itself + b.start = end + 1 + + return rows[:n], nil +} diff --git a/cmd/influx_tools/parquet/command.go b/cmd/influx_tools/parquet/command.go new file mode 100644 index 00000000000..36c972155ee --- /dev/null +++ b/cmd/influx_tools/parquet/command.go @@ -0,0 +1,101 @@ +package parquet + +import ( + "context" + "errors" + "flag" + "fmt" + "io" + "os" + + "go.uber.org/zap" + + "github.com/influxdata/influxdb/cmd/influx_tools/server" +) + +// Command represents the program execution for "store query". +type Command struct { + // Standard input/output, overridden for testing. + Stderr io.Writer + Logger *zap.Logger + + server server.Interface +} + +// NewCommand returns a new instance of the export Command. +func NewCommand(server server.Interface) *Command { + return &Command{ + Stderr: os.Stderr, + server: server, + } +} + +// Run executes the export command using the specified args. +func (cmd *Command) Run(args []string) (err error) { + var ( + configPath string + database string + rp string + measurements string + typeResolutions string + nameResolutions string + output string + dryRun bool + ) + + cwd, err := os.Getwd() + if err != nil { + return fmt.Errorf("getting current working directory failed: %w", err) + } + + flags := flag.NewFlagSet("export", flag.ContinueOnError) + flags.StringVar(&configPath, "config", "", "Config file of the InfluxDB v1 instance") + flags.StringVar(&database, "database", "", "Database to export") + flags.StringVar(&rp, "rp", "", "Retention policy in the database to export") + flags.StringVar(&measurements, "measurements", "*", "Comma-separated list of measurements to export") + flags.StringVar(&typeResolutions, "resolve-types", "", "Comma-separated list of field type resolutions in the form .=") + flags.StringVar(&nameResolutions, "resolve-names", "", "Comma-separated list of field renamings in the form .=") + flags.StringVar(&output, "output", cwd, "Output directory for exported parquet files") + flags.BoolVar(&dryRun, "dry-run", false, "Print plan and exit") + + if err := flags.Parse(args); err != nil { + return err + } + + if database == "" { + return errors.New("database is required") + } + + if err := cmd.server.Open(configPath); err != nil { + return err + } + defer cmd.server.Close() + + cfg := &config{ + Database: database, + RP: rp, + Measurements: measurements, + TypeResolutions: typeResolutions, + NameResolutions: nameResolutions, + Output: output, + Stderr: cmd.Stderr, + } + exp, err := newExporter(cmd.server, cfg) + if err != nil { + return err + } + + ctx := context.Background() + if err := exp.open(ctx); err != nil { + return err + } + defer exp.close() + + exp.printPlan() + + if dryRun { + return nil + } + + return exp.export(ctx) +} diff --git a/cmd/influx_tools/parquet/converters.go b/cmd/influx_tools/parquet/converters.go new file mode 100644 index 00000000000..3130cea72ce --- /dev/null +++ b/cmd/influx_tools/parquet/converters.go @@ -0,0 +1,121 @@ +package parquet + +import ( + "fmt" + "math" + "strconv" +) + +func toFloat(value interface{}) (interface{}, error) { + switch v := value.(type) { + case float64: + return v, nil + case uint64: + return float64(v), nil + case int64: + return float64(v), nil + case bool: + if v { + return float64(1), nil + } + return float64(0), nil + case string: + return strconv.ParseFloat(v, 64) + } + return nil, fmt.Errorf("unexpected type %T for conversion", value) +} + +func toUint(value interface{}) (interface{}, error) { + switch v := value.(type) { + case float64: + if v < 0 || v > math.MaxUint64 { + return nil, fmt.Errorf("float64 value %f not convertible to uint", v) + } + return uint64(v), nil + case uint64: + return v, nil + case int64: + if v < 0 { + return nil, fmt.Errorf("int64 value %d not convertible to uint", v) + } + return uint64(v), nil + case bool: + if v { + return uint64(1), nil + } + return uint64(0), nil + case string: + return strconv.ParseUint(v, 10, 64) + } + return nil, fmt.Errorf("unexpected type %T for conversion", value) +} + +func toInt(value interface{}) (interface{}, error) { + switch v := value.(type) { + case float64: + if v > math.MaxInt64 { + return nil, fmt.Errorf("float64 value %f not convertible to int", v) + } + return int64(v), nil + case uint64: + if v > math.MaxInt64 { + return nil, fmt.Errorf("uint64 value %d not convertible to int", v) + } + return int64(v), nil + case int64: + return v, nil + case bool: + if v { + return int64(1), nil + } + return int64(0), nil + case string: + return strconv.ParseInt(v, 10, 64) + } + return nil, fmt.Errorf("unexpected type %T for conversion", value) +} + +func toBool(value interface{}) (interface{}, error) { + switch v := value.(type) { + case float64: + return v != 0, nil + case uint64: + return v != 0, nil + case int64: + return v != 0, nil + case bool: + return v, nil + case string: + if v == "true" { + return true, nil + } + if v == "false" { + return false, nil + } + return nil, fmt.Errorf("value %q not convertible to bool", v) + } + return nil, fmt.Errorf("unexpected type %T for conversion", value) +} + +func toString(value interface{}) (interface{}, error) { + switch v := value.(type) { + case float64: + return strconv.FormatFloat(v, 'f', -1, 64), nil + case uint64: + return strconv.FormatUint(v, 10), nil + case int64: + return strconv.FormatInt(v, 10), nil + case bool: + if v { + return "true", nil + } + return "false", nil + case string: + return v, nil + } + return nil, fmt.Errorf("unexpected type %T for conversion", value) +} + +func identity(value interface{}) (interface{}, error) { + return value, nil +} diff --git a/cmd/influx_tools/parquet/exporter.go b/cmd/influx_tools/parquet/exporter.go new file mode 100644 index 00000000000..4839fd3b9b7 --- /dev/null +++ b/cmd/influx_tools/parquet/exporter.go @@ -0,0 +1,488 @@ +package parquet + +import ( + "context" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "sort" + "strconv" + "strings" + "text/tabwriter" + "time" + + "github.com/apache/arrow/go/v16/arrow" + "github.com/apache/arrow/go/v16/arrow/array" + "github.com/apache/arrow/go/v16/parquet" + "github.com/apache/arrow/go/v16/parquet/pqarrow" + + "github.com/influxdata/flux/memory" + "github.com/influxdata/influxdb/cmd/influx_tools/server" + "github.com/influxdata/influxdb/models" + "github.com/influxdata/influxdb/services/meta" + "github.com/influxdata/influxdb/tsdb" + "github.com/influxdata/influxql" +) + +type config struct { + Database string + RP string + Measurements string + TypeResolutions string + NameResolutions string + Output string + Stderr io.Writer +} + +type exporter struct { + client server.MetaClient + store *tsdb.Store + + // Input selection + db, rp string + measurements []string + + // Output file settings + path string + filenames map[string]int + + // Source data and corresponding information + groups []meta.ShardGroupInfo + startDate time.Time + endDate time.Time + shards []*tsdb.Shard + + // Schema information and resolutions + schemata map[string]*schemaCreator + typeResolutions map[string]map[string]influxql.DataType + nameResolutions map[string]map[string]string + + // Parquet information + writer io.Writer + exportStart time.Time +} + +func newExporter(server server.Interface, cfg *config) (*exporter, error) { + client := server.MetaClient() + + db := client.Database(cfg.Database) + if db == nil { + return nil, fmt.Errorf("database %q does not exist", cfg.Database) + } + + if cfg.RP == "" { + cfg.RP = db.DefaultRetentionPolicy + } + + rp, err := client.RetentionPolicy(cfg.Database, cfg.RP) + if rp == nil || err != nil { + return nil, fmt.Errorf("retention policy %q does not exist", cfg.RP) + } + + store := tsdb.NewStore(server.TSDBConfig().Dir) + if server.Logger() != nil { + store.WithLogger(server.Logger()) + } + store.EngineOptions.MonitorDisabled = true + store.EngineOptions.CompactionDisabled = true + store.EngineOptions.Config = server.TSDBConfig() + store.EngineOptions.EngineVersion = server.TSDBConfig().Engine + store.EngineOptions.IndexVersion = server.TSDBConfig().Index + store.EngineOptions.DatabaseFilter = func(database string) bool { + return database == cfg.Database + } + store.EngineOptions.RetentionPolicyFilter = func(_, rpolicy string) bool { + return rpolicy == cfg.RP + } + + // Create the exporter + e := &exporter{ + client: client, + store: store, + db: cfg.Database, + rp: cfg.RP, + path: cfg.Output, + typeResolutions: make(map[string]map[string]influxql.DataType), + nameResolutions: make(map[string]map[string]string), + filenames: make(map[string]int), + writer: cfg.Stderr, + } + + // Split the given measurements + if cfg.Measurements != "" && cfg.Measurements != "*" { + e.measurements = strings.Split(cfg.Measurements, ",") + } + + // Prepare type resolutions + if cfg.TypeResolutions != "" { + for _, r := range strings.Split(cfg.TypeResolutions, ",") { + field, ftype, found := strings.Cut(r, "=") + if !found { + return nil, fmt.Errorf("invalid format in type conflict resolution %q", r) + } + measurement, field, found := strings.Cut(field, ".") + if !found { + return nil, fmt.Errorf("invalid measurement in type conflict resolution %q", r) + } + if _, exists := e.typeResolutions[measurement]; !exists { + e.typeResolutions[measurement] = make(map[string]influxql.DataType) + } + + switch strings.ToLower(ftype) { + case "float": + e.typeResolutions[measurement][field] = influxql.Float + case "int": + e.typeResolutions[measurement][field] = influxql.Integer + case "uint": + e.typeResolutions[measurement][field] = influxql.Unsigned + case "bool": + e.typeResolutions[measurement][field] = influxql.Boolean + case "string": + e.typeResolutions[measurement][field] = influxql.String + default: + return nil, fmt.Errorf("invalid type in conflict resolution %q", r) + } + } + } + + // Prepare name resolutions + if cfg.NameResolutions != "" { + for _, r := range strings.Split(cfg.NameResolutions, ",") { + field, name, found := strings.Cut(r, "=") + if !found { + return nil, fmt.Errorf("invalid format in name conflict resolution %q", r) + } + measurement, field, found := strings.Cut(field, ".") + if !found { + return nil, fmt.Errorf("invalid measurement in name conflict resolution %q", r) + } + if _, exists := e.nameResolutions[measurement]; !exists { + e.nameResolutions[measurement] = make(map[string]string) + } + e.nameResolutions[measurement][field] = name + } + } + return e, nil +} + +func (e *exporter) open(ctx context.Context) error { + if err := e.store.Open(); err != nil { + return err + } + + // Determine all shard groups in the database + min := time.Unix(0, models.MinNanoTime) + max := time.Unix(0, models.MaxNanoTime) + groups, err := e.client.NodeShardGroupsByTimeRange(e.db, e.rp, min, max) + if err != nil { + return err + } + + if len(groups) == 0 { + return nil + } + + sort.Sort(meta.ShardGroupInfos(groups)) + e.startDate = groups[0].StartTime + e.endDate = groups[len(groups)-1].EndTime + e.groups = groups + + // Collect all shards + for _, grp := range groups { + ids := make([]uint64, 0, len(grp.Shards)) + for _, s := range grp.Shards { + ids = append(ids, s.ID) + } + e.shards = append(e.shards, e.store.Shards(ids)...) + } + + // Determine all measurements in all shards + if len(e.measurements) == 0 { + measurements := make(map[string]bool) + for _, shard := range e.shards { + if err := shard.ForEachMeasurementName(func(name []byte) error { + measurements[string(name)] = true + return nil + }); err != nil { + return fmt.Errorf("getting measurement names failed: %w", err) + } + } + for m := range measurements { + e.measurements = append(e.measurements, m) + } + } + sort.Strings(e.measurements) + + // Collect the schemata for all measurments + e.schemata = make(map[string]*schemaCreator, len(e.measurements)) + for _, m := range e.measurements { + creator := newSchemaCreator(m, e.shards, e.typeResolutions[m], e.nameResolutions[m]) + if err := creator.extractSchema(ctx); err != nil { + return fmt.Errorf("extracting schema for measurement %q failed: %w", m, err) + } + e.schemata[m] = creator + } + + return nil +} + +func (e *exporter) close() error { + return e.store.Close() +} + +func (e *exporter) printPlan() { + w := tabwriter.NewWriter(e.writer, 10, 8, 1, '\t', 0) + + fmt.Fprintf(e.writer, "Exporting source data from %s to %s in %d shard group(s):\n", e.startDate, e.endDate, len(e.groups)) + fmt.Fprintln(w, " Group\tStart\tEnd\t#Shards") + fmt.Fprintln(w, " -----\t-----\t---\t-------") + for _, g := range e.groups { + fmt.Fprintf(w, " %d\t%s\t%s\t%d\n", g.ID, g.StartTime, g.EndTime, len(g.Shards)) + } + fmt.Fprintln(w) + + fmt.Fprintf(e.writer, "Creating the following schemata for %d measurement(s):\n", len(e.measurements)) + for _, measurement := range e.measurements { + creator := e.schemata[measurement] + hasConflicts, err := creator.validate() + if err != nil { + fmt.Fprintf( + e.writer, + "!!Measurement %q with conflict(s) in %d tag(s), %d field(s):\n", + measurement, + len(creator.tags), + len(creator.fieldKeys), + ) + } else if hasConflicts { + fmt.Fprintf( + e.writer, + "* Measurement %q with resolved conflicts in %d tag(s), %d field(s):\n", + measurement, + len(creator.tags), + len(creator.fieldKeys), + ) + } else { + fmt.Fprintf( + e.writer, + " Measurement %q with %d tag(s) and %d field(s):\n", + measurement, + len(creator.tags), + len(creator.fieldKeys), + ) + + } + fmt.Fprintln(w, " Column\tKind\tDatatype") + fmt.Fprintln(w, " ------\t----\t--------") + fmt.Fprintln(w, " time\ttimestamp\ttimestamp (nanosecond)") + for _, name := range creator.tags { + fmt.Fprintf(w, " %s\ttag\tstring\n", name) + } + for _, name := range creator.fieldKeys { + ftype := creator.fields[name].String() + if types, found := creator.conflicts[name]; found { + parts := make([]string, 0, len(types)) + for _, t := range types { + parts = append(parts, t.String()) + } + ftype = strings.Join(parts, "|") + if rftype, found := creator.typeResolutions[name]; found { + ftype += " -> " + rftype.String() + } + } + fname := name + if n, found := creator.nameResolutions[name]; found { + fname += " -> " + n + } + fmt.Fprintf(w, " %s\tfield\t%s\n", fname, ftype) + } + if err != nil { + fmt.Fprintln(w, " ", err) + } + fmt.Fprintln(w) + } + + w.Flush() +} + +func (e *exporter) export(ctx context.Context) error { + // Check if the schema has unresolved conflicts + for m, s := range e.schemata { + if _, err := s.validate(); err != nil { + return fmt.Errorf("%w in schema of measurement %q", err, m) + } + } + + // Create the export directory if it doesn't exist + if err := os.MkdirAll(e.path, 0700); err != nil { + return fmt.Errorf("creating directory %q failed: %w", e.path, err) + } + + // Export shard-wise + e.exportStart = time.Now() + fmt.Fprintf(e.writer, "Starting export at %v...\n", e.exportStart) + for _, shard := range e.shards { + start := time.Now() + fmt.Fprintf(e.writer, "Starting export of shard %d...\n", shard.ID()) + + for _, measurement := range e.measurements { + if err := e.exportMeasurement(ctx, shard, measurement); err != nil { + return fmt.Errorf("exporting measurement %q in shard %d failed: %w", measurement, shard.ID(), err) + } + } + fmt.Fprintf(e.writer, "Finished export of shard %d in %v...\n", shard.ID(), time.Since(start)) + } + fmt.Fprintf(e.writer, "Finished export in %v...\n", time.Since(e.exportStart)) + + return nil +} + +func (e *exporter) exportMeasurement(ctx context.Context, shard *tsdb.Shard, measurement string) error { + startMeasurement := time.Now() + + // Get the cumulative scheme with all tags and fields for the measurement + creator, found := e.schemata[measurement] + if !found { + return errors.New("no schema creator found") + } + + if len(creator.fieldKeys) == 0 { + fmt.Fprintf(e.writer, " Skipping measurement %q without fields\n", measurement) + return nil + } + + // Create a batch processor + series := creator.series[shard.ID()] + batcher, err := newBatcher(ctx, shard, measurement, series, creator.typeConverters, creator.nameResolutions) + if err != nil { + return fmt.Errorf("creating batcher failed: %w", err) + } + + // Check if we do have data for the measurement and skip the shard if + // that's not the case + rows, err := batcher.next(ctx) + if err != nil { + return fmt.Errorf("checking data failed: %w", err) + } + if len(rows) == 0 { + fmt.Fprintf(e.writer, " Skipping measurement %q without data\n", measurement) + return nil + } + batcher.reset() + + fmt.Fprintf(e.writer, " Exporting measurement %q...\n", measurement) + + // Create a parquet schema for writing the data + metadata := map[string]string{ + "export": e.exportStart.Format(time.RFC3339), + "database": e.db, + "retention": e.rp, + "measurement": measurement, + "shard": strconv.FormatUint(shard.ID(), 10), + "start_time": e.startDate.Format(time.RFC3339Nano), + "end_time": e.endDate.Format(time.RFC3339Nano), + } + schema, err := creator.schema(metadata) + if err != nil { + return fmt.Errorf("creating arrow schema failed: %w", err) + } + + // Create parquet file with a increasing sequence number per measurement + filename := filepath.Join( + e.path, + fmt.Sprintf("%s-%05d.parquet", measurement, e.filenames[measurement]), + ) + e.filenames[measurement]++ + file, err := os.Create(filename) + if err != nil { + return fmt.Errorf("creating file %q failed: %w", filename, err) + } + defer file.Close() + + writer, err := pqarrow.NewFileWriter( + schema, + file, + parquet.NewWriterProperties(parquet.WithCreatedBy("influx_tools")), + pqarrow.NewArrowWriterProperties(pqarrow.WithCoerceTimestamps(arrow.Nanosecond)), + ) + if err != nil { + return fmt.Errorf("creating parquet writer for file %q failed: %w", filename, err) + } + defer writer.Close() + + // Prepare the record builder + builder := array.NewRecordBuilder(memory.DefaultAllocator, schema) + defer builder.Release() + + // Process the data in batches + var count int + for { + last := time.Now() + + // Read the next batch + rows, err := batcher.next(ctx) + if err != nil { + return fmt.Errorf("reading batch failed: %w", err) + } + if len(rows) == 0 { + break + } + count += len(rows) + + // Convert the data to an arrow representation + record := e.convertData(rows, builder, creator.tags, creator.fieldKeys) + + // Write data + if err := writer.WriteBuffered(record); err != nil { + return fmt.Errorf("writing parquet file %q failed: %w", filename, err) + } + fmt.Fprintf(e.writer, " exported %d rows in %v\n", len(rows), time.Since(last)) + } + fmt.Fprintf( + e.writer, + " exported %d rows of measurement %q to %q in %v...\n", + count, + measurement, + filename, + time.Since(startMeasurement), + ) + + return nil +} + +func (e *exporter) convertData(rows []row, builder *array.RecordBuilder, tags, fields []string) arrow.Record { + for _, r := range rows { + builder.Field(0).(*array.TimestampBuilder).Append(arrow.Timestamp(r.timestamp)) + base := 1 + for i, k := range tags { + if v, found := r.tags[k]; found { + builder.Field(base + i).(*array.StringBuilder).Append(v) + } else { + builder.Field(base + i).AppendNull() + } + } + base = len(tags) + 1 + for i, k := range fields { + v, found := r.fields[k] + if !found { + builder.Field(base + i).AppendNull() + continue + } + switch b := builder.Field(base + i).(type) { + case *array.Float64Builder: + b.Append(v.(float64)) + case *array.Int64Builder: + b.Append(v.(int64)) + case *array.Uint64Builder: + b.Append(v.(uint64)) + case *array.StringBuilder: + b.Append(v.(string)) + case *array.BooleanBuilder: + b.Append(v.(bool)) + } + } + } + + return builder.NewRecord() +} diff --git a/cmd/influx_tools/parquet/schema.go b/cmd/influx_tools/parquet/schema.go new file mode 100644 index 00000000000..45801663e72 --- /dev/null +++ b/cmd/influx_tools/parquet/schema.go @@ -0,0 +1,273 @@ +package parquet + +import ( + "context" + "fmt" + "sort" + "strings" + + "github.com/apache/arrow/go/v16/arrow" + + "github.com/influxdata/influxdb/models" + "github.com/influxdata/influxdb/tsdb" + "github.com/influxdata/influxql" +) + +type seriesEntry struct { + key string + tags models.Tags + fields map[string]influxql.DataType +} + +type schemaCreator struct { + measurement string + shards []*tsdb.Shard + + series map[uint64][]seriesEntry + + tags []string + fields map[string]influxql.DataType + fieldKeys []string + conflicts map[string][]influxql.DataType + + typeResolutions map[string]influxql.DataType + typeConverters map[string]func(interface{}) (interface{}, error) + nameResolutions map[string]string +} + +func newSchemaCreator( + measurement string, + shards []*tsdb.Shard, + typeResolutions map[string]influxql.DataType, + nameResolutions map[string]string, +) *schemaCreator { + s := &schemaCreator{ + measurement: measurement, + shards: shards, + series: make(map[uint64][]seriesEntry, len(shards)), + typeResolutions: typeResolutions, + nameResolutions: nameResolutions, + } + + // Setup the type converters for the conflicting fields + s.typeConverters = make(map[string]func(interface{}) (interface{}, error), len(s.typeResolutions)) + for field, ftype := range s.typeResolutions { + switch ftype { + case influxql.Float: + s.typeConverters[field] = toFloat + case influxql.Unsigned: + s.typeConverters[field] = toUint + case influxql.Integer: + s.typeConverters[field] = toInt + case influxql.Boolean: + s.typeConverters[field] = toBool + case influxql.String: + s.typeConverters[field] = toString + default: + panic(fmt.Errorf("unknown converter %v for field %q", ftype, field)) + } + } + return s +} + +func (s *schemaCreator) extractSchema(ctx context.Context) error { + // Iterate over the shards and extract all series + for _, shard := range s.shards { + // Extract all fields of the measurement + fields := shard.MeasurementFields([]byte(s.measurement)).FieldSet() + if len(fields) == 0 { + continue + } + + // Collect all available series in the shard and measurement and store + // them for later use and series accumulation + seriesCursor, err := shard.CreateSeriesCursor( + ctx, + tsdb.SeriesCursorRequest{}, + influxql.MustParseExpr("_name = '"+s.measurement+"'"), + ) + if err != nil { + return fmt.Errorf("getting series cursor failed: %w", err) + } + defer seriesCursor.Close() + + for { + cur, err := seriesCursor.Next() + if err != nil { + return fmt.Errorf("advancing series cursor failed: %w", err) + } + if cur == nil { + break + } + mname := string(cur.Name) + if mname != s.measurement { + continue + } + + s.series[shard.ID()] = append(s.series[shard.ID()], seriesEntry{ + key: s.measurement + "." + string(cur.Tags.HashKey(true)), + tags: cur.Tags.Clone(), + fields: fields, + }) + } + } + + // Collect all tags and fields for creating the overall schema + tags := make(map[string]bool) + conflicts := make(map[string]map[influxql.DataType]bool) + s.fields = make(map[string]influxql.DataType) + for _, series := range s.series { + for _, serie := range series { + // Dedup tag keys + for _, tag := range serie.tags { + tags[string(tag.Key)] = true + } + // Detect field type conflicts and collect the fields + for name, current := range serie.fields { + if existing, found := s.fields[name]; found && current != existing { + if _, exists := conflicts[name]; !exists { + conflicts[name] = make(map[influxql.DataType]bool) + } + conflicts[name][current] = true + conflicts[name][existing] = true + } + s.fields[name] = current + } + } + } + + // Apply the type resolutions + for name, ftype := range s.typeResolutions { + if _, found := s.fields[name]; !found { + continue + } + s.fields[name] = ftype + } + + // Boil down all tags, fields and conflicts for later reuse + s.tags = make([]string, 0, len(tags)) + for name := range tags { + s.tags = append(s.tags, name) + } + sort.Strings(s.tags) + + s.fieldKeys = make([]string, 0, len(s.fields)) + for name, ftype := range s.fields { + // Detect unconvertible fields + switch ftype { + case influxql.Float, influxql.Integer, influxql.String, influxql.Boolean, influxql.Unsigned: + // Accepted type + s.fieldKeys = append(s.fieldKeys, name) + default: + // Unconvertible type + panic(fmt.Errorf("unconvertible field %q with type %v", name, ftype)) + } + } + sort.Strings(s.fieldKeys) + + s.conflicts = make(map[string][]influxql.DataType, len(conflicts)) + for name, conflict := range conflicts { + s.conflicts[name] = make([]influxql.DataType, 0, len(conflict)) + for ftype := range conflict { + s.conflicts[name] = append(s.conflicts[name], ftype) + } + } + + return nil +} + +func (s *schemaCreator) validate() (bool, error) { + var hasConflicts bool + var typeConflicts []string + for field := range s.conflicts { + hasConflicts = true + if _, resolved := s.typeResolutions[field]; !resolved { + typeConflicts = append(typeConflicts, field) + } + } + if len(typeConflicts) > 0 { + return true, fmt.Errorf("unresolved type conflicts for %q", strings.Join(typeConflicts, ",")) + } + + var nameConflicts []string + for _, field := range s.fieldKeys { + for _, tag := range s.tags { + if tag == field { + hasConflicts = true + if _, resolved := s.nameResolutions[field]; !resolved { + nameConflicts = append(nameConflicts, tag) + } + } + } + } + if len(nameConflicts) > 0 { + return true, fmt.Errorf("unresolved name conflicts for %q", strings.Join(nameConflicts, ",")) + } + + return hasConflicts, nil +} + +func (s *schemaCreator) schema(info map[string]string) (*arrow.Schema, error) { + columns := make([]arrow.Field, 0, 1+len(s.tags)+len(s.fields)) + + // Add the timestamp column first + columns = append(columns, arrow.Field{ + Name: "time", + Type: &arrow.TimestampType{Unit: arrow.Nanosecond}, + Metadata: arrow.MetadataFrom(map[string]string{"kind": "timestamp"}), + }) + + // Add tags in alphabetical order + for _, tag := range s.tags { + columns = append(columns, arrow.Field{ + Name: tag, + Type: &arrow.StringType{}, + Nullable: true, + Metadata: arrow.MetadataFrom(map[string]string{"kind": "tag"}), + }) + } + + // Add fields in alphabetical order with the corresponding arrow type + for _, name := range s.fieldKeys { + ftype := s.fields[name] + if t, found := s.typeResolutions[name]; found { + ftype = t + } + fname := name + if n, found := s.nameResolutions[name]; found { + fname = n + } + + var dtype arrow.DataType + switch ftype { + case influxql.Float: + dtype = &arrow.Float64Type{} + case influxql.Integer: + dtype = &arrow.Int64Type{} + case influxql.String: + dtype = &arrow.StringType{} + case influxql.Boolean: + dtype = &arrow.BooleanType{} + case influxql.Unsigned: + dtype = &arrow.Uint64Type{} + default: + panic(fmt.Errorf("unconvertible field %q", name)) + } + columns = append(columns, arrow.Field{ + Name: fname, + Type: dtype, + Nullable: true, + Metadata: arrow.MetadataFrom(map[string]string{"kind": "field"}), + }) + } + + // Add the metadata given as argument and add info about column kinds + if info == nil { + info = make(map[string]string) + } + info["tags"] = strings.Join(s.tags, ",") + info["fields"] = strings.Join(s.fieldKeys, ",") + metadata := arrow.MetadataFrom(info) + + return arrow.NewSchema(columns, &metadata), nil +} diff --git a/go.mod b/go.mod index 7619c7d81f2..429c190c703 100644 --- a/go.mod +++ b/go.mod @@ -6,6 +6,7 @@ require ( collectd.org v0.3.0 github.com/BurntSushi/toml v0.3.1 github.com/apache/arrow/go/arrow v0.0.0-20211112161151-bc219186db40 + github.com/apache/arrow/go/v16 v16.1.0 github.com/benbjohnson/tmpl v1.0.0 github.com/bmizerany/pat v0.0.0-20170815010413-6226ea591a40 github.com/cespare/xxhash v1.1.0 @@ -15,7 +16,7 @@ require ( github.com/golang-jwt/jwt v3.2.1+incompatible github.com/golang/mock v1.5.0 github.com/golang/snappy v0.0.4 - github.com/google/go-cmp v0.5.9 + github.com/google/go-cmp v0.6.0 github.com/influxdata/flux v0.194.5 github.com/influxdata/httprouter v1.3.1-0.20191122104820-ee83e2772f69 github.com/influxdata/influxql v1.3.0 @@ -25,7 +26,7 @@ require ( github.com/jsternberg/zap-logfmt v1.2.0 github.com/jwilder/encoding v0.0.0-20170811194829-b4e1701a28ef github.com/klauspost/pgzip v1.0.2-0.20170402124221-0bf5dcad4ada - github.com/mattn/go-isatty v0.0.16 + github.com/mattn/go-isatty v0.0.19 github.com/mileusna/useragent v0.0.0-20190129205925-3e331f0949a5 github.com/opentracing/opentracing-go v1.2.0 github.com/peterh/liner v1.0.1-0.20180619022028-8c1271fcf47f @@ -37,32 +38,32 @@ require ( github.com/retailnext/hllpp v1.0.1-0.20180308014038-101a6d2f8b52 github.com/spf13/cast v1.3.0 github.com/spf13/cobra v0.0.3 - github.com/stretchr/testify v1.8.1 + github.com/stretchr/testify v1.9.0 github.com/tinylib/msgp v1.1.0 github.com/uber/jaeger-client-go v2.28.0+incompatible github.com/xlab/treeprint v0.0.0-20180616005107-d6fb6747feb6 go.uber.org/multierr v1.6.0 go.uber.org/zap v1.16.0 golang.org/x/crypto v0.21.0 - golang.org/x/exp v0.0.0-20231214170342-aacd6d4b4611 - golang.org/x/sync v0.5.0 + golang.org/x/exp v0.0.0-20240222234643-814bf88cf225 + golang.org/x/sync v0.6.0 golang.org/x/sys v0.18.0 golang.org/x/term v0.18.0 golang.org/x/text v0.14.0 - golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba - golang.org/x/tools v0.16.0 - google.golang.org/grpc v1.56.3 + golang.org/x/time v0.5.0 + golang.org/x/tools v0.19.0 + google.golang.org/grpc v1.62.1 google.golang.org/protobuf v1.33.0 ) require ( - cloud.google.com/go v0.110.0 // indirect - cloud.google.com/go/bigquery v1.50.0 // indirect + cloud.google.com/go v0.112.0 // indirect + cloud.google.com/go/bigquery v1.58.0 // indirect cloud.google.com/go/bigtable v1.10.1 // indirect - cloud.google.com/go/compute v1.19.1 // indirect + cloud.google.com/go/compute v1.23.3 // indirect cloud.google.com/go/compute/metadata v0.2.3 // indirect - cloud.google.com/go/iam v0.13.0 // indirect - cloud.google.com/go/longrunning v0.4.1 // indirect + cloud.google.com/go/iam v1.1.5 // indirect + cloud.google.com/go/longrunning v0.5.4 // indirect github.com/Azure/azure-pipeline-go v0.2.3 // indirect github.com/Azure/azure-storage-blob-go v0.14.0 // indirect github.com/Azure/go-autorest v14.2.0+incompatible // indirect @@ -74,15 +75,16 @@ require ( github.com/Azure/go-autorest/logger v0.2.1 // indirect github.com/Azure/go-autorest/tracing v0.6.0 // indirect github.com/DATA-DOG/go-sqlmock v1.4.1 // indirect + github.com/JohnCGriffin/overflow v0.0.0-20211019200055-46fa312c352c // indirect github.com/Masterminds/semver v1.4.2 // indirect github.com/Masterminds/sprig v2.16.0+incompatible // indirect github.com/SAP/go-hdb v0.14.1 // indirect github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883 // indirect - github.com/andybalholm/brotli v1.0.4 // indirect + github.com/andybalholm/brotli v1.1.0 // indirect github.com/aokoli/goutils v1.0.1 // indirect - github.com/apache/arrow/go/v11 v11.0.0 // indirect + github.com/apache/arrow/go/v12 v12.0.1 // indirect github.com/apache/arrow/go/v7 v7.0.1 // indirect - github.com/apache/thrift v0.16.0 // indirect + github.com/apache/thrift v0.19.0 // indirect github.com/aws/aws-sdk-go v1.34.0 // indirect github.com/aws/aws-sdk-go-v2 v1.11.0 // indirect github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.0.0 // indirect @@ -102,24 +104,28 @@ require ( github.com/deepmap/oapi-codegen v1.6.0 // indirect github.com/denisenkom/go-mssqldb v0.10.0 // indirect github.com/dimchansky/utfbom v1.1.0 // indirect - github.com/dustin/go-humanize v1.0.0 // indirect + github.com/dustin/go-humanize v1.0.1 // indirect github.com/eclipse/paho.mqtt.golang v1.2.0 // indirect - github.com/fatih/color v1.13.0 // indirect + github.com/fatih/color v1.15.0 // indirect + github.com/felixge/httpsnoop v1.0.4 // indirect github.com/form3tech-oss/jwt-go v3.2.5+incompatible // indirect github.com/gabriel-vasile/mimetype v1.4.0 // indirect github.com/glycerine/go-unsnap-stream v0.0.0-20180323001048-9f0cb55181dd // indirect + github.com/go-logr/logr v1.3.0 // indirect + github.com/go-logr/stdr v1.2.2 // indirect github.com/go-sql-driver/mysql v1.5.0 // indirect - github.com/goccy/go-json v0.9.11 // indirect + github.com/goccy/go-json v0.10.2 // indirect github.com/gofrs/uuid v3.3.0+incompatible // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe // indirect github.com/golang/geo v0.0.0-20190916061304-5b978397cfec // indirect - github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e // indirect + github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.4 // indirect - github.com/google/flatbuffers v22.9.30-0.20221019131441-5792623df42e+incompatible // indirect - github.com/google/uuid v1.3.0 // indirect - github.com/googleapis/enterprise-certificate-proxy v0.2.3 // indirect - github.com/googleapis/gax-go/v2 v2.7.1 // indirect + github.com/google/flatbuffers v24.3.25+incompatible // indirect + github.com/google/s2a-go v0.1.7 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/googleapis/enterprise-certificate-proxy v0.3.2 // indirect + github.com/googleapis/gax-go/v2 v2.12.0 // indirect github.com/grpc-ecosystem/grpc-gateway v1.16.0 // indirect github.com/huandu/xstrings v1.0.0 // indirect github.com/imdario/mergo v0.3.5 // indirect @@ -132,11 +138,11 @@ require ( github.com/influxdata/tdigest v0.0.2-0.20210216194612-fc98d27c9e8b // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/klauspost/asmfmt v1.3.2 // indirect - github.com/klauspost/compress v1.15.9 // indirect - github.com/klauspost/cpuid/v2 v2.0.9 // indirect + github.com/klauspost/compress v1.17.7 // indirect + github.com/klauspost/cpuid/v2 v2.2.7 // indirect github.com/klauspost/crc32 v0.0.0-20161016154125-cb6bfca970f6 // indirect github.com/lib/pq v1.0.0 // indirect - github.com/mattn/go-colorable v0.1.9 // indirect + github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-ieproxy v0.0.1 // indirect github.com/mattn/go-runewidth v0.0.3 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect @@ -145,7 +151,7 @@ require ( github.com/mitchellh/go-homedir v1.1.0 // indirect github.com/mschoch/smat v0.0.0-20160514031455-90eadee771ae // indirect github.com/philhofer/fwd v1.0.0 // indirect - github.com/pierrec/lz4/v4 v4.1.15 // indirect + github.com/pierrec/lz4/v4 v4.1.21 // indirect github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/prometheus/procfs v0.6.0 // indirect @@ -160,15 +166,22 @@ require ( github.com/willf/bitset v1.1.9 // indirect github.com/zeebo/xxh3 v1.0.2 // indirect go.opencensus.io v0.24.0 // indirect + go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.46.1 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.46.1 // indirect + go.opentelemetry.io/otel v1.21.0 // indirect + go.opentelemetry.io/otel/metric v1.21.0 // indirect + go.opentelemetry.io/otel/trace v1.21.0 // indirect go.uber.org/atomic v1.7.0 // indirect - golang.org/x/mod v0.14.0 // indirect + golang.org/x/mod v0.16.0 // indirect golang.org/x/net v0.23.0 // indirect - golang.org/x/oauth2 v0.7.0 // indirect - golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect - gonum.org/v1/gonum v0.11.0 // indirect - google.golang.org/api v0.114.0 // indirect - google.golang.org/appengine v1.6.7 // indirect - google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 // indirect + golang.org/x/oauth2 v0.16.0 // indirect + golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 // indirect + gonum.org/v1/gonum v0.15.0 // indirect + google.golang.org/api v0.155.0 // indirect + google.golang.org/appengine v1.6.8 // indirect + google.golang.org/genproto v0.0.0-20240123012728-ef4313101c80 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20240123012728-ef4313101c80 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240123012728-ef4313101c80 // indirect gopkg.in/yaml.v2 v2.3.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index da30038c9e7..b611b954a71 100644 --- a/go.sum +++ b/go.sum @@ -21,31 +21,31 @@ cloud.google.com/go v0.78.0/go.mod h1:QjdrLG0uq+YwhjoVOLsS1t7TW8fs36kLs4XO5R5ECH cloud.google.com/go v0.79.0/go.mod h1:3bzgcEeQlzbuEAYu4mrWhKqWjmpprinYgKJLgKHnbb8= cloud.google.com/go v0.81.0/go.mod h1:mk/AM35KwGk/Nm2YSeZbxXdrNK3KZOYHmLkOqC2V6E0= cloud.google.com/go v0.82.0/go.mod h1:vlKccHJGuFBFufnAnuB08dfEH9Y3H7dzDzRECFdC2TA= -cloud.google.com/go v0.110.0 h1:Zc8gqp3+a9/Eyph2KDmcGaPtbKRIoqq4YTlL4NMD0Ys= -cloud.google.com/go v0.110.0/go.mod h1:SJnCLqQ0FCFGSZMUNUf84MV3Aia54kn7pi8st7tMzaY= +cloud.google.com/go v0.112.0 h1:tpFCD7hpHFlQ8yPwT3x+QeXqc2T6+n6T+hmABHfDUSM= +cloud.google.com/go v0.112.0/go.mod h1:3jEEVwZ/MHU4djK5t5RHuKOA/GbLddgTdVubX1qnPD4= cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= -cloud.google.com/go/bigquery v1.50.0 h1:RscMV6LbnAmhAzD893Lv9nXXy2WCaJmbxYPWDLbGqNQ= -cloud.google.com/go/bigquery v1.50.0/go.mod h1:YrleYEh2pSEbgTBZYMJ5SuSr0ML3ypjRB1zgf7pvQLU= +cloud.google.com/go/bigquery v1.58.0 h1:drSd9RcPVLJP2iFMimvOB9SCSIrcl+9HD4II03Oy7A0= +cloud.google.com/go/bigquery v1.58.0/go.mod h1:0eh4mWNY0KrBTjUzLjoYImapGORq9gEPT7MWjCy9lik= cloud.google.com/go/bigtable v1.2.0/go.mod h1:JcVAOl45lrTmQfLj7T6TxyMzIN/3FGGcFm+2xVAli2o= cloud.google.com/go/bigtable v1.10.1 h1:QKcRHeAsraxIlrdCZ3LLobXKBvITqcOEnSbHG2rzL9g= cloud.google.com/go/bigtable v1.10.1/go.mod h1:cyHeKlx6dcZCO0oSQucYdauseD8kIENGuDOJPKMCVg8= -cloud.google.com/go/compute v1.19.1 h1:am86mquDUgjGNWxiGn+5PGLbmgiWXlE/yNWpIpNvuXY= -cloud.google.com/go/compute v1.19.1/go.mod h1:6ylj3a05WF8leseCdIf77NK0g1ey+nj5IKd5/kvShxE= +cloud.google.com/go/compute v1.23.3 h1:6sVlXXBmbd7jNX0Ipq0trII3e4n1/MsADLK6a+aiVlk= +cloud.google.com/go/compute v1.23.3/go.mod h1:VCgBUoMnIVIR0CscqQiPJLAG25E3ZRZMzcFZeQ+h8CI= cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGBW5aJ7UnBMY= cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA= -cloud.google.com/go/datacatalog v1.13.0 h1:4H5IJiyUE0X6ShQBqgFFZvGGcrwGVndTwUSLP4c52gw= -cloud.google.com/go/datacatalog v1.13.0/go.mod h1:E4Rj9a5ZtAxcQJlEBTLgMTphfP11/lNaAshpoBgemX8= +cloud.google.com/go/datacatalog v1.19.2 h1:BV5sB7fPc8ccv/obwtHwQtCdLMAgI4KyaQWfkh8/mWg= +cloud.google.com/go/datacatalog v1.19.2/go.mod h1:2YbODwmhpLM4lOFe3PuEhHK9EyTzQJ5AXgIy7EDKTEE= cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= -cloud.google.com/go/iam v0.13.0 h1:+CmB+K0J/33d0zSQ9SlFWUeCCEn5XJA0ZMZ3pHE9u8k= -cloud.google.com/go/iam v0.13.0/go.mod h1:ljOg+rcNfzZ5d6f1nAUJ8ZIxOaZUVoS14bKCtaLZ/D0= -cloud.google.com/go/longrunning v0.4.1 h1:v+yFJOfKC3yZdY6ZUI933pIYdhyhV8S3NpWrXWmg7jM= -cloud.google.com/go/longrunning v0.4.1/go.mod h1:4iWDqhBZ70CvZ6BfETbvam3T8FMvLK+eFj0E6AaRQTo= +cloud.google.com/go/iam v1.1.5 h1:1jTsCu4bcsNsE4iiqNT5SHwrDRCfRmIaaaVFhRveTJI= +cloud.google.com/go/iam v1.1.5/go.mod h1:rB6P/Ic3mykPbFio+vo7403drjlgvoWfYpJhMXEbzv8= +cloud.google.com/go/longrunning v0.5.4 h1:w8xEcbZodnA2BbW6sVirkkoC+1gP8wS57EUUgGS0GVg= +cloud.google.com/go/longrunning v0.5.4/go.mod h1:zqNVncI0BOP8ST6XQD1+VcvuShMmq7+xFSzOL++V0dI= cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= @@ -55,8 +55,8 @@ cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0Zeo cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= -cloud.google.com/go/storage v1.29.0 h1:6weCgzRvMg7lzuUurI4697AqIRPU1SvzHhynwpW31jI= -cloud.google.com/go/storage v1.29.0/go.mod h1:4puEjyTKnku6gfKoTfNOU/W+a9JyuVNxjpS5GBrB8h4= +cloud.google.com/go/storage v1.36.0 h1:P0mOkAcaJxhCTvAkMhxMfrTKiNcub4YmmPBtlhAyTr8= +cloud.google.com/go/storage v1.36.0/go.mod h1:M6M/3V/D3KpzMTJyPOR/HU6n2Si5QdaXYEsng2xgOs8= collectd.org v0.3.0 h1:iNBHGw1VvPJxH2B6RiFWFZ+vsjo1lCdRszBeOuwGi00= collectd.org v0.3.0/go.mod h1:A/8DzQBkF6abtvrT2j/AU/4tiBgJWYyh0y/oB/4MlWE= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= @@ -141,8 +141,8 @@ github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk5 github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883 h1:bvNMNQO63//z+xNgfBlViaCIJKLlCJ6/fmUseuG0wVQ= github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8= github.com/andybalholm/brotli v1.0.3/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= -github.com/andybalholm/brotli v1.0.4 h1:V7DdXeJtZscaqfNuAdSRuRFzuiKlHSC/Zh3zl9qY3JY= -github.com/andybalholm/brotli v1.0.4/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= +github.com/andybalholm/brotli v1.1.0 h1:eLKJA0d02Lf0mVpIDgYnqXcUn0GqVmEFny3VuID1U3M= +github.com/andybalholm/brotli v1.1.0/go.mod h1:sms7XGricyQI9K10gOSf56VKKWS4oLer58Q+mhRPtnY= github.com/antihax/optional v0.0.0-20180407024304-ca021399b1a6/go.mod h1:V8iCPQYkqmusNa815XgQio277wI47sdRh1dUOLdyC6Q= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= github.com/aokoli/goutils v1.0.1 h1:7fpzNGoJ3VA8qcrm++XEE1QUe0mIwNeLa02Nwq7RDkg= @@ -150,15 +150,17 @@ github.com/aokoli/goutils v1.0.1/go.mod h1:SijmP0QR8LtwsmDs8Yii5Z/S4trXFGFC2oO5g github.com/apache/arrow/go/arrow v0.0.0-20191024131854-af6fa24be0db/go.mod h1:VTxUBvSJ3s3eHAg65PNgrsn5BtqCRPdmyXh6rAfdxN0= github.com/apache/arrow/go/arrow v0.0.0-20211112161151-bc219186db40 h1:q4dksr6ICHXqG5hm0ZW5IHyeEJXoIJSOZeBLmWPNeIQ= github.com/apache/arrow/go/arrow v0.0.0-20211112161151-bc219186db40/go.mod h1:Q7yQnSMnLvcXlZ8RV+jwz/6y1rQTqbX6C82SndT52Zs= -github.com/apache/arrow/go/v11 v11.0.0 h1:hqauxvFQxww+0mEU/2XHG6LT7eZternCZq+A5Yly2uM= -github.com/apache/arrow/go/v11 v11.0.0/go.mod h1:Eg5OsL5H+e299f7u5ssuXsuHQVEGC4xei5aX110hRiI= +github.com/apache/arrow/go/v12 v12.0.1 h1:JsR2+hzYYjgSUkBSaahpqCetqZMr76djX80fF/DiJbg= +github.com/apache/arrow/go/v12 v12.0.1/go.mod h1:weuTY7JvTG/HDPtMQxEUp7pU73vkLWMLpY67QwZ/WWw= +github.com/apache/arrow/go/v16 v16.1.0 h1:dwgfOya6s03CzH9JrjCBx6bkVb4yPD4ma3haj9p7FXI= +github.com/apache/arrow/go/v16 v16.1.0/go.mod h1:9wnc9mn6vEDTRIm4+27pEjQpRKuTvBaessPoEXQzxWA= github.com/apache/arrow/go/v7 v7.0.1 h1:WpCfq+AQxvXaI6/KplHE27MPMFx5av0o5NbPCTAGfy4= github.com/apache/arrow/go/v7 v7.0.1/go.mod h1:JxDpochJbCVxqbX4G8i1jRqMrnTCQdf8pTccAfLD8Es= github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= github.com/apache/thrift v0.13.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= github.com/apache/thrift v0.15.0/go.mod h1:PHK3hniurgQaNMZYaCLEqXKsYK8upmhPbmdP2FXSqgU= -github.com/apache/thrift v0.16.0 h1:qEy6UW60iVOlUy+b9ZR0d5WzUWYGOo4HfopoyBaNmoY= -github.com/apache/thrift v0.16.0/go.mod h1:PHK3hniurgQaNMZYaCLEqXKsYK8upmhPbmdP2FXSqgU= +github.com/apache/thrift v0.19.0 h1:sOqkWPzMj7w6XaYbJQG7m4sGqVolaW/0D28Ln7yPzMk= +github.com/apache/thrift v0.19.0/go.mod h1:SUALL216IiaOw2Oy+5Vs9lboJ/t9g40C+G07Dc0QC1I= github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= github.com/armon/go-metrics v0.3.3/go.mod h1:4O98XIr/9W0sxpJ8UaYkvjk10Iff7SnFrb4QAOwNTFc= @@ -246,6 +248,8 @@ github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnht github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20210805033703-aa0b78936158/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20231128003011-0fa0005c9caa h1:jQCWAUqqlij9Pgj2i/PB79y4KOPYVyFYdROxgaCwdTQ= +github.com/cncf/xds/go v0.0.0-20231128003011-0fa0005c9caa/go.mod h1:x/1Gn8zydmfq8dk6e9PdstVsDgu9RuyIIJqAaF//0IM= github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8= github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI= github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= @@ -275,8 +279,9 @@ github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDD github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM= github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= -github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/eapache/go-resiliency v1.1.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs= github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU= github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I= @@ -295,11 +300,15 @@ github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.m github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.mod h1:hliV/p42l8fGbc6Y9bQ70uLwIvmJyVE5k4iMKlh8wCQ= github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go.mod h1:AFq3mo9L8Lqqiid3OhADV3RfLJnjiw63cSpi+fDTRC0= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/envoyproxy/protoc-gen-validate v1.0.4 h1:gVPz/FMfvh57HdSJQyvBtF00j8JU4zdyUgIUNhlgg0A= +github.com/envoyproxy/protoc-gen-validate v1.0.4/go.mod h1:qys6tmnRsYrQqIhm2bvKZH4Blx/1gTIZ2UKVY1M+Yew= github.com/evanphx/json-patch v4.2.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU= -github.com/fatih/color v1.13.0 h1:8LOYc1KYPPmyKMuN8QV2DNRWNbLo6LZ0iLs8+mlH53w= -github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= +github.com/fatih/color v1.15.0 h1:kOqh6YHBtK8aywxGerMG2Eq3H6Qgoqeo13Bk2Mv/nBs= +github.com/fatih/color v1.15.0/go.mod h1:0h5ZqXfHYED7Bhv2ZJamyIOUej9KtShiJESRwBDUSsw= +github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= +github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/fogleman/gg v1.2.1-0.20190220221249-0403632d5b90/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k= github.com/fogleman/gg v1.3.0/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k= github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= @@ -345,6 +354,11 @@ github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= github.com/go-logr/logr v0.4.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.3.0 h1:2y3SDp0ZXuc6/cjLSZ+Q3ir+QB9T/iG5yYRXqsagWSY= +github.com/go-logr/logr v1.3.0/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-openapi/analysis v0.0.0-20180825180245-b006789cd277/go.mod h1:k70tL6pCuVxPJOHXQ+wIac1FUrvNkHolPie/cLEU6hI= github.com/go-openapi/analysis v0.17.0/go.mod h1:IowGgpVeD0vNm45So8nr+IcQ3pxVtpRoBWb8PVZO0ik= github.com/go-openapi/analysis v0.18.0/go.mod h1:IowGgpVeD0vNm45So8nr+IcQ3pxVtpRoBWb8PVZO0ik= @@ -434,8 +448,8 @@ github.com/gobuffalo/packr/v2 v2.0.9/go.mod h1:emmyGweYTm6Kdper+iywB6YK5YzuKchGt github.com/gobuffalo/packr/v2 v2.2.0/go.mod h1:CaAwI0GPIAv+5wKLtv8Afwl+Cm78K/I/VCm/3ptBN+0= github.com/gobuffalo/syncx v0.0.0-20190224160051-33c29581e754/go.mod h1:HhnNqWY95UYwwW3uSASeV7vtgYkT2t16hJgV3AEPUpw= github.com/goccy/go-json v0.7.10/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= -github.com/goccy/go-json v0.9.11 h1:/pAaQDLHEoCq/5FFmSKBswWmK6H0e8g4159Kc/X/nqk= -github.com/goccy/go-json v0.9.11/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= +github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU= +github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= github.com/gofrs/uuid v3.3.0+incompatible h1:8K4tyRfvU1CYPgJsveYFQMhpFd/wXNM7iK6rR7UHz84= github.com/gofrs/uuid v3.3.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= github.com/gogo/googleapis v1.1.0/go.mod h1:gf4bu3Q80BeJ6H1S1vYPm8/ELATdvryBaNFGgqEef3s= @@ -458,8 +472,9 @@ github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfU github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e h1:1r7pUrabqp18hOBcwBwiTsbnFeTZHV9eER/QT5JVZxY= github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= @@ -501,8 +516,8 @@ github.com/google/btree v1.0.1 h1:gK4Kx5IaGY9CD5sPJ36FHiBJ6ZXl0kilRiiCj+jdYp4= github.com/google/btree v1.0.1/go.mod h1:xXMiIv4Fb/0kKde4SpL7qlzvu5cMJDRkFDxJfI9uaxA= github.com/google/flatbuffers v1.11.0/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= github.com/google/flatbuffers v2.0.0+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= -github.com/google/flatbuffers v22.9.30-0.20221019131441-5792623df42e+incompatible h1:Bqgl5d9t2UlT8pv9Oc/lkkI8yYk0jCwHkZKkHzbxEsc= -github.com/google/flatbuffers v22.9.30-0.20221019131441-5792623df42e+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= +github.com/google/flatbuffers v24.3.25+incompatible h1:CX395cjN9Kke9mmalRoL3d81AtFUxJM+yDthflgJGkI= +github.com/google/flatbuffers v24.3.25+incompatible/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= @@ -515,8 +530,8 @@ github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= -github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPgecwXBIDzw5no= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= @@ -538,18 +553,21 @@ github.com/google/pprof v0.0.0-20210122040257-d980be63207e/go.mod h1:kpwsk12EmLe github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210506205249-923b5ab0fc1a/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= +github.com/google/s2a-go v0.1.7 h1:60BLSyTrOV4/haCDW4zb1guZItoSq8foHCXrAnjBo/o= +github.com/google/s2a-go v0.1.7/go.mod h1:50CgR4k1jNlWBu4UfS4AcfhVe1r6pdZPygJ3R8F0Qdw= github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/googleapis/enterprise-certificate-proxy v0.2.3 h1:yk9/cqRKtT9wXZSsRH9aurXEpJX+U6FLtpYTdC3R06k= -github.com/googleapis/enterprise-certificate-proxy v0.2.3/go.mod h1:AwSRAtLfXpU5Nm3pW+v7rGDHp09LsPtGY9MduiEsR9k= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/googleapis/enterprise-certificate-proxy v0.3.2 h1:Vie5ybvEvT75RniqhfFxPRy3Bf7vr3h0cechB90XaQs= +github.com/googleapis/enterprise-certificate-proxy v0.3.2/go.mod h1:VLSiSSBs/ksPL8kq3OBOQ6WRI2QnaFynd1DCjZ62+V0= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= -github.com/googleapis/gax-go/v2 v2.7.1 h1:gF4c0zjUP2H/s/hEGyLA3I0fA2ZWjzYiONAD6cvPr8A= -github.com/googleapis/gax-go/v2 v2.7.1/go.mod h1:4orTrqY6hXxxaUL4LHIPl6lGo8vAE38/qKbhSAKP6QI= +github.com/googleapis/gax-go/v2 v2.12.0 h1:A+gCJKdRfqXkr+BIRGtZLibNXf0m1f9E4HG56etFpas= +github.com/googleapis/gax-go/v2 v2.12.0/go.mod h1:y+aIqrI5eb1YGMVJfuV3185Ts/D7qKpsEkdD5+I6QGU= github.com/googleapis/gnostic v0.0.0-20170729233727-0c5108395e2d/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY= github.com/googleapis/gnostic v0.4.0/go.mod h1:on+2t9HRStVgn95RSsFWFz+6Q0Snyqv1awfrALZdbtU= github.com/gophercloud/gophercloud v0.1.0/go.mod h1:vxM41WHh5uqHVBMZHzuwNOHh8XEoIEcSTewFxm1c5g8= @@ -691,11 +709,12 @@ github.com/klauspost/compress v1.4.0/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0 github.com/klauspost/compress v1.9.5/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= github.com/klauspost/compress v1.13.1/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg= github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= -github.com/klauspost/compress v1.15.9 h1:wKRjX6JRtDdrE9qwa4b/Cip7ACOshUI4smpCQanqjSY= -github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU= +github.com/klauspost/compress v1.17.7 h1:ehO88t2UGzQK66LMdE8tibEd1ErmzZjNEqWkjLAKQQg= +github.com/klauspost/compress v1.17.7/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= github.com/klauspost/cpuid v0.0.0-20170728055534-ae7887de9fa5/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= -github.com/klauspost/cpuid/v2 v2.0.9 h1:lgaqFMSdTdQYdZ04uHyN2d/eKdOMyi2YLSvlQIBFYa4= github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= +github.com/klauspost/cpuid/v2 v2.2.7 h1:ZWSB3igEs+d0qvnxR/ZBzXVmxkgt8DdzP6m9pfuVLDM= +github.com/klauspost/cpuid/v2 v2.2.7/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= github.com/klauspost/crc32 v0.0.0-20161016154125-cb6bfca970f6 h1:KAZ1BW2TCmT6PRihDPpocIy1QTtsAsrx6TneU/4+CMg= github.com/klauspost/crc32 v0.0.0-20161016154125-cb6bfca970f6/go.mod h1:+ZoRqAPRLkC4NPOvfYeR5KNOrY6TD+/sAC3HXPZgDYg= github.com/klauspost/pgzip v1.0.2-0.20170402124221-0bf5dcad4ada h1:3L+neHp83cTjegPdCiOxVOJtRIy7/8RldvMTsyPYH10= @@ -738,8 +757,8 @@ github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVc github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-colorable v0.1.7/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= -github.com/mattn/go-colorable v0.1.9 h1:sqDoxXbdeALODt0DAeJCVp38ps9ZogZEAXjus69YV3U= -github.com/mattn/go-colorable v0.1.9/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= +github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= +github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= github.com/mattn/go-ieproxy v0.0.1 h1:qiyop7gCflfhwCzGyeT0gro3sF9AIg9HU98JORTkqfI= github.com/mattn/go-ieproxy v0.0.1/go.mod h1:pYabZ6IHcRpFh7vIaLfK7rdcWgFEb3SFJ6/gNWuh88E= github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= @@ -749,9 +768,9 @@ github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2y github.com/mattn/go-isatty v0.0.10/go.mod h1:qgIWMr58cqv1PHHyhnkY9lrL7etaEgOFcMEpPG5Rm84= github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOAqxQCu2WE= github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= -github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= -github.com/mattn/go-isatty v0.0.16 h1:bq3VjFmv/sOjHtdEhmkEV4x1AJtvUvOJ2PFAZ5+peKQ= github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= +github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA= +github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= github.com/mattn/go-runewidth v0.0.3 h1:a+kO+98RDGEfo6asOGMmpodZq4FNtnGP54yps8BzLR4= github.com/mattn/go-runewidth v0.0.3/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= @@ -847,8 +866,8 @@ github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi github.com/pierrec/lz4/v4 v4.1.8/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= github.com/pierrec/lz4/v4 v4.1.11/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= github.com/pierrec/lz4/v4 v4.1.12/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= -github.com/pierrec/lz4/v4 v4.1.15 h1:MO0/ucJhngq7299dKLwIMtgTfbkoSPF6AoMYDd8Q4q0= -github.com/pierrec/lz4/v4 v4.1.15/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= +github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ= +github.com/pierrec/lz4/v4 v4.1.21/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8 h1:KoWmjvw+nsYOo29YJK9vDA65RGE3NrOnUtO7a+RF9HU= github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8/go.mod h1:HKlIX3XHQyzLZPlr7++PzdhaXEj94dEiJgZDTsxEqUI= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -960,8 +979,9 @@ github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+ github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= -github.com/stretchr/objx v0.5.0 h1:1zr/of2m5FGMsad5YfcqgdqdWrIhu+EBEJRhR1U7z/c= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= +github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/testify v1.2.0/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= @@ -972,8 +992,9 @@ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.2/go.mod h1:R6va5+xMeoiuVRoj+gSkQ7d3FALtqAAGI1FQKckRals= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= -github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= github.com/tinylib/msgp v1.0.2/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE= github.com/tinylib/msgp v1.1.0 h1:9fQd+ICuRIu/ue4vxJZu6/LzxN0HwMds2nq/0cFvxHU= @@ -1012,6 +1033,7 @@ github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= github.com/zeebo/assert v1.3.0 h1:g7C04CbJuIDKNPFHmsk4hwZDO5O+kntRxzaUoNXj+IQ= github.com/zeebo/assert v1.3.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0= github.com/zeebo/xxh3 v1.0.1/go.mod h1:8VHV24/3AZLn3b6Mlp/KuC33LWH687Wq6EnziEB+rsA= @@ -1035,11 +1057,23 @@ go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.46.1 h1:SpGay3w+nEwMpfVnbqOLH5gY52/foP8RE8UzTZ1pdSE= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.46.1/go.mod h1:4UoMYEZOC0yN/sPGH76KPkkU7zgiEWYWL9vwmbnTJPE= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.46.1 h1:aFJWCqJMNjENlcleuuOkGAPH82y0yULBScfXcIEdS24= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.46.1/go.mod h1:sEGXWArGqc3tVa+ekntsN65DmVbVeW+7lTKTjZF3/Fo= go.opentelemetry.io/otel v0.20.0/go.mod h1:Y3ugLH2oa81t5QO+Lty+zXf8zC9L26ax4Nzoxm/dooo= +go.opentelemetry.io/otel v1.21.0 h1:hzLeKBZEL7Okw2mGzZ0cc4k/A7Fta0uoPgaJCr8fsFc= +go.opentelemetry.io/otel v1.21.0/go.mod h1:QZzNPQPm1zLX4gZK4cMi+71eaorMSGT3A4znnUvNNEo= go.opentelemetry.io/otel/metric v0.20.0/go.mod h1:598I5tYlH1vzBjn+BTuhzTCSb/9debfNp6R3s7Pr1eU= +go.opentelemetry.io/otel/metric v1.21.0 h1:tlYWfeo+Bocx5kLEloTjbcDwBuELRrIFxwdQ36PlJu4= +go.opentelemetry.io/otel/metric v1.21.0/go.mod h1:o1p3CA8nNHW8j5yuQLdc1eeqEaPfzug24uvsyIEJRWM= go.opentelemetry.io/otel/oteltest v0.20.0/go.mod h1:L7bgKf9ZB7qCwT9Up7i9/pn0PWIa9FqQ2IQ8LoxiGnw= go.opentelemetry.io/otel/sdk v0.20.0/go.mod h1:g/IcepuwNsoiX5Byy2nNV0ySUF1em498m7hBWC279Yc= +go.opentelemetry.io/otel/sdk v1.21.0 h1:FTt8qirL1EysG6sTQRZ5TokkU8d0ugCj8htOgThZXQ8= +go.opentelemetry.io/otel/sdk v1.21.0/go.mod h1:Nna6Yv7PWTdgJHVRD9hIYywQBRx7pbox6nwBnZIxl/E= go.opentelemetry.io/otel/trace v0.20.0/go.mod h1:6GjCW8zgDjwGHGa6GkyeB8+/5vjT16gUEi0Nf1iBdgw= +go.opentelemetry.io/otel/trace v1.21.0 h1:WD9i5gzvoUPuXIXH24ZNBudiarZDKuekPqi/E8fpfLc= +go.opentelemetry.io/otel/trace v1.21.0/go.mod h1:LGbsEB0f9LGjN+OZaQQ26sohbOmiMR+BaslueVtS/qQ= go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= @@ -1104,8 +1138,8 @@ golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u0 golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= golang.org/x/exp v0.0.0-20211216164055-b2b84827b756/go.mod h1:b9TAUYHmRtqA6klRHApnXMnj+OyLce4yF5cZCUbk2ps= -golang.org/x/exp v0.0.0-20231214170342-aacd6d4b4611 h1:qCEDpW1G+vcj3Y7Fy52pEM1AWm3abj8WimGYejI3SC4= -golang.org/x/exp v0.0.0-20231214170342-aacd6d4b4611/go.mod h1:iRJReGqOEeBhDZGkGbynYwcHlctCvnjTYIamk7uXpHI= +golang.org/x/exp v0.0.0-20240222234643-814bf88cf225 h1:LfspQV/FYTatPTr/3HzIcmiUFH7PGP+OQ6mgDYo3yuQ= +golang.org/x/exp v0.0.0-20240222234643-814bf88cf225/go.mod h1:CxmFvTBINI24O/j8iY7H1xHzx2i4OsyguNBmN/uPtqc= golang.org/x/image v0.0.0-20180708004352-c73c2afc3b81/go.mod h1:ux5Hcp/YLpHSI86hEcLt0YII63i6oz57MZXIpbrjZUs= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= @@ -1144,8 +1178,9 @@ golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.5.1/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro= golang.org/x/mod v0.6.0-dev.0.20211013180041-c96bc1413d57/go.mod h1:3p9vT2HGsQu2K1YbXdKPJLVgG5VJdoTa1poYQBtP1AY= golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3/go.mod h1:3p9vT2HGsQu2K1YbXdKPJLVgG5VJdoTa1poYQBtP1AY= -golang.org/x/mod v0.14.0 h1:dGoOF9QVLYng8IHTm7BAyWqCqSheQ5pYWGhzW00YJr0= -golang.org/x/mod v0.14.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/mod v0.16.0 h1:QX4fJ0Rr5cPQCF7O9lh9Se4pmwfwskqZfq5moyldzic= +golang.org/x/mod v0.16.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/net v0.0.0-20170114055629-f2499483f923/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -1205,6 +1240,7 @@ golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qx golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211118161319-6a13c67c3ce4/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.23.0 h1:7EYJ93RZ9vYSZAIb2x3lnuvqO5zneoD6IvWjuhfxjTs= golang.org/x/net v0.23.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -1220,8 +1256,8 @@ golang.org/x/oauth2 v0.0.0-20210220000619-9bb904979d93/go.mod h1:KelEdhl1UZF7XfJ golang.org/x/oauth2 v0.0.0-20210313182246-cd4f82c27b84/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210427180440-81ed05c6b58c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.7.0 h1:qe6s0zUXlPX80/dITx3440hWZ7GwMwgDDyrSGTPJG/g= -golang.org/x/oauth2 v0.7.0/go.mod h1:hPLQkd9LyjfXTiRohC/41GhcFqxisoUQ99sCUOHO9x4= +golang.org/x/oauth2 v0.16.0 h1:aDkGMBSYxElaoP81NpoUoz2oo2R2wHdZpGToUxfyQrQ= +golang.org/x/oauth2 v0.16.0/go.mod h1:hqZ+0LWXsiVoZpeld6jVt06P3adbS2Uu911W1SsJv2o= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -1234,8 +1270,9 @@ golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.5.0 h1:60k92dhOjHxJkrqnwsfl8KuaHbn/5dl0lUPUklKo3qE= -golang.org/x/sync v0.5.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.6.0 h1:5BMeUDZ7vkXGfEr1x9B4bRcTH4lpkTkpdh0T/J+qjbQ= +golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20170830134202-bb24a47a89ea/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -1322,7 +1359,11 @@ golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20211117180635-dee7805ff2e1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4= golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= @@ -1340,6 +1381,7 @@ golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -1348,8 +1390,9 @@ golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxb golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20200416051211-89c76fbcd5d1/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20201208040808-7e3f01d25324/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba h1:O8mE0/t419eoIwhTFpKVkHiTs/Igowgfkj25AcZrtiE= golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= +golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180525024113-a5b4c53f6e8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -1428,22 +1471,23 @@ golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.4/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.8-0.20211029000441-d6a9af8af023/go.mod h1:nABZi5QlRsZVlzPpHl034qft6wpY4eDcsTt5AaioBiU= golang.org/x/tools v0.1.10/go.mod h1:Uh6Zz+xoGYZom868N8YTex3t7RhtHDBrE8Gzo9bV56E= -golang.org/x/tools v0.16.0 h1:GO788SKMRunPIBCXiQyo2AaexLstOrVhuAL5YwsckQM= -golang.org/x/tools v0.16.0/go.mod h1:kYVVN6I1mBNoB1OX+noeBjbRk4IUEPa7JJ+TJMEooJ0= +golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/tools v0.19.0 h1:tfGCXNR1OsFG+sVdLAitlpjAvD/I6dHDKnYrpEZUHkw= +golang.org/x/tools v0.19.0/go.mod h1:qoJWxmGSIBmAeriMx19ogtrEPrGtDbPK634QFIcLAhc= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20220411194840-2f41105eb62f/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 h1:H2TDz8ibqkAF6YGhCdN3jS9O0/s90v0rJh3X/OLHEUk= -golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2/go.mod h1:K8+ghG5WaK9qNqU5K3HdILfMLy1f3aNYFI/wnl100a8= +golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 h1:+cNy6SZtPcJQH3LJVLOSmiC7MMxXNOb3PU/VUEz+EhU= +golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028/go.mod h1:NDW/Ps6MPRej6fsCIbMTohpP40sJ/P/vI1MoTEGwX90= gonum.org/v1/gonum v0.0.0-20180816165407-929014505bf4/go.mod h1:Y+Yx5eoAFn32cQvJDxZx5Dpnq+c3wtXuadVZAcxbbBo= gonum.org/v1/gonum v0.0.0-20181121035319-3f7ecaa7e8ca/go.mod h1:Y+Yx5eoAFn32cQvJDxZx5Dpnq+c3wtXuadVZAcxbbBo= gonum.org/v1/gonum v0.6.0/go.mod h1:9mxDZsDKxgMAuccQkewq682L+0eCu4dCN2yonUJTCLU= gonum.org/v1/gonum v0.8.2/go.mod h1:oe/vMfY3deqTw+1EZJhuvEW2iwGF1bW9wwu7XCu0+v0= gonum.org/v1/gonum v0.9.3/go.mod h1:TZumC3NeyVQskjXqmyWt4S3bINhy7B4eYwW69EbyX+0= -gonum.org/v1/gonum v0.11.0 h1:f1IJhK4Km5tBJmaiJXtk/PkL4cdVX6J+tGiM187uT5E= -gonum.org/v1/gonum v0.11.0/go.mod h1:fSG4YDCxxUZQJ7rKsQrj0gMOg00Il0Z96/qMA4bVQhA= +gonum.org/v1/gonum v0.15.0 h1:2lYxjRbTYyxkJxlhC+LvJIx3SsANPdRybu1tGj9/OrQ= +gonum.org/v1/gonum v0.15.0/go.mod h1:xzZVBJBtS+Mz4q0Yl2LJTk+OxOg4jiXZ7qBoM0uISGo= gonum.org/v1/netlib v0.0.0-20181029234149-ec6d1f5cefe6/go.mod h1:wa6Ws7BG/ESfp6dHfk7C6KdzKA7wR7u/rKwOGE66zvw= gonum.org/v1/netlib v0.0.0-20190313105609-8cb42192e0e0/go.mod h1:wa6Ws7BG/ESfp6dHfk7C6KdzKA7wR7u/rKwOGE66zvw= gonum.org/v1/plot v0.0.0-20190515093506-e2840ee46a6b/go.mod h1:Wt8AAjI+ypCyYX3nZBvf6cAIx93T+c/OS2HFAYskSZc= @@ -1472,8 +1516,8 @@ google.golang.org/api v0.41.0/go.mod h1:RkxM5lITDfTzmyKFPt+wGrCJbVfniCr2ool8kTBz google.golang.org/api v0.43.0/go.mod h1:nQsDGjRXMo4lvh5hP0TKqF244gqhGcr/YSIykhUk/94= google.golang.org/api v0.46.0/go.mod h1:ceL4oozhkAiTID8XMmJBsIxID/9wMXJVVFXPg4ylg3I= google.golang.org/api v0.47.0/go.mod h1:Wbvgpq1HddcWVtzsVLyfLp8lDg6AA241LmgIL59tHXo= -google.golang.org/api v0.114.0 h1:1xQPji6cO2E2vLiI+C/XiFAnsn1WV3mjaEwGLhi3grE= -google.golang.org/api v0.114.0/go.mod h1:ifYI2ZsFK6/uGddGfAD5BMxlnkBqCmqHSDUVi45N5Yg= +google.golang.org/api v0.155.0 h1:vBmGhCYs0djJttDNynWo44zosHlPvHmA0XiN2zP2DtA= +google.golang.org/api v0.155.0/go.mod h1:GI5qK5f40kCpHfPn6+YzGAByIKWv8ujFnmoWm7Igduk= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.2.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= @@ -1481,8 +1525,9 @@ google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7 google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= -google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/appengine v1.6.8 h1:IhEN5q69dyKagZPYMSdIjS2HqprW324FRQZJcGqPAsM= +google.golang.org/appengine v1.6.8/go.mod h1:1jJ3jBArFh5pcgW8gCtRJnepW8FzD1V44FJffLiz/Ds= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= @@ -1533,8 +1578,12 @@ google.golang.org/genproto v0.0.0-20210513213006-bf773b8c8384/go.mod h1:P3QM42oQ google.golang.org/genproto v0.0.0-20210517163617-5e0236093d7a/go.mod h1:P3QM42oQyzQSnHPnZ/vqoCdDmzH28fzWByN9asMeM8A= google.golang.org/genproto v0.0.0-20210601144548-a796c710e9b6/go.mod h1:P3QM42oQyzQSnHPnZ/vqoCdDmzH28fzWByN9asMeM8A= google.golang.org/genproto v0.0.0-20210630183607-d20f26d13c79/go.mod h1:yiaVoXHpRzHGyxV3o4DktVWY4mSUErTKaeEOq6C3t3U= -google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 h1:KpwkzHKEF7B9Zxg18WzOa7djJ+Ha5DzthMyZYQfEn2A= -google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1/go.mod h1:nKE/iIaLqn2bQwXBg8f1g2Ylh6r5MN5CmZvuzZCgsCU= +google.golang.org/genproto v0.0.0-20240123012728-ef4313101c80 h1:KAeGQVN3M9nD0/bQXnr/ClcEMJ968gUXJQ9pwfSynuQ= +google.golang.org/genproto v0.0.0-20240123012728-ef4313101c80/go.mod h1:cc8bqMqtv9gMOr0zHg2Vzff5ULhhL2IXP4sbcn32Dro= +google.golang.org/genproto/googleapis/api v0.0.0-20240123012728-ef4313101c80 h1:Lj5rbfG876hIAYFjqiJnPHfhXbv+nzTWfm04Fg/XSVU= +google.golang.org/genproto/googleapis/api v0.0.0-20240123012728-ef4313101c80/go.mod h1:4jWUdICTdgc3Ibxmr8nAJiiLHwQBY0UI0XZcEMaFKaA= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240123012728-ef4313101c80 h1:AjyfHzEPEFp/NpvfN5g+KDla3EMojjhRVZc1i7cj+oM= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240123012728-ef4313101c80/go.mod h1:PAREbraiVEVGVdTZsVWjSbbTtSyGbAgIIvni8a8CD5s= google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.0/go.mod h1:chYK+tFQF0nDUGJgXMSgLCQk3phJEuONr2DCgLDdAQM= @@ -1566,8 +1615,8 @@ google.golang.org/grpc v1.37.1/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQ google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= google.golang.org/grpc v1.39.0/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnDzfrE= google.golang.org/grpc v1.41.0/go.mod h1:U3l9uK9J0sini8mHphKoXyaqDA/8VyGnDee1zzIUK6k= -google.golang.org/grpc v1.56.3 h1:8I4C0Yq1EjstUzUJzpcRVbuYA2mODtEmpWiQoN/b2nc= -google.golang.org/grpc v1.56.3/go.mod h1:I9bI3vqKfayGqPUAwGdOSu7kt6oIJLixfffKrpXqQ9s= +google.golang.org/grpc v1.62.1 h1:B4n+nfKzOICUXMgyrNd19h/I9oH0L1pizfk1d4zSgTk= +google.golang.org/grpc v1.62.1/go.mod h1:IWTG0VlJLCh1SkC58F7np9ka9mx/WNkjl4PGJaiq+QE= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= From 9ed1d01fb6a05bad0c6cb79558c9709861f1fcff Mon Sep 17 00:00:00 2001 From: Sven Rebhan Date: Wed, 18 Sep 2024 12:45:08 +0200 Subject: [PATCH 02/14] chore: Wrap errors in influx_tools main --- cmd/influx_tools/main.go | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/cmd/influx_tools/main.go b/cmd/influx_tools/main.go index 28a2e89aa79..88946cc705a 100644 --- a/cmd/influx_tools/main.go +++ b/cmd/influx_tools/main.go @@ -56,41 +56,41 @@ func (m *Main) Run(args ...string) error { switch name { case "", "help": if err := help.NewCommand().Run(args...); err != nil { - return fmt.Errorf("help failed: %s", err) + return fmt.Errorf("help failed: %w", err) } case "compact-shard": c := compact.NewCommand() if err := c.Run(args); err != nil { - return fmt.Errorf("compact-shard failed: %s", err) + return fmt.Errorf("compact-shard failed: %w", err) } case "export": c := export.NewCommand(&ossServer{logger: zap.NewNop()}) if err := c.Run(args); err != nil { - return fmt.Errorf("export failed: %s", err) + return fmt.Errorf("export failed: %w", err) } case "export-parquet": c := parquet.NewCommand(&ossServer{logger: zap.NewNop()}) if err := c.Run(args); err != nil { - return fmt.Errorf("export failed: %s", err) + return fmt.Errorf("export failed: %w", err) } case "import": c := importer.NewCommand(&ossServer{logger: zap.NewNop()}) if err := c.Run(args); err != nil { - return fmt.Errorf("import failed: %s", err) + return fmt.Errorf("import failed: %w", err) } case "gen-init": c := geninit.NewCommand(&ossServer{logger: zap.NewNop()}) if err := c.Run(args); err != nil { - return fmt.Errorf("gen-init failed: %s", err) + return fmt.Errorf("gen-init failed: %w", err) } case "gen-exec": deps := genexec.Dependencies{Server: &ossServer{logger: zap.NewNop()}} c := genexec.NewCommand(deps) if err := c.Run(args); err != nil { - return fmt.Errorf("gen-exec failed: %s", err) + return fmt.Errorf("gen-exec failed: %w", err) } default: - return fmt.Errorf(`unknown command "%s"`+"\n"+`Run 'influx-tools help' for usage`+"\n\n", name) + return fmt.Errorf("unknown command %q\nRun 'influx-tools help' for usage", name) } return nil @@ -112,7 +112,7 @@ func (s *ossServer) Open(path string) (err error) { // Validate the configuration. if err = s.config.Validate(); err != nil { - return fmt.Errorf("validate config: %s", err) + return fmt.Errorf("validate config: %w", err) } if s.noClient { From c12f2932281e25abdde82c0d434693d55d8cabc0 Mon Sep 17 00:00:00 2001 From: Sven Rebhan Date: Wed, 18 Sep 2024 12:48:09 +0200 Subject: [PATCH 03/14] chore: Do not create unused series cursor and simplify batcher creation --- cmd/influx_tools/parquet/batcher.go | 29 ---------------------------- cmd/influx_tools/parquet/exporter.go | 11 +++++++---- 2 files changed, 7 insertions(+), 33 deletions(-) diff --git a/cmd/influx_tools/parquet/batcher.go b/cmd/influx_tools/parquet/batcher.go index 43c8af25fd3..f96d8404334 100644 --- a/cmd/influx_tools/parquet/batcher.go +++ b/cmd/influx_tools/parquet/batcher.go @@ -8,7 +8,6 @@ import ( "github.com/influxdata/influxdb/models" "github.com/influxdata/influxdb/tsdb" - "github.com/influxdata/influxql" ) type row struct { @@ -28,34 +27,6 @@ type batcher struct { start int64 } -func newBatcher( - ctx context.Context, - shard *tsdb.Shard, - measurement string, - series []seriesEntry, - converter map[string]func(interface{}) (interface{}, error), - nameMapping map[string]string, -) (*batcher, error) { - seriesCursor, err := shard.CreateSeriesCursor( - ctx, - tsdb.SeriesCursorRequest{}, - influxql.MustParseExpr("_name = '"+measurement+"'"), - ) - if err != nil { - return nil, fmt.Errorf("getting series cursor failed: %w", err) - } - defer seriesCursor.Close() - - return &batcher{ - measurement: []byte(measurement), - shard: shard, - series: series, - converter: converter, - nameMapping: nameMapping, - start: models.MinNanoTime, - }, nil -} - func (b *batcher) reset() { b.start = models.MinNanoTime } diff --git a/cmd/influx_tools/parquet/exporter.go b/cmd/influx_tools/parquet/exporter.go index 4839fd3b9b7..e245c3c6bfc 100644 --- a/cmd/influx_tools/parquet/exporter.go +++ b/cmd/influx_tools/parquet/exporter.go @@ -353,10 +353,13 @@ func (e *exporter) exportMeasurement(ctx context.Context, shard *tsdb.Shard, mea } // Create a batch processor - series := creator.series[shard.ID()] - batcher, err := newBatcher(ctx, shard, measurement, series, creator.typeConverters, creator.nameResolutions) - if err != nil { - return fmt.Errorf("creating batcher failed: %w", err) + batcher := &batcher{ + measurement: []byte(measurement), + shard: shard, + series: creator.series[shard.ID()], + converter: creator.typeConverters, + nameMapping: creator.nameResolutions, + start: models.MinNanoTime, } // Check if we do have data for the measurement and skip the shard if From a2367ee1d9a15c12f239cc65da9ade03dec0bc17 Mon Sep 17 00:00:00 2001 From: Sven Rebhan Date: Wed, 18 Sep 2024 12:54:40 +0200 Subject: [PATCH 04/14] chore: Move converter creation to batcher as it is only used there --- cmd/influx_tools/parquet/batcher.go | 33 ++++++++++++++++++++++--- cmd/influx_tools/parquet/exporter.go | 23 ++++++++++++------ cmd/influx_tools/parquet/schema.go | 36 ---------------------------- 3 files changed, 46 insertions(+), 46 deletions(-) diff --git a/cmd/influx_tools/parquet/batcher.go b/cmd/influx_tools/parquet/batcher.go index f96d8404334..73ea40db992 100644 --- a/cmd/influx_tools/parquet/batcher.go +++ b/cmd/influx_tools/parquet/batcher.go @@ -8,6 +8,7 @@ import ( "github.com/influxdata/influxdb/models" "github.com/influxdata/influxdb/tsdb" + "github.com/influxdata/influxql" ) type row struct { @@ -20,13 +21,39 @@ type batcher struct { measurement []byte shard *tsdb.Shard - converter map[string]func(interface{}) (interface{}, error) - nameMapping map[string]string + typeResolutions map[string]influxql.DataType + converter map[string]func(interface{}) (interface{}, error) + nameResolutions map[string]string series []seriesEntry start int64 } +func (b *batcher) init() error { + // Setup the type converters for the conflicting fields + b.converter = make(map[string]func(interface{}) (interface{}, error), len(b.typeResolutions)) + for field, ftype := range b.typeResolutions { + switch ftype { + case influxql.Float: + b.converter[field] = toFloat + case influxql.Unsigned: + b.converter[field] = toUint + case influxql.Integer: + b.converter[field] = toInt + case influxql.Boolean: + b.converter[field] = toBool + case influxql.String: + b.converter[field] = toString + default: + return fmt.Errorf("unknown converter %v for field %q", ftype, field) + } + } + + b.start = models.MinNanoTime + + return nil +} + func (b *batcher) reset() { b.start = models.MinNanoTime } @@ -66,7 +93,7 @@ func (b *batcher) next(ctx context.Context) ([]row, error) { // Prepare mappings fname := field - if n, found := b.nameMapping[field]; found { + if n, found := b.nameResolutions[field]; found { fname = n } converter := identity diff --git a/cmd/influx_tools/parquet/exporter.go b/cmd/influx_tools/parquet/exporter.go index e245c3c6bfc..882f0f7cc07 100644 --- a/cmd/influx_tools/parquet/exporter.go +++ b/cmd/influx_tools/parquet/exporter.go @@ -218,7 +218,13 @@ func (e *exporter) open(ctx context.Context) error { // Collect the schemata for all measurments e.schemata = make(map[string]*schemaCreator, len(e.measurements)) for _, m := range e.measurements { - creator := newSchemaCreator(m, e.shards, e.typeResolutions[m], e.nameResolutions[m]) + creator := &schemaCreator{ + measurement: m, + shards: e.shards, + series: make(map[uint64][]seriesEntry, len(e.shards)), + typeResolutions: e.typeResolutions[m], + nameResolutions: e.nameResolutions[m], + } if err := creator.extractSchema(ctx); err != nil { return fmt.Errorf("extracting schema for measurement %q failed: %w", m, err) } @@ -354,12 +360,15 @@ func (e *exporter) exportMeasurement(ctx context.Context, shard *tsdb.Shard, mea // Create a batch processor batcher := &batcher{ - measurement: []byte(measurement), - shard: shard, - series: creator.series[shard.ID()], - converter: creator.typeConverters, - nameMapping: creator.nameResolutions, - start: models.MinNanoTime, + measurement: []byte(measurement), + shard: shard, + series: creator.series[shard.ID()], + typeResolutions: creator.typeResolutions, + nameResolutions: creator.nameResolutions, + start: models.MinNanoTime, + } + if err := batcher.init(); err != nil { + return fmt.Errorf("creating batcher failed: %w", err) } // Check if we do have data for the measurement and skip the shard if diff --git a/cmd/influx_tools/parquet/schema.go b/cmd/influx_tools/parquet/schema.go index 45801663e72..c67d1188cda 100644 --- a/cmd/influx_tools/parquet/schema.go +++ b/cmd/influx_tools/parquet/schema.go @@ -31,45 +31,9 @@ type schemaCreator struct { conflicts map[string][]influxql.DataType typeResolutions map[string]influxql.DataType - typeConverters map[string]func(interface{}) (interface{}, error) nameResolutions map[string]string } -func newSchemaCreator( - measurement string, - shards []*tsdb.Shard, - typeResolutions map[string]influxql.DataType, - nameResolutions map[string]string, -) *schemaCreator { - s := &schemaCreator{ - measurement: measurement, - shards: shards, - series: make(map[uint64][]seriesEntry, len(shards)), - typeResolutions: typeResolutions, - nameResolutions: nameResolutions, - } - - // Setup the type converters for the conflicting fields - s.typeConverters = make(map[string]func(interface{}) (interface{}, error), len(s.typeResolutions)) - for field, ftype := range s.typeResolutions { - switch ftype { - case influxql.Float: - s.typeConverters[field] = toFloat - case influxql.Unsigned: - s.typeConverters[field] = toUint - case influxql.Integer: - s.typeConverters[field] = toInt - case influxql.Boolean: - s.typeConverters[field] = toBool - case influxql.String: - s.typeConverters[field] = toString - default: - panic(fmt.Errorf("unknown converter %v for field %q", ftype, field)) - } - } - return s -} - func (s *schemaCreator) extractSchema(ctx context.Context) error { // Iterate over the shards and extract all series for _, shard := range s.shards { From 41dacceef5b14a2ea035a73d4c796bbf77a8d92e Mon Sep 17 00:00:00 2001 From: Sven Rebhan Date: Wed, 18 Sep 2024 12:58:01 +0200 Subject: [PATCH 05/14] fix: Caputure error when closing series cursor --- cmd/influx_tools/parquet/schema.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/cmd/influx_tools/parquet/schema.go b/cmd/influx_tools/parquet/schema.go index c67d1188cda..2193a5441a1 100644 --- a/cmd/influx_tools/parquet/schema.go +++ b/cmd/influx_tools/parquet/schema.go @@ -9,6 +9,7 @@ import ( "github.com/apache/arrow/go/v16/arrow" "github.com/influxdata/influxdb/models" + "github.com/influxdata/influxdb/pkg/errors" "github.com/influxdata/influxdb/tsdb" "github.com/influxdata/influxql" ) @@ -34,7 +35,7 @@ type schemaCreator struct { nameResolutions map[string]string } -func (s *schemaCreator) extractSchema(ctx context.Context) error { +func (s *schemaCreator) extractSchema(ctx context.Context) (err error) { // Iterate over the shards and extract all series for _, shard := range s.shards { // Extract all fields of the measurement @@ -53,7 +54,7 @@ func (s *schemaCreator) extractSchema(ctx context.Context) error { if err != nil { return fmt.Errorf("getting series cursor failed: %w", err) } - defer seriesCursor.Close() + defer errors.Capture(&err, seriesCursor.Close) for { cur, err := seriesCursor.Next() From b7c9475bfbd9a749a69ee31fb5289734c4678103 Mon Sep 17 00:00:00 2001 From: Sven Rebhan Date: Wed, 18 Sep 2024 13:02:47 +0200 Subject: [PATCH 06/14] feat: Print shard series-file path on error --- cmd/influx_tools/parquet/exporter.go | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/cmd/influx_tools/parquet/exporter.go b/cmd/influx_tools/parquet/exporter.go index 882f0f7cc07..a0cf5f58938 100644 --- a/cmd/influx_tools/parquet/exporter.go +++ b/cmd/influx_tools/parquet/exporter.go @@ -334,7 +334,13 @@ func (e *exporter) export(ctx context.Context) error { for _, measurement := range e.measurements { if err := e.exportMeasurement(ctx, shard, measurement); err != nil { - return fmt.Errorf("exporting measurement %q in shard %d failed: %w", measurement, shard.ID(), err) + var path string + if f, serr := shard.SeriesFile(); serr != nil { + path = "ERR:" + serr.Error() + } else { + path = f.Path() + } + return fmt.Errorf("exporting measurement %q in shard %d at %q failed: %w", measurement, shard.ID(), path, err) } } fmt.Fprintf(e.writer, "Finished export of shard %d in %v...\n", shard.ID(), time.Since(start)) From 182195f34a0da4915a1475da7c58c3e00a9646d7 Mon Sep 17 00:00:00 2001 From: Sven Rebhan Date: Wed, 18 Sep 2024 13:06:51 +0200 Subject: [PATCH 07/14] chore: Replace panic by returning an error --- cmd/influx_tools/parquet/batcher.go | 2 +- cmd/influx_tools/parquet/schema.go | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/cmd/influx_tools/parquet/batcher.go b/cmd/influx_tools/parquet/batcher.go index 73ea40db992..7a54121f027 100644 --- a/cmd/influx_tools/parquet/batcher.go +++ b/cmd/influx_tools/parquet/batcher.go @@ -204,7 +204,7 @@ func (b *batcher) next(ctx context.Context) ([]row, error) { } default: cursor.Close() - panic(fmt.Errorf("unexpected type %T", cursor)) + return nil, fmt.Errorf("unexpected type %T", cursor) } cursor.Close() end = min(end, fieldEnd) diff --git a/cmd/influx_tools/parquet/schema.go b/cmd/influx_tools/parquet/schema.go index 2193a5441a1..5e0f85fa3df 100644 --- a/cmd/influx_tools/parquet/schema.go +++ b/cmd/influx_tools/parquet/schema.go @@ -125,7 +125,7 @@ func (s *schemaCreator) extractSchema(ctx context.Context) (err error) { s.fieldKeys = append(s.fieldKeys, name) default: // Unconvertible type - panic(fmt.Errorf("unconvertible field %q with type %v", name, ftype)) + return fmt.Errorf("unconvertible field %q with type %v", name, ftype) } } sort.Strings(s.fieldKeys) @@ -216,7 +216,7 @@ func (s *schemaCreator) schema(info map[string]string) (*arrow.Schema, error) { case influxql.Unsigned: dtype = &arrow.Uint64Type{} default: - panic(fmt.Errorf("unconvertible field %q", name)) + return nil, fmt.Errorf("unconvertible field %q", name) } columns = append(columns, arrow.Field{ Name: fname, From 795e5812ee6ebe6ea67f1980bc1c9e7062c46e4c Mon Sep 17 00:00:00 2001 From: Sven Rebhan Date: Wed, 18 Sep 2024 14:49:00 +0200 Subject: [PATCH 08/14] feat: Use logger instead of raw printing --- cmd/influx_tools/parquet/batcher.go | 14 ++--- cmd/influx_tools/parquet/command.go | 19 ++++--- cmd/influx_tools/parquet/exporter.go | 81 ++++++++++++++-------------- 3 files changed, 62 insertions(+), 52 deletions(-) diff --git a/cmd/influx_tools/parquet/batcher.go b/cmd/influx_tools/parquet/batcher.go index 7a54121f027..e3bfdfcfec2 100644 --- a/cmd/influx_tools/parquet/batcher.go +++ b/cmd/influx_tools/parquet/batcher.go @@ -3,12 +3,12 @@ package parquet import ( "context" "fmt" - "os" "sort" "github.com/influxdata/influxdb/models" "github.com/influxdata/influxdb/tsdb" "github.com/influxdata/influxql" + "go.uber.org/zap" ) type row struct { @@ -27,6 +27,8 @@ type batcher struct { series []seriesEntry start int64 + + logger *zap.SugaredLogger } func (b *batcher) init() error { @@ -107,7 +109,7 @@ func (b *batcher) next(ctx context.Context) ([]row, error) { for i, t := range values.Timestamps { v, err := converter(values.Values[i]) if err != nil { - fmt.Fprintf(os.Stderr, "converting %v of field %q failed: %v", values.Values[i], field, err) + b.logger.Errorf("converting %v of field %q failed: %v", values.Values[i], field, err) continue } @@ -127,7 +129,7 @@ func (b *batcher) next(ctx context.Context) ([]row, error) { for i, t := range values.Timestamps { v, err := converter(values.Values[i]) if err != nil { - fmt.Fprintf(os.Stderr, "converting %v of field %q failed: %v", values.Values[i], field, err) + b.logger.Errorf("converting %v of field %q failed: %v", values.Values[i], field, err) continue } @@ -147,7 +149,7 @@ func (b *batcher) next(ctx context.Context) ([]row, error) { for i, t := range values.Timestamps { v, err := converter(values.Values[i]) if err != nil { - fmt.Fprintf(os.Stderr, "converting %v of field %q failed: %v", values.Values[i], field, err) + b.logger.Errorf("converting %v of field %q failed: %v", values.Values[i], field, err) continue } @@ -167,7 +169,7 @@ func (b *batcher) next(ctx context.Context) ([]row, error) { for i, t := range values.Timestamps { v, err := converter(values.Values[i]) if err != nil { - fmt.Fprintf(os.Stderr, "converting %v of field %q failed: %v", values.Values[i], field, err) + b.logger.Errorf("converting %v of field %q failed: %v", values.Values[i], field, err) continue } @@ -187,7 +189,7 @@ func (b *batcher) next(ctx context.Context) ([]row, error) { for i, t := range values.Timestamps { v, err := converter(values.Values[i]) if err != nil { - fmt.Fprintf(os.Stderr, "converting %v of field %q failed: %v", values.Values[i], field, err) + b.logger.Errorf("converting %v of field %q failed: %v", values.Values[i], field, err) continue } diff --git a/cmd/influx_tools/parquet/command.go b/cmd/influx_tools/parquet/command.go index 36c972155ee..c38afb41927 100644 --- a/cmd/influx_tools/parquet/command.go +++ b/cmd/influx_tools/parquet/command.go @@ -59,15 +59,23 @@ func (cmd *Command) Run(args []string) (err error) { flags.BoolVar(&dryRun, "dry-run", false, "Print plan and exit") if err := flags.Parse(args); err != nil { - return err + return fmt.Errorf("parsing flags failed: %w", err) } if database == "" { return errors.New("database is required") } + loggerCfg := zap.NewDevelopmentConfig() + loggerCfg.DisableStacktrace = true + loggerCfg.DisableCaller = true + cmd.Logger, err = loggerCfg.Build() + if err != nil { + return fmt.Errorf("creating logger failed: %w", err) + } + if err := cmd.server.Open(configPath); err != nil { - return err + return fmt.Errorf("opening server failed: %w", err) } defer cmd.server.Close() @@ -78,20 +86,19 @@ func (cmd *Command) Run(args []string) (err error) { TypeResolutions: typeResolutions, NameResolutions: nameResolutions, Output: output, - Stderr: cmd.Stderr, } - exp, err := newExporter(cmd.server, cfg) + exp, err := newExporter(cmd.server, cfg, cmd.Logger) if err != nil { return err } ctx := context.Background() if err := exp.open(ctx); err != nil { - return err + return fmt.Errorf("opening exporter failed: %w", err) } defer exp.close() - exp.printPlan() + exp.printPlan(cmd.Stderr) if dryRun { return nil diff --git a/cmd/influx_tools/parquet/exporter.go b/cmd/influx_tools/parquet/exporter.go index a0cf5f58938..02b7fa22740 100644 --- a/cmd/influx_tools/parquet/exporter.go +++ b/cmd/influx_tools/parquet/exporter.go @@ -17,6 +17,7 @@ import ( "github.com/apache/arrow/go/v16/arrow/array" "github.com/apache/arrow/go/v16/parquet" "github.com/apache/arrow/go/v16/parquet/pqarrow" + "go.uber.org/zap" "github.com/influxdata/flux/memory" "github.com/influxdata/influxdb/cmd/influx_tools/server" @@ -33,7 +34,6 @@ type config struct { TypeResolutions string NameResolutions string Output string - Stderr io.Writer } type exporter struct { @@ -59,12 +59,13 @@ type exporter struct { typeResolutions map[string]map[string]influxql.DataType nameResolutions map[string]map[string]string - // Parquet information - writer io.Writer + // Parquet metadata information exportStart time.Time + + logger *zap.SugaredLogger } -func newExporter(server server.Interface, cfg *config) (*exporter, error) { +func newExporter(server server.Interface, cfg *config, logger *zap.Logger) (*exporter, error) { client := server.MetaClient() db := client.Database(cfg.Database) @@ -107,7 +108,7 @@ func newExporter(server server.Interface, cfg *config) (*exporter, error) { typeResolutions: make(map[string]map[string]influxql.DataType), nameResolutions: make(map[string]map[string]string), filenames: make(map[string]int), - writer: cfg.Stderr, + logger: logger.Sugar().Named("exporter"), } // Split the given measurements @@ -238,24 +239,24 @@ func (e *exporter) close() error { return e.store.Close() } -func (e *exporter) printPlan() { - w := tabwriter.NewWriter(e.writer, 10, 8, 1, '\t', 0) +func (e *exporter) printPlan(w io.Writer) { + tw := tabwriter.NewWriter(w, 10, 8, 1, '\t', 0) - fmt.Fprintf(e.writer, "Exporting source data from %s to %s in %d shard group(s):\n", e.startDate, e.endDate, len(e.groups)) - fmt.Fprintln(w, " Group\tStart\tEnd\t#Shards") - fmt.Fprintln(w, " -----\t-----\t---\t-------") + fmt.Fprintf(w, "Exporting source data from %s to %s in %d shard group(s):\n", e.startDate, e.endDate, len(e.groups)) + fmt.Fprintln(tw, " Group\tStart\tEnd\t#Shards") + fmt.Fprintln(tw, " -----\t-----\t---\t-------") for _, g := range e.groups { - fmt.Fprintf(w, " %d\t%s\t%s\t%d\n", g.ID, g.StartTime, g.EndTime, len(g.Shards)) + fmt.Fprintf(tw, " %d\t%s\t%s\t%d\n", g.ID, g.StartTime, g.EndTime, len(g.Shards)) } - fmt.Fprintln(w) + fmt.Fprintln(tw) - fmt.Fprintf(e.writer, "Creating the following schemata for %d measurement(s):\n", len(e.measurements)) + fmt.Fprintf(w, "Creating the following schemata for %d measurement(s):\n", len(e.measurements)) for _, measurement := range e.measurements { creator := e.schemata[measurement] hasConflicts, err := creator.validate() if err != nil { fmt.Fprintf( - e.writer, + w, "!!Measurement %q with conflict(s) in %d tag(s), %d field(s):\n", measurement, len(creator.tags), @@ -263,7 +264,7 @@ func (e *exporter) printPlan() { ) } else if hasConflicts { fmt.Fprintf( - e.writer, + w, "* Measurement %q with resolved conflicts in %d tag(s), %d field(s):\n", measurement, len(creator.tags), @@ -271,7 +272,7 @@ func (e *exporter) printPlan() { ) } else { fmt.Fprintf( - e.writer, + w, " Measurement %q with %d tag(s) and %d field(s):\n", measurement, len(creator.tags), @@ -279,11 +280,11 @@ func (e *exporter) printPlan() { ) } - fmt.Fprintln(w, " Column\tKind\tDatatype") - fmt.Fprintln(w, " ------\t----\t--------") - fmt.Fprintln(w, " time\ttimestamp\ttimestamp (nanosecond)") + fmt.Fprintln(tw, " Column\tKind\tDatatype") + fmt.Fprintln(tw, " ------\t----\t--------") + fmt.Fprintln(tw, " time\ttimestamp\ttimestamp (nanosecond)") for _, name := range creator.tags { - fmt.Fprintf(w, " %s\ttag\tstring\n", name) + fmt.Fprintf(tw, " %s\ttag\tstring\n", name) } for _, name := range creator.fieldKeys { ftype := creator.fields[name].String() @@ -301,15 +302,15 @@ func (e *exporter) printPlan() { if n, found := creator.nameResolutions[name]; found { fname += " -> " + n } - fmt.Fprintf(w, " %s\tfield\t%s\n", fname, ftype) + fmt.Fprintf(tw, " %s\tfield\t%s\n", fname, ftype) } if err != nil { - fmt.Fprintln(w, " ", err) + fmt.Fprintln(tw, " ", err) } - fmt.Fprintln(w) + fmt.Fprintln(tw) } - w.Flush() + tw.Flush() } func (e *exporter) export(ctx context.Context) error { @@ -327,25 +328,25 @@ func (e *exporter) export(ctx context.Context) error { // Export shard-wise e.exportStart = time.Now() - fmt.Fprintf(e.writer, "Starting export at %v...\n", e.exportStart) + e.logger.Info("Starting export...") for _, shard := range e.shards { start := time.Now() - fmt.Fprintf(e.writer, "Starting export of shard %d...\n", shard.ID()) + e.logger.Infof("Starting export of shard %d...", shard.ID()) - for _, measurement := range e.measurements { - if err := e.exportMeasurement(ctx, shard, measurement); err != nil { - var path string + for _, m := range e.measurements { + if err := e.exportMeasurement(ctx, shard, m); err != nil { + path := "unknown" if f, serr := shard.SeriesFile(); serr != nil { - path = "ERR:" + serr.Error() + e.logger.Errorf("determining series file failed: %v", serr) } else { path = f.Path() } - return fmt.Errorf("exporting measurement %q in shard %d at %q failed: %w", measurement, shard.ID(), path, err) + return fmt.Errorf("exporting measurement %q in shard %d at %q failed: %w", m, shard.ID(), path, err) } } - fmt.Fprintf(e.writer, "Finished export of shard %d in %v...\n", shard.ID(), time.Since(start)) + e.logger.Infof("Finished export of shard %d in %v...", shard.ID(), time.Since(start)) } - fmt.Fprintf(e.writer, "Finished export in %v...\n", time.Since(e.exportStart)) + e.logger.Infof("Finished export in %v...", time.Since(e.exportStart)) return nil } @@ -360,7 +361,7 @@ func (e *exporter) exportMeasurement(ctx context.Context, shard *tsdb.Shard, mea } if len(creator.fieldKeys) == 0 { - fmt.Fprintf(e.writer, " Skipping measurement %q without fields\n", measurement) + e.logger.Warnf(" Skipping measurement %q without fields", measurement) return nil } @@ -372,6 +373,7 @@ func (e *exporter) exportMeasurement(ctx context.Context, shard *tsdb.Shard, mea typeResolutions: creator.typeResolutions, nameResolutions: creator.nameResolutions, start: models.MinNanoTime, + logger: e.logger.Named("batcher"), } if err := batcher.init(); err != nil { return fmt.Errorf("creating batcher failed: %w", err) @@ -384,12 +386,12 @@ func (e *exporter) exportMeasurement(ctx context.Context, shard *tsdb.Shard, mea return fmt.Errorf("checking data failed: %w", err) } if len(rows) == 0 { - fmt.Fprintf(e.writer, " Skipping measurement %q without data\n", measurement) + e.logger.Warnf(" Skipping measurement %q without data", measurement) return nil } batcher.reset() - fmt.Fprintf(e.writer, " Exporting measurement %q...\n", measurement) + e.logger.Infof(" Exporting measurement %q...", measurement) // Create a parquet schema for writing the data metadata := map[string]string{ @@ -455,11 +457,10 @@ func (e *exporter) exportMeasurement(ctx context.Context, shard *tsdb.Shard, mea if err := writer.WriteBuffered(record); err != nil { return fmt.Errorf("writing parquet file %q failed: %w", filename, err) } - fmt.Fprintf(e.writer, " exported %d rows in %v\n", len(rows), time.Since(last)) + e.logger.Infof(" exported %d rows in %v", len(rows), time.Since(last)) } - fmt.Fprintf( - e.writer, - " exported %d rows of measurement %q to %q in %v...\n", + e.logger.Infof( + " exported %d rows of measurement %q to %q in %v...", count, measurement, filename, From 59b60e68dfff9a3341fc4527478eb936860e7893 Mon Sep 17 00:00:00 2001 From: Sven Rebhan Date: Wed, 18 Sep 2024 15:01:36 +0200 Subject: [PATCH 09/14] fix: Caputure error when closing exporter --- cmd/influx_tools/parquet/command.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cmd/influx_tools/parquet/command.go b/cmd/influx_tools/parquet/command.go index c38afb41927..e9f8f21e837 100644 --- a/cmd/influx_tools/parquet/command.go +++ b/cmd/influx_tools/parquet/command.go @@ -11,6 +11,7 @@ import ( "go.uber.org/zap" "github.com/influxdata/influxdb/cmd/influx_tools/server" + internal_errors "github.com/influxdata/influxdb/pkg/errors" ) // Command represents the program execution for "store query". @@ -96,7 +97,7 @@ func (cmd *Command) Run(args []string) (err error) { if err := exp.open(ctx); err != nil { return fmt.Errorf("opening exporter failed: %w", err) } - defer exp.close() + defer internal_errors.Capture(&err, exp.close) exp.printPlan(cmd.Stderr) From 390cf30ca71bc4d93a08aa0f98797b322c7cb3ae Mon Sep 17 00:00:00 2001 From: Sven Rebhan Date: Wed, 18 Sep 2024 15:45:50 +0200 Subject: [PATCH 10/14] fix: Caputure more defer errors --- cmd/influx_tools/parquet/exporter.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/cmd/influx_tools/parquet/exporter.go b/cmd/influx_tools/parquet/exporter.go index 02b7fa22740..5857dfd1673 100644 --- a/cmd/influx_tools/parquet/exporter.go +++ b/cmd/influx_tools/parquet/exporter.go @@ -22,6 +22,7 @@ import ( "github.com/influxdata/flux/memory" "github.com/influxdata/influxdb/cmd/influx_tools/server" "github.com/influxdata/influxdb/models" + internal_errors "github.com/influxdata/influxdb/pkg/errors" "github.com/influxdata/influxdb/services/meta" "github.com/influxdata/influxdb/tsdb" "github.com/influxdata/influxql" @@ -351,7 +352,7 @@ func (e *exporter) export(ctx context.Context) error { return nil } -func (e *exporter) exportMeasurement(ctx context.Context, shard *tsdb.Shard, measurement string) error { +func (e *exporter) exportMeasurement(ctx context.Context, shard *tsdb.Shard, measurement string) (err error) { startMeasurement := time.Now() // Get the cumulative scheme with all tags and fields for the measurement @@ -418,7 +419,7 @@ func (e *exporter) exportMeasurement(ctx context.Context, shard *tsdb.Shard, mea if err != nil { return fmt.Errorf("creating file %q failed: %w", filename, err) } - defer file.Close() + defer internal_errors.Capture(&err, file.Close) writer, err := pqarrow.NewFileWriter( schema, @@ -429,7 +430,7 @@ func (e *exporter) exportMeasurement(ctx context.Context, shard *tsdb.Shard, mea if err != nil { return fmt.Errorf("creating parquet writer for file %q failed: %w", filename, err) } - defer writer.Close() + defer internal_errors.Capture(&err, writer.Close) // Prepare the record builder builder := array.NewRecordBuilder(memory.DefaultAllocator, schema) From 3bfe17c88a994b364744605cc02e8f0e1335c5b0 Mon Sep 17 00:00:00 2001 From: Sven Rebhan Date: Wed, 18 Sep 2024 18:31:19 +0200 Subject: [PATCH 11/14] feat: Detect name conflicts after name resolution --- cmd/influx_tools/parquet/schema.go | 36 ++++++++++++++++++++++++++++-- 1 file changed, 34 insertions(+), 2 deletions(-) diff --git a/cmd/influx_tools/parquet/schema.go b/cmd/influx_tools/parquet/schema.go index 5e0f85fa3df..dfd8cae7bd3 100644 --- a/cmd/influx_tools/parquet/schema.go +++ b/cmd/influx_tools/parquet/schema.go @@ -143,6 +143,8 @@ func (s *schemaCreator) extractSchema(ctx context.Context) (err error) { func (s *schemaCreator) validate() (bool, error) { var hasConflicts bool + + // Check for conflicting field types var typeConflicts []string for field := range s.conflicts { hasConflicts = true @@ -151,9 +153,10 @@ func (s *schemaCreator) validate() (bool, error) { } } if len(typeConflicts) > 0 { - return true, fmt.Errorf("unresolved type conflicts for %q", strings.Join(typeConflicts, ",")) + return true, fmt.Errorf("unresolved type conflicts for %s", strings.Join(typeConflicts, ",")) } + // Check for name clashes between tags and fields var nameConflicts []string for _, field := range s.fieldKeys { for _, tag := range s.tags { @@ -166,7 +169,36 @@ func (s *schemaCreator) validate() (bool, error) { } } if len(nameConflicts) > 0 { - return true, fmt.Errorf("unresolved name conflicts for %q", strings.Join(nameConflicts, ",")) + return true, fmt.Errorf("unresolved name conflicts for %s", strings.Join(nameConflicts, ",")) + } + + // Check for name clashes after resolving field names + resolvedFieldKeys := make([]string, 0, len(s.fieldKeys)) + for _, field := range s.fieldKeys { + if n, found := s.nameResolutions[field]; found { + resolvedFieldKeys = append(resolvedFieldKeys, n) + } else { + resolvedFieldKeys = append(resolvedFieldKeys, field) + } + } + var resolvedConflicts []string + for i, field := range resolvedFieldKeys { + origField := s.fieldKeys[i] + for _, tag := range s.tags { + if tag == field { + hasConflicts = true + resolvedConflicts = append(resolvedConflicts, "resolved '"+origField+"' with tag '"+tag+"'") + } + } + for j, f := range resolvedFieldKeys { + if i > j && field == f { + hasConflicts = true + resolvedConflicts = append(resolvedConflicts, "resolved '"+origField+"' with field '"+f+"'") + } + } + } + if len(resolvedConflicts) > 0 { + return true, fmt.Errorf("conflicts after field name resolution for %s", strings.Join(resolvedConflicts, ", ")) } return hasConflicts, nil From 76a88d179e9b662fe4cfa5126e54c418e2bccada Mon Sep 17 00:00:00 2001 From: Sven Rebhan Date: Thu, 19 Sep 2024 15:52:33 +0200 Subject: [PATCH 12/14] fix: Make sure deferred functions are actually called --- cmd/influx_tools/parquet/command.go | 2 +- cmd/influx_tools/parquet/exporter.go | 6 ++++-- cmd/influx_tools/parquet/schema.go | 2 +- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/cmd/influx_tools/parquet/command.go b/cmd/influx_tools/parquet/command.go index e9f8f21e837..ba4ee63d986 100644 --- a/cmd/influx_tools/parquet/command.go +++ b/cmd/influx_tools/parquet/command.go @@ -97,7 +97,7 @@ func (cmd *Command) Run(args []string) (err error) { if err := exp.open(ctx); err != nil { return fmt.Errorf("opening exporter failed: %w", err) } - defer internal_errors.Capture(&err, exp.close) + defer internal_errors.Capture(&err, exp.close)() exp.printPlan(cmd.Stderr) diff --git a/cmd/influx_tools/parquet/exporter.go b/cmd/influx_tools/parquet/exporter.go index 5857dfd1673..cf33b2159bc 100644 --- a/cmd/influx_tools/parquet/exporter.go +++ b/cmd/influx_tools/parquet/exporter.go @@ -419,7 +419,6 @@ func (e *exporter) exportMeasurement(ctx context.Context, shard *tsdb.Shard, mea if err != nil { return fmt.Errorf("creating file %q failed: %w", filename, err) } - defer internal_errors.Capture(&err, file.Close) writer, err := pqarrow.NewFileWriter( schema, @@ -428,9 +427,12 @@ func (e *exporter) exportMeasurement(ctx context.Context, shard *tsdb.Shard, mea pqarrow.NewArrowWriterProperties(pqarrow.WithCoerceTimestamps(arrow.Nanosecond)), ) if err != nil { + if err := file.Close(); err != nil { + e.logger.Errorf("closing file failed: %v", err) + } return fmt.Errorf("creating parquet writer for file %q failed: %w", filename, err) } - defer internal_errors.Capture(&err, writer.Close) + defer internal_errors.Capture(&err, writer.Close)() // Prepare the record builder builder := array.NewRecordBuilder(memory.DefaultAllocator, schema) diff --git a/cmd/influx_tools/parquet/schema.go b/cmd/influx_tools/parquet/schema.go index dfd8cae7bd3..958849d206c 100644 --- a/cmd/influx_tools/parquet/schema.go +++ b/cmd/influx_tools/parquet/schema.go @@ -54,7 +54,7 @@ func (s *schemaCreator) extractSchema(ctx context.Context) (err error) { if err != nil { return fmt.Errorf("getting series cursor failed: %w", err) } - defer errors.Capture(&err, seriesCursor.Close) + defer errors.Capture(&err, seriesCursor.Close)() for { cur, err := seriesCursor.Next() From f2423af67b3239d9e054d7e5642a0d6edfcabb04 Mon Sep 17 00:00:00 2001 From: Sven Rebhan Date: Thu, 19 Sep 2024 15:38:34 +0200 Subject: [PATCH 13/14] feat: Move out cursor handling --- cmd/influx_tools/parquet/batcher.go | 122 ++++----------- cmd/influx_tools/parquet/cursors.go | 213 +++++++++++++++++++++++++++ cmd/influx_tools/parquet/exporter.go | 2 +- 3 files changed, 239 insertions(+), 98 deletions(-) create mode 100644 cmd/influx_tools/parquet/cursors.go diff --git a/cmd/influx_tools/parquet/batcher.go b/cmd/influx_tools/parquet/batcher.go index e3bfdfcfec2..e91f300fb7a 100644 --- a/cmd/influx_tools/parquet/batcher.go +++ b/cmd/influx_tools/parquet/batcher.go @@ -5,10 +5,11 @@ import ( "fmt" "sort" + "go.uber.org/zap" + "github.com/influxdata/influxdb/models" "github.com/influxdata/influxdb/tsdb" "github.com/influxdata/influxql" - "go.uber.org/zap" ) type row struct { @@ -103,112 +104,39 @@ func (b *batcher) next(ctx context.Context) ([]row, error) { converter = c } fieldEnd := models.MaxNanoTime - switch c := cursor.(type) { - case tsdb.IntegerArrayCursor: - values := c.Next() - for i, t := range values.Timestamps { - v, err := converter(values.Values[i]) - if err != nil { - b.logger.Errorf("converting %v of field %q failed: %v", values.Values[i], field, err) - continue - } - - if _, found := data[s.key][t]; !found { - data[s.key][t] = row{ - timestamp: t, - tags: tags, - fields: make(map[string]interface{}), - } - } - data[s.key][t].fields[fname] = v - fieldEnd = t - } - case tsdb.FloatArrayCursor: - values := c.Next() - for i, t := range values.Timestamps { - v, err := converter(values.Values[i]) - if err != nil { - b.logger.Errorf("converting %v of field %q failed: %v", values.Values[i], field, err) - continue - } - - if _, found := data[s.key][t]; !found { - data[s.key][t] = row{ - timestamp: t, - tags: tags, - fields: make(map[string]interface{}), - } - } + c, err := newValueCursor(cursor) + if err != nil { + return nil, fmt.Errorf("creating value cursor failed: %w", err) + } - data[s.key][t].fields[fname] = v - fieldEnd = t + for { + // Check if we do still have data + timestamp, ok := c.peek() + if !ok { + break } - case tsdb.UnsignedArrayCursor: - values := c.Next() - for i, t := range values.Timestamps { - v, err := converter(values.Values[i]) - if err != nil { - b.logger.Errorf("converting %v of field %q failed: %v", values.Values[i], field, err) - continue - } - - if _, found := data[s.key][t]; !found { - data[s.key][t] = row{ - timestamp: t, - tags: tags, - fields: make(map[string]interface{}), - } - } - data[s.key][t].fields[fname] = v - fieldEnd = t + timestamp, value := c.next() + v, err := converter(value) + if err != nil { + b.logger.Errorf("converting %v of field %q failed: %v", value, field, err) + continue } - case tsdb.BooleanArrayCursor: - values := c.Next() - for i, t := range values.Timestamps { - v, err := converter(values.Values[i]) - if err != nil { - b.logger.Errorf("converting %v of field %q failed: %v", values.Values[i], field, err) - continue - } - if _, found := data[s.key][t]; !found { - data[s.key][t] = row{ - timestamp: t, - tags: tags, - fields: make(map[string]interface{}), - } + if _, found := data[s.key][timestamp]; !found { + data[s.key][timestamp] = row{ + timestamp: timestamp, + tags: tags, + fields: make(map[string]interface{}), } - - data[s.key][t].fields[fname] = v - fieldEnd = t } - case tsdb.StringArrayCursor: - values := c.Next() - for i, t := range values.Timestamps { - v, err := converter(values.Values[i]) - if err != nil { - b.logger.Errorf("converting %v of field %q failed: %v", values.Values[i], field, err) - continue - } - - if _, found := data[s.key][t]; !found { - data[s.key][t] = row{ - timestamp: t, - tags: tags, - fields: make(map[string]interface{}), - } - } - data[s.key][t].fields[fname] = v - fieldEnd = t - } - default: - cursor.Close() - return nil, fmt.Errorf("unexpected type %T", cursor) + data[s.key][timestamp].fields[fname] = v + fieldEnd = timestamp } - cursor.Close() + + c.close() end = min(end, fieldEnd) } } diff --git a/cmd/influx_tools/parquet/cursors.go b/cmd/influx_tools/parquet/cursors.go new file mode 100644 index 00000000000..cb871d12695 --- /dev/null +++ b/cmd/influx_tools/parquet/cursors.go @@ -0,0 +1,213 @@ +package parquet + +import ( + "fmt" + + "github.com/influxdata/influxdb/tsdb" + "github.com/influxdata/influxdb/tsdb/cursors" +) + +type valueCursor interface { + next() (int64, interface{}) + peek() (int64, bool) + close() +} + +func newValueCursor(cursor cursors.Cursor) (valueCursor, error) { + switch c := cursor.(type) { + case tsdb.FloatArrayCursor: + return &floatValueCursor{cur: c}, nil + case tsdb.UnsignedArrayCursor: + return &uintValueCursor{cur: c}, nil + case tsdb.IntegerArrayCursor: + return &intValueCursor{cur: c}, nil + case tsdb.BooleanArrayCursor: + return &boolValueCursor{cur: c}, nil + case tsdb.StringArrayCursor: + return &stringValueCursor{cur: c}, nil + } + return nil, fmt.Errorf("unexpected type %T", cursor) + +} + +type floatValueCursor struct { + cur tsdb.FloatArrayCursor + arr *cursors.FloatArray + idx int +} + +func (c *floatValueCursor) next() (int64, interface{}) { + // Initialize the array on first call + if c.arr == nil { + c.idx = 0 + c.arr = c.cur.Next() + } + // Indicate no elements early + if c.arr.Len() == 0 || c.idx >= c.arr.Len() { + return 0, nil + } + + defer func() { c.idx++ }() + return c.arr.Timestamps[c.idx], c.arr.Values[c.idx] +} + +func (c *floatValueCursor) peek() (int64, bool) { + // Initialize the array on first call + if c.arr == nil { + c.idx = 0 + c.arr = c.cur.Next() + } + // Indicate no elements early + if c.arr.Len() == 0 || c.idx >= c.arr.Len() { + return 0, false + } + return c.arr.Timestamps[c.idx], true +} + +func (c *floatValueCursor) close() { + c.cur.Close() +} + +type uintValueCursor struct { + cur tsdb.UnsignedArrayCursor + arr *cursors.UnsignedArray + idx int +} + +func (c *uintValueCursor) next() (int64, interface{}) { + // Initialize the array on first call + if c.arr == nil { + c.arr = c.cur.Next() + } + // Indicate no elements early + if c.arr.Len() == 0 || c.idx >= c.arr.Len() { + return 0, nil + } + defer func() { c.idx++ }() + return c.arr.Timestamps[c.idx], c.arr.Values[c.idx] +} + +func (c *uintValueCursor) peek() (int64, bool) { + // Initialize the array on first call + if c.arr == nil { + c.idx = 0 + c.arr = c.cur.Next() + } + // Indicate no elements early + if c.arr.Len() == 0 || c.idx >= c.arr.Len() { + return 0, false + } + return c.arr.Timestamps[c.idx], true +} + +func (c *uintValueCursor) close() { + c.cur.Close() +} + +type intValueCursor struct { + cur tsdb.IntegerArrayCursor + arr *cursors.IntegerArray + idx int +} + +func (c *intValueCursor) next() (int64, interface{}) { + // Initialize the array on first call + if c.arr == nil { + c.arr = c.cur.Next() + } + // Indicate no elements early + if c.arr.Len() == 0 || c.idx >= c.arr.Len() { + return 0, nil + } + defer func() { c.idx++ }() + return c.arr.Timestamps[c.idx], c.arr.Values[c.idx] +} + +func (c *intValueCursor) peek() (int64, bool) { + // Initialize the array on first call + if c.arr == nil { + c.idx = 0 + c.arr = c.cur.Next() + } + // Indicate no elements early + if c.arr.Len() == 0 || c.idx >= c.arr.Len() { + return 0, false + } + return c.arr.Timestamps[c.idx], true +} + +func (c *intValueCursor) close() { + c.cur.Close() +} + +type boolValueCursor struct { + cur tsdb.BooleanArrayCursor + arr *cursors.BooleanArray + idx int +} + +func (c *boolValueCursor) next() (int64, interface{}) { + // Initialize the array on first call + if c.arr == nil { + c.arr = c.cur.Next() + } + // Indicate no elements early + if c.arr.Len() == 0 || c.idx >= c.arr.Len() { + return 0, nil + } + defer func() { c.idx++ }() + return c.arr.Timestamps[c.idx], c.arr.Values[c.idx] +} + +func (c *boolValueCursor) peek() (int64, bool) { + // Initialize the array on first call + if c.arr == nil { + c.idx = 0 + c.arr = c.cur.Next() + } + // Indicate no elements early + if c.arr.Len() == 0 || c.idx >= c.arr.Len() { + return 0, false + } + return c.arr.Timestamps[c.idx], true +} + +func (c *boolValueCursor) close() { + c.cur.Close() +} + +type stringValueCursor struct { + cur tsdb.StringArrayCursor + arr *cursors.StringArray + idx int +} + +func (c *stringValueCursor) next() (int64, interface{}) { + // Initialize the array on first call + if c.arr == nil { + c.arr = c.cur.Next() + } + // Indicate no elements early + if c.arr.Len() == 0 || c.idx >= c.arr.Len() { + return 0, nil + } + defer func() { c.idx++ }() + return c.arr.Timestamps[c.idx], c.arr.Values[c.idx] +} + +func (c *stringValueCursor) peek() (int64, bool) { + // Initialize the array on first call + if c.arr == nil { + c.idx = 0 + c.arr = c.cur.Next() + } + // Indicate no elements early + if c.arr.Len() == 0 || c.idx >= c.arr.Len() { + return 0, false + } + return c.arr.Timestamps[c.idx], true +} + +func (c *stringValueCursor) close() { + c.cur.Close() +} diff --git a/cmd/influx_tools/parquet/exporter.go b/cmd/influx_tools/parquet/exporter.go index cf33b2159bc..91f1cf06c27 100644 --- a/cmd/influx_tools/parquet/exporter.go +++ b/cmd/influx_tools/parquet/exporter.go @@ -457,7 +457,7 @@ func (e *exporter) exportMeasurement(ctx context.Context, shard *tsdb.Shard, mea record := e.convertData(rows, builder, creator.tags, creator.fieldKeys) // Write data - if err := writer.WriteBuffered(record); err != nil { + if err := writer.Write(record); err != nil { return fmt.Errorf("writing parquet file %q failed: %w", filename, err) } e.logger.Infof(" exported %d rows in %v", len(rows), time.Since(last)) From a7d0f1b89a808c4b5ab54410a06c1f505333d091 Mon Sep 17 00:00:00 2001 From: Sven Rebhan Date: Thu, 19 Sep 2024 15:57:28 +0200 Subject: [PATCH 14/14] feat: Preallocate maps and slices --- cmd/influx_tools/parquet/batcher.go | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/cmd/influx_tools/parquet/batcher.go b/cmd/influx_tools/parquet/batcher.go index e91f300fb7a..dbf452e1077 100644 --- a/cmd/influx_tools/parquet/batcher.go +++ b/cmd/influx_tools/parquet/batcher.go @@ -68,10 +68,11 @@ func (b *batcher) next(ctx context.Context) ([]row, error) { return nil, fmt.Errorf("getting cursor iterator for %q failed: %w", string(b.measurement), err) } - data := make(map[string]map[int64]row) + data := make(map[string]map[int64]row, len(b.series)) end := models.MaxNanoTime + var rowCount int for _, s := range b.series { - data[s.key] = make(map[int64]row) + data[s.key] = make(map[int64]row, tsdb.DefaultMaxPointsPerBlock) tags := make(map[string]string, len(s.tags)) for _, t := range s.tags { tags[string(t.Key)] = string(t.Value) @@ -130,6 +131,7 @@ func (b *batcher) next(ctx context.Context) ([]row, error) { tags: tags, fields: make(map[string]interface{}), } + rowCount++ } data[s.key][timestamp].fields[fname] = v @@ -145,7 +147,7 @@ func (b *batcher) next(ctx context.Context) ([]row, error) { } // Extract the rows ordered by timestamp - var rows []row + rows := make([]row, 0, rowCount) for _, tmap := range data { for _, r := range tmap { rows = append(rows, r)