Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Improve protobuf performance #103

Closed
wants to merge 2 commits into from
Closed

Improve protobuf performance #103

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
ccc22b1
Use `MarshalVT`/`UnmarshalVT` instead of `proto.Marshal`/`proto.Unmar…
arthurschreiber Mar 7, 2023
80dffac
Fix last remaining proto use.
arthurschreiber Apr 11, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion go/vt/tableacl/tableacl.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -113,7 +113,7 @@ func (tacl *tableACL) init(configFile string, aclCB func()) error {
return err
}
config := &tableaclpb.Config{}
if err := proto.Unmarshal(data, config); err != nil {
if err := config.UnmarshalVT(data); err != nil {
// try to parse tableacl as json file
if jsonErr := json2.Unmarshal(data, config); jsonErr != nil {
log.Infof("unable to parse tableACL config file as a protobuf or json file. protobuf err: %v json err: %v", err, jsonErr)
Expand Down
9 changes: 4 additions & 5 deletions go/vt/topo/cell_info.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,6 @@ import (
"path"
"strings"

"google.golang.org/protobuf/proto"
"k8s.io/apimachinery/pkg/util/sets"

"vitess.io/vitess/go/vt/vterrors"
Expand Down Expand Up @@ -74,7 +73,7 @@ func (ts *Server) GetCellInfo(ctx context.Context, cell string, strongRead bool)

// Unpack the contents.
ci := &topodatapb.CellInfo{}
if err := proto.Unmarshal(contents, ci); err != nil {
if err := ci.UnmarshalVT(contents); err != nil {
return nil, err
}
return ci, nil
Expand All @@ -83,7 +82,7 @@ func (ts *Server) GetCellInfo(ctx context.Context, cell string, strongRead bool)
// CreateCellInfo creates a new CellInfo with the provided content.
func (ts *Server) CreateCellInfo(ctx context.Context, cell string, ci *topodatapb.CellInfo) error {
// Pack the content.
contents, err := proto.Marshal(ci)
contents, err := ci.MarshalVT()
if err != nil {
return err
}
Expand All @@ -108,7 +107,7 @@ func (ts *Server) UpdateCellInfoFields(ctx context.Context, cell string, update
contents, version, err := ts.globalCell.Get(ctx, filePath)
switch {
case err == nil:
if err := proto.Unmarshal(contents, ci); err != nil {
if err := ci.UnmarshalVT(contents); err != nil {
return err
}
case IsErrType(err, NoNode):
Expand All @@ -126,7 +125,7 @@ func (ts *Server) UpdateCellInfoFields(ctx context.Context, cell string, update
}

// Pack and save.
contents, err = proto.Marshal(ci)
contents, err = ci.MarshalVT()
if err != nil {
return err
}
Expand Down
12 changes: 5 additions & 7 deletions go/vt/topo/cells_aliases.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,8 +20,6 @@ import (
"fmt"
"path"

"google.golang.org/protobuf/proto"

"context"

topodatapb "vitess.io/vitess/go/vt/proto/topodata"
Expand Down Expand Up @@ -63,7 +61,7 @@ func (ts *Server) GetCellsAliases(ctx context.Context, strongRead bool) (ret map

// Unpack the contents.
cellsAlias := &topodatapb.CellsAlias{}
if err := proto.Unmarshal(contents, cellsAlias); err != nil {
if err := cellsAlias.UnmarshalVT(contents); err != nil {
return nil, err
}

Expand All @@ -90,7 +88,7 @@ func (ts *Server) GetCellsAlias(ctx context.Context, name string, strongRead boo

// Unpack the contents.
cellsAlias := &topodatapb.CellsAlias{}
if err := proto.Unmarshal(contents, cellsAlias); err != nil {
if err := cellsAlias.UnmarshalVT(contents); err != nil {
return nil, err
}

Expand Down Expand Up @@ -119,7 +117,7 @@ func (ts *Server) CreateCellsAlias(ctx context.Context, alias string, cellsAlias
ts.clearCellAliasesCache()

// Pack the content.
contents, err := proto.Marshal(cellsAlias)
contents, err := cellsAlias.MarshalVT()
if err != nil {
return err
}
Expand All @@ -142,7 +140,7 @@ func (ts *Server) UpdateCellsAlias(ctx context.Context, alias string, update fun
contents, version, err := ts.globalCell.Get(ctx, filePath)
switch {
case err == nil:
if err := proto.Unmarshal(contents, cellsAlias); err != nil {
if err := cellsAlias.UnmarshalVT(contents); err != nil {
return err
}
case IsErrType(err, NoNode):
Expand All @@ -169,7 +167,7 @@ func (ts *Server) UpdateCellsAlias(ctx context.Context, alias string, update fun
}

// Pack and save.
contents, err = proto.Marshal(cellsAlias)
contents, err = cellsAlias.MarshalVT()
if err != nil {
return err
}
Expand Down
8 changes: 3 additions & 5 deletions go/vt/topo/external_vitess_cluster.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,8 +20,6 @@ import (
"context"
"path"

"google.golang.org/protobuf/proto"

"vitess.io/vitess/go/event"
topodatapb "vitess.io/vitess/go/vt/proto/topodata"
"vitess.io/vitess/go/vt/topo/events"
Expand Down Expand Up @@ -49,7 +47,7 @@ func GetExternalVitessClusterPath(clusterName string) string {

// CreateExternalVitessCluster creates a topo record for the passed vitess cluster
func (ts *Server) CreateExternalVitessCluster(ctx context.Context, clusterName string, value *topodatapb.ExternalVitessCluster) error {
data, err := proto.Marshal(value)
data, err := value.MarshalVT()
if err != nil {
return err
}
Expand Down Expand Up @@ -77,7 +75,7 @@ func (ts *Server) GetExternalVitessCluster(ctx context.Context, clusterName stri
return nil, err
}
vc := &topodatapb.ExternalVitessCluster{}
if err = proto.Unmarshal(data, vc); err != nil {
if err = vc.UnmarshalVT(data); err != nil {
return nil, vterrors.Wrap(err, "bad vitess cluster data")
}

Expand All @@ -91,7 +89,7 @@ func (ts *Server) GetExternalVitessCluster(ctx context.Context, clusterName stri
// UpdateExternalVitessCluster updates the topo record for the named vitess cluster
func (ts *Server) UpdateExternalVitessCluster(ctx context.Context, vc *ExternalVitessClusterInfo) error {
//FIXME: check for cluster lock
data, err := proto.Marshal(vc.ExternalVitessCluster)
data, err := vc.ExternalVitessCluster.MarshalVT()
if err != nil {
return err
}
Expand Down
8 changes: 3 additions & 5 deletions go/vt/topo/keyspace.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,8 +20,6 @@ import (
"path"
"strings"

"google.golang.org/protobuf/proto"

"context"

"vitess.io/vitess/go/vt/vterrors"
Expand Down Expand Up @@ -180,7 +178,7 @@ func (ts *Server) CreateKeyspace(ctx context.Context, keyspace string, value *to
return vterrors.Wrapf(err, "CreateKeyspace: %s", err)
}

data, err := proto.Marshal(value)
data, err := value.MarshalVT()
if err != nil {
return err
}
Expand Down Expand Up @@ -211,7 +209,7 @@ func (ts *Server) GetKeyspace(ctx context.Context, keyspace string) (*KeyspaceIn
}

k := &topodatapb.Keyspace{}
if err = proto.Unmarshal(data, k); err != nil {
if err = k.UnmarshalVT(data); err != nil {
return nil, vterrors.Wrap(err, "bad keyspace data")
}

Expand Down Expand Up @@ -251,7 +249,7 @@ func (ts *Server) UpdateKeyspace(ctx context.Context, ki *KeyspaceInfo) error {
return err
}

data, err := proto.Marshal(ki.Keyspace)
data, err := ki.Keyspace.MarshalVT()
if err != nil {
return err
}
Expand Down
6 changes: 3 additions & 3 deletions go/vt/topo/replication.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -186,7 +186,7 @@ func (ts *Server) UpdateShardReplicationFields(ctx context.Context, cell, keyspa
// Empty node, version is nil
case err == nil:
// Use any data we got.
if err = proto.Unmarshal(data, sr); err != nil {
if err = sr.UnmarshalVT(data); err != nil {
return vterrors.Wrap(err, "bad ShardReplication data")
}
default:
Expand All @@ -204,7 +204,7 @@ func (ts *Server) UpdateShardReplicationFields(ctx context.Context, cell, keyspa
}

// marshall and save
data, err = proto.Marshal(sr)
data, err = sr.MarshalVT()
if err != nil {
return err
}
Expand Down Expand Up @@ -243,7 +243,7 @@ func (ts *Server) GetShardReplication(ctx context.Context, cell, keyspace, shard
}

sr := &topodatapb.ShardReplication{}
if err = proto.Unmarshal(data, sr); err != nil {
if err = sr.UnmarshalVT(data); err != nil {
return nil, vterrors.Wrap(err, "bad ShardReplication data")
}

Expand Down
10 changes: 5 additions & 5 deletions go/vt/topo/shard.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -213,7 +213,7 @@ func (ts *Server) GetShard(ctx context.Context, keyspace, shard string) (*ShardI
}

value := &topodatapb.Shard{}
if err = proto.Unmarshal(data, value); err != nil {
if err = value.UnmarshalVT(data); err != nil {
return nil, vterrors.Wrapf(err, "GetShard(%v,%v): bad shard data", keyspace, shard)
}
return &ShardInfo{
Expand All @@ -232,7 +232,7 @@ func (ts *Server) updateShard(ctx context.Context, si *ShardInfo) error {
span.Annotate("shard", si.shardName)
defer span.Finish()

data, err := proto.Marshal(si.Shard)
data, err := si.Shard.MarshalVT()
if err != nil {
return err
}
Expand Down Expand Up @@ -315,7 +315,7 @@ func (ts *Server) CreateShard(ctx context.Context, keyspace, shard string) (err
}

// Marshal and save.
data, err := proto.Marshal(value)
data, err := value.MarshalVT()
if err != nil {
return err
}
Expand Down Expand Up @@ -670,7 +670,7 @@ func (ts *Server) WatchShard(ctx context.Context, keyspace, shard string) (*Watc
return nil, nil, err
}
value := &topodatapb.Shard{}
if err := proto.Unmarshal(current.Contents, value); err != nil {
if err := value.UnmarshalVT(current.Contents); err != nil {
// Cancel the watch, drain channel.
cancel()
for range wdChannel {
Expand Down Expand Up @@ -698,7 +698,7 @@ func (ts *Server) WatchShard(ctx context.Context, keyspace, shard string) (*Watc
}

value := &topodatapb.Shard{}
if err := proto.Unmarshal(wd.Contents, value); err != nil {
if err := value.UnmarshalVT(wd.Contents); err != nil {
cancel()
for range wdChannel {
}
Expand Down
10 changes: 4 additions & 6 deletions go/vt/topo/srv_keyspace.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,8 +23,6 @@ import (
"path"
"sync"

"google.golang.org/protobuf/proto"

"vitess.io/vitess/go/vt/vterrors"

"vitess.io/vitess/go/vt/concurrency"
Expand Down Expand Up @@ -64,7 +62,7 @@ func (ts *Server) WatchSrvKeyspace(ctx context.Context, cell, keyspace string) (
return nil, nil, err
}
value := &topodatapb.SrvKeyspace{}
if err := proto.Unmarshal(current.Contents, value); err != nil {
if err := value.UnmarshalVT(current.Contents); err != nil {
// Cancel the watch, drain channel.
cancel()
for range wdChannel {
Expand Down Expand Up @@ -93,7 +91,7 @@ func (ts *Server) WatchSrvKeyspace(ctx context.Context, cell, keyspace string) (
}

value := &topodatapb.SrvKeyspace{}
if err := proto.Unmarshal(wd.Contents, value); err != nil {
if err := value.UnmarshalVT(wd.Contents); err != nil {
cancel()
for range wdChannel {
}
Expand Down Expand Up @@ -637,7 +635,7 @@ func (ts *Server) UpdateSrvKeyspace(ctx context.Context, cell, keyspace string,
}

nodePath := srvKeyspaceFileName(keyspace)
data, err := proto.Marshal(srvKeyspace)
data, err := srvKeyspace.MarshalVT()
if err != nil {
return err
}
Expand Down Expand Up @@ -691,7 +689,7 @@ func (ts *Server) GetSrvKeyspace(ctx context.Context, cell, keyspace string) (*t
return nil, err
}
srvKeyspace := &topodatapb.SrvKeyspace{}
if err := proto.Unmarshal(data, srvKeyspace); err != nil {
if err := srvKeyspace.UnmarshalVT(data); err != nil {
return nil, vterrors.Wrapf(err, "SrvKeyspace unmarshal failed: %v", data)
}
return srvKeyspace, nil
Expand Down
10 changes: 4 additions & 6 deletions go/vt/topo/srv_vschema.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,8 +21,6 @@ import (
"fmt"
"sync"

"google.golang.org/protobuf/proto"

"vitess.io/vitess/go/vt/log"
"vitess.io/vitess/go/vt/vterrors"

Expand Down Expand Up @@ -54,7 +52,7 @@ func (ts *Server) WatchSrvVSchema(ctx context.Context, cell string) (*WatchSrvVS
return nil, nil, err
}
value := &vschemapb.SrvVSchema{}
if err := proto.Unmarshal(current.Contents, value); err != nil {
if err := value.UnmarshalVT(current.Contents); err != nil {
// Cancel the watch, drain channel.
cancel()
for range wdChannel {
Expand Down Expand Up @@ -83,7 +81,7 @@ func (ts *Server) WatchSrvVSchema(ctx context.Context, cell string) (*WatchSrvVS
}

value := &vschemapb.SrvVSchema{}
if err := proto.Unmarshal(wd.Contents, value); err != nil {
if err := value.UnmarshalVT(wd.Contents); err != nil {
cancel()
for range wdChannel {
}
Expand All @@ -105,7 +103,7 @@ func (ts *Server) UpdateSrvVSchema(ctx context.Context, cell string, srvVSchema
}

nodePath := SrvVSchemaFile
data, err := proto.Marshal(srvVSchema)
data, err := srvVSchema.MarshalVT()
if err != nil {
return err
}
Expand All @@ -126,7 +124,7 @@ func (ts *Server) GetSrvVSchema(ctx context.Context, cell string) (*vschemapb.Sr
return nil, err
}
srvVSchema := &vschemapb.SrvVSchema{}
if err := proto.Unmarshal(data, srvVSchema); err != nil {
if err := srvVSchema.UnmarshalVT(data); err != nil {
return nil, vterrors.Wrapf(err, "SrvVSchema unmarshal failed: %v", data)
}
return srvVSchema, nil
Expand Down
8 changes: 4 additions & 4 deletions go/vt/topo/tablet.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -245,7 +245,7 @@ func (ts *Server) GetTablet(ctx context.Context, alias *topodatapb.TabletAlias)
return nil, err
}
tablet := &topodatapb.Tablet{}
if err := proto.Unmarshal(data, tablet); err != nil {
if err := tablet.UnmarshalVT(data); err != nil {
return nil, err
}

Expand Down Expand Up @@ -310,7 +310,7 @@ func (ts *Server) GetTabletsByCell(ctx context.Context, cellAlias string) ([]*Ta
tablets := make([]*TabletInfo, len(listResults))
for n := range listResults {
tablet := &topodatapb.Tablet{}
if err := proto.Unmarshal(listResults[n].Value, tablet); err != nil {
if err := tablet.UnmarshalVT(listResults[n].Value); err != nil {
return nil, err
}
tablets[n] = &TabletInfo{Tablet: tablet, version: listResults[n].Version}
Expand Down Expand Up @@ -363,7 +363,7 @@ func (ts *Server) UpdateTablet(ctx context.Context, ti *TabletInfo) error {
span.Annotate("tablet", topoproto.TabletAliasString(ti.Alias))
defer span.Finish()

data, err := proto.Marshal(ti.Tablet)
data, err := ti.Tablet.MarshalVT()
if err != nil {
return err
}
Expand Down Expand Up @@ -441,7 +441,7 @@ func (ts *Server) CreateTablet(ctx context.Context, tablet *topodatapb.Tablet) e
return err
}

data, err := proto.Marshal(tablet)
data, err := tablet.MarshalVT()
if err != nil {
return err
}
Expand Down
Loading
Loading