Skip to content

Commit

Permalink
Ygot diff tool (sonic-net#103)
Browse files Browse the repository at this point in the history 
* Ygot diff tool

Standard ygot.Diff API flattens both structs into {path, value} pair
list and compares them to resolve new, modified and deleted values.
This approach does not scale well for large ygot trees. Following are
the go benchmarks for ygot.Diff API with openconfig-acl ygot trees.
Size "MxN" stands for ygot tree with M ACLs and N rules in each ACL.

+-------------------+---------------+----------------+
|   Test case       |  size=10x10   |  size=10x100   |
+-------------------+---------------+----------------+
| No changes        | 9024089697ns  | 854023452896ns |
| One attr changed  | 9285911808ns  | 857292949475ns |
| All attrs changed | 9333680743ns  | 855581319202ns |
| Create all attrs  |   80830018ns  |    858752644ns |
| Delete all attrs  |   83082154ns  |    755762682ns |
+-------------------+---------------+----------------+

Implemented a custom ygot diff tool to compare the ygot struct fields
inline. Uses reflection to traverse and compare structs.
Function signature:

func Diff(s1, s2 ygot.GoStruct, opt DiffOptions) (DiffResults, error)

type DiffOptions struct {
    RecordAll bool // include all s2 attrs, even if not modified
}

type DiffResults struct {
    Update []*gnmi.Update
    Delete []*gnmi.Path
}

Diff function will be used for generating notification messages for
SMPLE subscriptions -- by comparing ygot objects from the previous
iteration with the current objects. By default, Diff returns delete
paths and update values for new/modified fields only. Suitable for
SAMPLE subscription processing when suppress_redundant=true. The
suppress_redundant=false case requires all set attributes of the ygot
struct as update values (even if they are not modified). This can be
achieved by passing RecordAll=true in the DiffOptions.

Go benchmark for the new Diff API with openconfig-acl model:
+-------------------+------------+-------------+---------------+
|   Test case       | size=10x10 | size=10x100 | size=100x1000 |
+-------------------+---------------+--------------------------+
| No changes        | 1784778ns  | 15460520ns  | 1757874254ns  |
| One attr changed  | 1547512ns  | 16506496ns  | 1786245108ns  |
| All attrs changed | 3562498ns  | 34306626ns  | 3846524858ns  |
| Create all attrs  | 7101530ns  | 74663888ns  | 8245685970ns  |
| Delete all attrs  |  120232ns  |   125293ns  |     453306ns  |
+-------------------+------------+-------------+---------------+

Signed-off-by: Sachin Holla <[email protected]>

* Add transl_utils gotests to pipeline

* Removed unnecessary empty name check

---------

Signed-off-by: Sachin Holla <[email protected]>
  • Loading branch information
@sachinholla
sachinholla authored May 22, 2023
1 parent 2971f64 commit a9baca0
Show file tree
Hide file tree
Showing 4 changed files with 1,105 additions and 0 deletions.
1 change: 1 addition & 0 deletions Makefile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -92,6 +92,7 @@ check_gotest:
sudo CGO_LDFLAGS="$(CGO_LDFLAGS)" CGO_CXXFLAGS="$(CGO_CXXFLAGS)" $(GO) test -coverprofile=coverage-dialcout.txt -covermode=atomic -mod=vendor $(BLD_FLAGS) -v github.com/sonic-net/sonic-gnmi/dialout/dialout_client
sudo CGO_LDFLAGS="$(CGO_LDFLAGS)" CGO_CXXFLAGS="$(CGO_CXXFLAGS)" $(GO) test -race -coverprofile=coverage-data.txt -covermode=atomic -mod=vendor -v github.com/sonic-net/sonic-gnmi/sonic_data_client
sudo CGO_LDFLAGS="$(CGO_LDFLAGS)" CGO_CXXFLAGS="$(CGO_CXXFLAGS)" $(GO) test -race -coverprofile=coverage-dbus.txt -covermode=atomic -mod=vendor -v github.com/sonic-net/sonic-gnmi/sonic_service_client
sudo CGO_LDFLAGS="$(CGO_LDFLAGS)" CGO_CXXFLAGS="$(CGO_CXXFLAGS)" $(GO) test -coverprofile=coverage-translutils.txt -covermode=atomic -mod=vendor -v github.com/sonic-net/sonic-gnmi/transl_utils
$(GO) get github.com/axw/gocov/...
$(GO) get github.com/AlekSi/gocov-xml
$(GO) mod vendor
Expand Down
333 changes: 333 additions & 0 deletions transl_utils/ygot_diff.go
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,333 @@
////////////////////////////////////////////////////////////////////////////////
// //
// Copyright 2021 Broadcom. The term Broadcom refers to Broadcom Inc. and/or //
// its subsidiaries. //
// //
// Licensed under the Apache License, Version 2.0 (the "License"); //
// you may not use this file except in compliance with the License. //
// You may obtain a copy of the License at //
// //
// http://www.apache.org/licenses/LICENSE-2.0 //
// //
// Unless required by applicable law or agreed to in writing, software //
// distributed under the License is distributed on an "AS IS" BASIS, //
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. //
// See the License for the specific language governing permissions and //
// limitations under the License. //
// //
////////////////////////////////////////////////////////////////////////////////

package transl_utils

import (
"fmt"
"reflect"

"github.com/Azure/sonic-mgmt-common/translib/ocbinds"
"github.com/openconfig/gnmi/errlist"
"github.com/openconfig/gnmi/proto/gnmi"
"github.com/openconfig/ygot/ygot"
)

// DiffOptions holds the optional parameters foi Diff API.
type DiffOptions struct {
// RecordAll indicates if all attributes of modified GoStruct
// should be recorded - even if the values are same.
RecordAll bool
}

// DiffResults holds updated {path, value} pairs and deleted paths
// resolved by the Diff API.
type DiffResults struct {
Update []*gnmi.Update
Delete []*gnmi.Path
}

// Diff compares original and modified ygot structs; returns
// the changes as a DiffResults.
// Works with translib generated ygot structs only.
func Diff(original, modified ygot.GoStruct, opts DiffOptions) (DiffResults, error) {
var yd ygotDiff
yd.DiffOptions = opts
yd.errors.Separator = "\n"
if original != nil && reflect.TypeOf(original) != reflect.TypeOf(modified) {
return yd.DiffResults, fmt.Errorf("Type mismatch")
}

yd.forStruct(reflect.ValueOf(original), reflect.ValueOf(modified))
return yd.DiffResults, yd.errors.Err()
}

// ygotDiff is a utility to compare ygot structs.
type ygotDiff struct {
DiffOptions
DiffResults
prefix []*gnmi.PathElem // current prefix
errors errlist.List
}

func (yd *ygotDiff) forStruct(v1, v2 reflect.Value) {
if !v1.IsValid() || v1.IsNil() {
if v2.IsValid() && !v2.IsNil() {
yd.recordUpdates(v2.Interface())
}
return
}
if !v2.IsValid() || v2.IsNil() {
yd.recordDeletePath(nil)
return
}

v1 = v1.Elem()
v2 = v2.Elem()
st := v1.Type()

for i := v1.NumField() - 1; i >= 0; i-- {
f1 := v1.Field(i)
f2 := v2.Field(i)
ft := st.Field(i)

switch ft.Type.Kind() {
case reflect.Ptr: // leaf or container
if ft.Type.Elem().Kind() == reflect.Struct {
yd.pushPrefix(getElemName(&ft), nil)
yd.forStruct(f1, f2)
yd.popPrefix()
} else {
yd.forLeaf(&ft, unboxPtr(f1), unboxPtr(f2))
}
case reflect.Map: // list
yd.forList(&ft, &f1, &f2)
case reflect.Slice: // leaf-list
yd.forLeafList(&ft, &f1, &f2)
case reflect.Int64: // enum
yd.forEnum(&ft, f1.Int(), f2.Int())
case reflect.Interface: // union
yd.forUnion(&ft, &f1, &f2)
case reflect.Bool: // empty leaf
yd.forLeaf(&ft, f1.Bool(), f2.Bool())
default:
yd.recordError("Unexpected type %v for field %s", ft.Type.Kind(), ft.Name)
}
}
}

// forList compares ygot lists. Both m1 and m2 are assumed
// to be map values.
func (yd *ygotDiff) forList(f *reflect.StructField, m1, m2 *reflect.Value) {
listName := getElemName(f)
for iter := m1.MapRange(); iter.Next(); {
v1 := iter.Value()
if v2 := m2.MapIndex(iter.Key()); v2.IsValid() {
// Instance present in both m1 and m2. Compare them
yd.pushPrefix(listName, &v2)
yd.forStruct(v1, v2)
yd.popPrefix()
} else {
// Instance deleted in m2
suffix := newPathElem(listName, &v1)
yd.recordDeletePath(suffix)
}
}
// Look for new instances in m2 (that are not in m1)
for iter := m2.MapRange(); iter.Next(); {
if v1 := m1.MapIndex(iter.Key()); !v1.IsValid() {
v2 := iter.Value()
yd.pushPrefix(listName, &v2)
yd.recordUpdates(v2.Interface())
yd.popPrefix()
}
}
}

func (yd *ygotDiff) forUnion(f *reflect.StructField, v1, v2 *reflect.Value) {
if v2.IsNil() {
if !v1.IsNil() {
yd.recordDelete(f)
}
return
}
if yd.RecordAll || v1.IsNil() {
yd.recordUpdate(f, v2.Interface())
return
}

// Union is modeled as an interface wrapping a struct pointer
u1 := v1.Elem().Elem().Interface()
u2 := v2.Elem().Elem().Interface()
if u1 != u2 {
yd.recordUpdate(f, v2.Interface())
}
}

func (yd *ygotDiff) forEnum(f *reflect.StructField, v1, v2 int64) {
if v2 == 0 {
if v1 != 0 {
yd.recordDelete(f)
}
return
}
if !yd.RecordAll && v1 == v2 {
return
}

//TODO avoid directly referring to ocbinds
enumDef, ok := ocbinds.ΛEnum[f.Type.Name()][v2]
if !ok {
yd.recordError("%s is not a GoEnum", f.Type.Name())
} else {
yd.recordUpdate(f, enumDef.Name)
}
}

func (yd *ygotDiff) forLeaf(f *reflect.StructField, v1, v2 interface{}) {
if v2 == nil {
if v1 != nil {
yd.recordDelete(f)
}
} else if yd.RecordAll || v1 != v2 {
yd.recordUpdate(f, v2)
}
}

func (yd *ygotDiff) forLeafList(f *reflect.StructField, v1, v2 *reflect.Value) {
var len1, len2 int
if !v1.IsNil() {
len1 = v1.Len()
}
if !v2.IsNil() {
len2 = v2.Len()
}
if len2 == 0 {
if len1 != 0 {
yd.recordDelete(f)
}
return
}
if yd.RecordAll || len1 != len2 {
yd.recordUpdate(f, v2.Interface())
return
}
for i := 0; i < len1; i++ {
if v1.Index(i).Interface() != v2.Index(i).Interface() {
yd.recordUpdate(f, v2.Interface())
return
}
}
}

// recordUpdates records an update for of the set fields of a GoStruct.
// Object v should be a GoStruct.
func (yd *ygotDiff) recordUpdates(v interface{}) {
s, ok := v.(ygot.GoStruct)
if !ok {
yd.recordError("%T is not a ygot.GoStruct", v)
return
}

msg, err := ygot.TogNMINotifications(s, 0,
ygot.GNMINotificationsConfig{UsePathElem: true})
if err != nil {
yd.recordError("TogNMINotifications failed; %v", err)
return
}

for _, m := range msg {
for _, u := range m.Update {
elems := make([]*gnmi.PathElem, 0, len(yd.prefix)+len(u.Path.Elem))
elems = append(elems, yd.prefix...)
u.Path.Elem = append(elems, u.Path.Elem...)
}
yd.Update = append(yd.Update, m.Update...)
}
}

// recordUpdate records an update for the GoStruct field f, value v.
// Uses ygot.EncodeTypedValue API to construct a gnmi.TypedValue from
// the value v.
func (yd *ygotDiff) recordUpdate(f *reflect.StructField, v interface{}) {
tv, err := ygot.EncodeTypedValue(v, gnmi.Encoding_JSON)
if err != nil {
yd.recordError("EncodeTypedValue failed; %v", err)
return
}

suffix := newPathElem(getElemName(f), nil)
u := &gnmi.Update{
Path: yd.newPath(suffix),
Val: tv,
}
yd.Update = append(yd.Update, u)
}

func (yd *ygotDiff) recordDelete(f *reflect.StructField) {
suffix := newPathElem(getElemName(f), nil)
yd.recordDeletePath(suffix)
}

func (yd *ygotDiff) recordDeletePath(suffix *gnmi.PathElem) {
yd.Delete = append(yd.Delete, yd.newPath(suffix))
}

func (yd *ygotDiff) recordError(f string, args ...interface{}) {
p, _ := ygot.PathToString(&gnmi.Path{Elem: yd.prefix})
yd.errors.Add(fmt.Errorf(p+": "+f, args...))
}

// newPath returns a new *gnmi.Path by joining the path elements
// yd.prefix and an optional suffix.
func (yd *ygotDiff) newPath(suffix *gnmi.PathElem) *gnmi.Path {
n := len(yd.prefix)
if suffix != nil {
n++
}

elems := make([]*gnmi.PathElem, n)
copy(elems, yd.prefix)
if suffix != nil {
elems[len(yd.prefix)] = suffix
}

return &gnmi.Path{Elem: elems}
}

// pushPrefix appends a new path element to the prefix.
func (yd *ygotDiff) pushPrefix(name string, keyObj *reflect.Value) {
yd.prefix = append(yd.prefix, newPathElem(name, keyObj))
}

// popPrefix removes last path element from the prefix.
func (yd *ygotDiff) popPrefix() {
yd.prefix = yd.prefix[:len(yd.prefix)-1]
}

// getElemName returns the path element name for a ygot struct field f.
// It panic if there is no 'path' tag, which should never happen.
func getElemName(f *reflect.StructField) string {
name, ok := f.Tag.Lookup("path")
if !ok {
panic(f.Name + " has no path tag")
}
return name
}

// newPathElem creates a new gnmi.PathElem object for the given
// node name and an optional key object.
func newPathElem(name string, keyObj *reflect.Value) *gnmi.PathElem {
pElem := &gnmi.PathElem{Name: name}
if keyObj != nil {
var err error
pElem.Key, err = ygot.PathKeyFromStruct(*keyObj)
if err != nil {
panic("ygot.PathKeyFromStruct failed: " + err.Error())
}
}
return pElem
}

func unboxPtr(v reflect.Value) interface{} {
if v.IsNil() {
return nil
}
return v.Elem().Interface()
}
Loading

0 comments on commit a9baca0

Please sign in to comment.