diff --git a/azure-pipelines.yml b/azure-pipelines.yml index dd9434102b..6b2da4531d 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -52,6 +52,8 @@ stages: artifact: sonic-buildimage.vs runVersion: 'latestFromBranch' runBranch: 'refs/heads/master' + patterns: | + target/debs/buster/libyang*.deb displayName: "Download sonic buildimage" - script: | diff --git a/rest/server/handler.go b/rest/server/handler.go index d318cbdacc..584a09dc1b 100644 --- a/rest/server/handler.go +++ b/rest/server/handler.go @@ -260,6 +260,8 @@ type translibArgs struct { data []byte // payload version translib.Version // client version depth uint // RESTCONF depth, for Get API only + content string // RESTCONF content, for Get API only + fields []string // RESTCONF fields, for Get API only deleteEmpty bool // Delete empty entry during field delete } @@ -304,8 +306,12 @@ func invokeTranslib(args *translibArgs, rc *RequestContext) (int, []byte, error) case "GET", "HEAD": req := translib.GetRequest{ Path: args.path, - Depth: args.depth, ClientVersion: args.version, + QueryParams: translib.QueryParameters{ + Depth: args.depth, + Content: args.content, + Fields: args.fields, + }, } resp, err1 := translib.Get(req) if err1 == nil { diff --git a/rest/server/handler_test.go b/rest/server/handler_test.go index c29c150cbb..f13a111b38 100644 --- a/rest/server/handler_test.go +++ b/rest/server/handler_test.go @@ -561,10 +561,10 @@ func verifyParseVersion(t *testing.T, r *http.Request, expSuccess bool, expVer t func TestPanic(t *testing.T) { s := newEmptyRouter() - s.addRoute("panic", "GET", "/panic", + s.addRoute("panic", "GET", "/restconf/panic", func(w http.ResponseWriter, r *http.Request) { panic("testing 123") }) w := httptest.NewRecorder() - s.ServeHTTP(w, prepareRequest(t, "GET", "/panic", "")) + s.ServeHTTP(w, prepareRequest(t, "GET", "/restconf/panic", "")) verifyResponse(t, w, 500) } @@ -574,6 +574,60 @@ func TestProcessGET(t *testing.T) { verifyResponseData(t, w, 200, jsonObj{"path": "/api-tests:sample", "depth": 0}) } +func TestProcessGET_query_depth(t *testing.T) { + w := httptest.NewRecorder() + Process(w, prepareRequest(t, "GET", "/api-tests:sample?depth=10", "")) + if restconfCapabilities.depth { + verifyResponseData(t, w, 200, jsonObj{"depth": 10}) + } else { + verifyResponse(t, w, 400) + } +} + +func TestProcessGET_query_depth_error(t *testing.T) { + w := httptest.NewRecorder() + Process(w, prepareRequest(t, "GET", "/api-tests:sample?depth=none", "")) + verifyResponse(t, w, 400) +} + +func TestProcessGET_query_depth_content_field_capability_support(t *testing.T) { + if !restconfCapabilities.depth || !restconfCapabilities.content || !restconfCapabilities.fields { + t.Fatalf("depth/content/fields capability is expected to be supported in rest-server") + } +} + +func TestProcessGET_query_content(t *testing.T) { + w := httptest.NewRecorder() + Process(w, prepareRequest(t, "GET", "/api-tests:sample?content=all", "")) + if restconfCapabilities.content { + verifyResponseData(t, w, 200, jsonObj{"content": "all"}) + } else { + verifyResponse(t, w, 400) + } +} + +func TestProcessGET_query_content_error(t *testing.T) { + w := httptest.NewRecorder() + Process(w, prepareRequest(t, "GET", "/api-tests:sample?content=getall", "")) + verifyResponse(t, w, 400) +} + +func TestProcessGET_query_fields(t *testing.T) { + w := httptest.NewRecorder() + Process(w, prepareRequest(t, "GET", "/api-tests:sample?fields=home/name", "")) + if restconfCapabilities.fields { + verifyResponseData(t, w, 200, jsonObj{"fields": "[home/name]"}) + } else { + verifyResponse(t, w, 400) + } +} + +func TestProcessGET_query_fields_error(t *testing.T) { + w := httptest.NewRecorder() + Process(w, prepareRequest(t, "GET", "/api-tests:sample?fields=home&content=all", "")) + verifyResponse(t, w, 400) +} + func TestProcessGET_error(t *testing.T) { w := httptest.NewRecorder() Process(w, prepareRequest(t, "GET", "/api-tests:sample/error/not-found", "")) @@ -690,6 +744,10 @@ func TestProcessReadError(t *testing.T) { } func prepareRequest(t *testing.T, method, path, data string) *http.Request { + if !strings.Contains(path, "/restconf/") { + path = "/restconf/data" + path + } + r := httptest.NewRequest(method, path, strings.NewReader(data)) rc, r := GetContext(r) rc.ID = t.Name() diff --git a/rest/server/query.go b/rest/server/query.go index 12bc40d5e3..cad8d51eda 100644 --- a/rest/server/query.go +++ b/rest/server/query.go @@ -30,6 +30,9 @@ import ( // parseQueryParams parses the http request's query parameters // into a translibArgs args. func (args *translibArgs) parseQueryParams(r *http.Request) error { + if r.URL.RawQuery == "" { + return nil + } if strings.Contains(r.URL.Path, restconfDataPathPrefix) { return args.parseRestconfQueryParams(r) } @@ -42,12 +45,16 @@ func (args *translibArgs) parseQueryParams(r *http.Request) error { // if any parameter is unsupported or has invalid value. func (args *translibArgs) parseRestconfQueryParams(r *http.Request) error { var err error - qParams := r.URL.Query() + qParams := extractQuery(r.URL.RawQuery) for name, vals := range qParams { switch name { case "depth": args.depth, err = parseDepthParam(vals, r) + case "content": + args.content, err = parseContentParam(vals, r) + case "fields": + args.fields, err = parseFieldsParam(vals, r) case "deleteEmptyEntry": args.deleteEmpty, err = parseDeleteEmptyEntryParam(vals, r) default: @@ -57,10 +64,41 @@ func (args *translibArgs) parseRestconfQueryParams(r *http.Request) error { return err } } + if len(args.fields) > 0 { + if len(args.content) > 0 || args.depth > 0 { + return httpError(http.StatusBadRequest, "Fields query parameter is not supported along with other query parameters") + } + } return nil } +func extractQuery(rawQuery string) map[string][]string { + queryParamsMap := make(map[string][]string) + if len(rawQuery) == 0 { + return queryParamsMap + } + // The query parameters are seperated by & + qpList := strings.Split(rawQuery, "&") + for _, each := range qpList { + var valList []string + if strings.Contains(each, "=") { + eqIdx := strings.Index(each, "=") + key := each[:eqIdx] + val := each[eqIdx+1:] + if _, ok := queryParamsMap[key]; ok { + queryParamsMap[key] = append(queryParamsMap[key], val) + } else { + valList = append(valList, val) + queryParamsMap[key] = valList + } + } else { + queryParamsMap[each] = valList + } + } + return queryParamsMap +} + func newUnsupportedParamError(name string, r *http.Request) error { return httpError(http.StatusBadRequest, "query parameter '%s' not supported", name) } @@ -79,7 +117,7 @@ func parseDepthParam(v []string, r *http.Request) (uint, error) { if r.Method != "GET" && r.Method != "HEAD" { glog.V(1).Infof("[%s] 'depth' not supported for %s", getRequestID(r), r.Method) - return 0, newUnsupportedParamError("depth", r) + return 0, newUnsupportedParamError("depth supported only for GET/HEAD requests", r) } if len(v) != 1 { @@ -100,6 +138,95 @@ func parseDepthParam(v []string, r *http.Request) (uint, error) { return uint(d), nil } +// parseContentParam parses query parameter value for "content" parameter. +// See https://tools.ietf.org/html/rfc8040#section-4.8.1 +func parseContentParam(v []string, r *http.Request) (string, error) { + if !restconfCapabilities.content { + glog.V(1).Infof("'content' support disabled") + return "", newUnsupportedParamError("content", r) + } + + if r.Method != "GET" && r.Method != "HEAD" { + glog.V(1).Infof("'content' not supported for %s", r.Method) + return "", newUnsupportedParamError("content", r) + } + + if len(v) != 1 { + glog.V(1).Infof("Expecting only 1 content param; found %d", len(v)) + return "", newInvalidParamError("content", r) + } + + if v[0] == "all" || v[0] == "config" || v[0] == "nonconfig" { + return v[0], nil + } else { + glog.V(1).Infof("Bad content value '%s'", v[0]) + return "", newInvalidParamError("content", r) + } + + return v[0], nil +} + +func extractFields(s string) []string { + prefix := "" + cur := "" + res := make([]string, 0) + for i, c := range s { + if c == '(' { + prefix = cur + cur = "" + } else if c == ')' { + res = append(res, prefix+"/"+cur) + prefix = "" + cur = "" + } else if c == ';' { + fullpath := prefix + if len(prefix) > 0 { + fullpath += "/" + } + if len(fullpath+cur) > 0 { + res = append(res, fullpath+cur) + } + cur = "" + } else if c == ' ' { + continue + } else { + cur += string(c) + } + if i == (len(s) - 1) { + fullpath := prefix + if len(prefix) > 0 { + fullpath += "/" + } + if len(fullpath+cur) > 0 { + res = append(res, fullpath+cur) + } + } + } + return res +} + +// parseFieldsParam parses query parameter value for "fields" parameter. +// See https://tools.ietf.org/html/rfc8040#section-4.8.3 +func parseFieldsParam(v []string, r *http.Request) ([]string, error) { + if !restconfCapabilities.fields { + glog.V(1).Infof("'fields' support disabled") + return v, newUnsupportedParamError("fields", r) + } + + if r.Method != "GET" && r.Method != "HEAD" { + glog.V(1).Infof("'fields' not supported for %s", r.Method) + return v, newUnsupportedParamError("fields supported only for GET/HEAD query", r) + } + + if len(v) != 1 { + glog.V(1).Infof("Expecting atleast 1 fields param; found %d", len(v)) + return v, newInvalidParamError("fields", r) + } + + res := extractFields(v[0]) + return res, nil +} + // parseDeleteEmptyEntryParam parses the custom "deleteEmptyEntry" query parameter. func parseDeleteEmptyEntryParam(v []string, r *http.Request) (bool, error) { if r.Method != "DELETE" { diff --git a/rest/server/query_test.go b/rest/server/query_test.go index 85e64d46bd..a74ea4d030 100644 --- a/rest/server/query_test.go +++ b/rest/server/query_test.go @@ -21,6 +21,7 @@ package server import ( "net/http/httptest" + "reflect" "testing" ) @@ -51,6 +52,12 @@ func testQuery(method, queryStr string, exp *translibArgs) func(*testing.T) { if p.deleteEmpty != exp.deleteEmpty { t.Errorf("'deleteEmptyEntry' mismatch; expting %v, found %v", exp.deleteEmpty, p.deleteEmpty) } + if p.content != exp.content { + t.Errorf("'content' mismatch; expecting %s, found %s", exp.content, p.content) + } + if !reflect.DeepEqual(p.fields, exp.fields) { + t.Errorf("fields mismatch; expecting %s, found %s", exp.fields, p.fields) + } if t.Failed() { t.Errorf("Testcase failed for query '%s'", r.URL.RawQuery) } @@ -62,6 +69,11 @@ func TestQuery(t *testing.T) { t.Run("unknown", testQuery("GET", "one=1", nil)) } +func testGetQuery(t *testing.T, name, queryStr string, exp *translibArgs) { + t.Run("GET/"+name, testQuery("GET", queryStr, exp)) + t.Run("HEAD/"+name, testQuery("HEAD", queryStr, exp)) +} + func TestQuery_depth(t *testing.T) { rcCaps := restconfCapabilities defer func() { restconfCapabilities = rcCaps }() @@ -69,15 +81,15 @@ func TestQuery_depth(t *testing.T) { restconfCapabilities.depth = true // run depth test cases for GET and HEAD - testDepth(t, "=unbounded", "depth=unbounded", &translibArgs{depth: 0}) - testDepth(t, "=0", "depth=0", nil) - testDepth(t, "=1", "depth=1", &translibArgs{depth: 1}) - testDepth(t, "=101", "depth=101", &translibArgs{depth: 101}) - testDepth(t, "=65535", "depth=65535", &translibArgs{depth: 65535}) - testDepth(t, "=65536", "depth=65536", nil) - testDepth(t, "=junk", "depth=junk", nil) - testDepth(t, "extra", "depth=1&extra=1", nil) - testDepth(t, "dup", "depth=1&depth=2", nil) + testGetQuery(t, "=unbounded", "depth=unbounded", &translibArgs{depth: 0}) + testGetQuery(t, "=0", "depth=0", nil) + testGetQuery(t, "=1", "depth=1", &translibArgs{depth: 1}) + testGetQuery(t, "=101", "depth=101", &translibArgs{depth: 101}) + testGetQuery(t, "=65535", "depth=65535", &translibArgs{depth: 65535}) + testGetQuery(t, "=65536", "depth=65536", nil) + testGetQuery(t, "=junk", "depth=junk", nil) + testGetQuery(t, "extra", "depth=1&extra=1", nil) + testGetQuery(t, "dup", "depth=1&depth=2", nil) // check for other methods t.Run("OPTIONS", testQuery("OPTIONS", "depth=1", nil)) @@ -93,12 +105,8 @@ func TestQuery_depth_disabled(t *testing.T) { restconfCapabilities.depth = false - testDepth(t, "100", "depth=100", nil) -} - -func testDepth(t *testing.T, name, queryStr string, exp *translibArgs) { - t.Run("GET/"+name, testQuery("GET", queryStr, exp)) - t.Run("HEAD/"+name, testQuery("HEAD", queryStr, exp)) + testGetQuery(t, "100", "depth=100", nil) + restconfCapabilities.depth = true } func TestQuery_deleteEmptyEntry(t *testing.T) { @@ -114,3 +122,132 @@ func TestQuery_deleteEmptyEntry(t *testing.T) { t.Run("POST", testQuery("POST", "deleteEmptyEntry=true", nil)) t.Run("PATCH", testQuery("PATCH", "deleteEmptyEntry=true", nil)) } + +func TestQuery_content(t *testing.T) { + rcCaps := restconfCapabilities + defer func() { restconfCapabilities = rcCaps }() + + restconfCapabilities.content = true + + // run content query test cases for GET and HEAD + testGetQuery(t, "=all", "content=all", &translibArgs{content: "all"}) + testGetQuery(t, "=ALL", "content=ALL", nil) + testGetQuery(t, "=config", "content=config", &translibArgs{content: "config"}) + testGetQuery(t, "=Config", "content=Config", nil) + testGetQuery(t, "=nonconfig", "content=nonconfig", &translibArgs{content: "nonconfig"}) + testGetQuery(t, "=NonConfig", "content=NonConfig", nil) + testGetQuery(t, "=getall", "content=getall", nil) + testGetQuery(t, "=operational", "content=operational", nil) + testGetQuery(t, "=state", "content=state", nil) + testGetQuery(t, "=0", "content=0", nil) + testGetQuery(t, "dup", "content=config&content=nonconfig", nil) + + // check for other methods + t.Run("OPTIONS", testQuery("OPTIONS", "content=config", nil)) + t.Run("PUT", testQuery("PUT", "content=config", nil)) + t.Run("POST", testQuery("POST", "content=config", nil)) + t.Run("PATCH", testQuery("PATCH", "content=config", nil)) + t.Run("DELETE", testQuery("DELETE", "content=config", nil)) +} + +func TestQuery_content_disabled(t *testing.T) { + rcCaps := restconfCapabilities + defer func() { restconfCapabilities = rcCaps }() + + restconfCapabilities.content = false + testGetQuery(t, "config", "content=config", nil) + restconfCapabilities.content = true +} + +func TestQuery_fields(t *testing.T) { + rcCaps := restconfCapabilities + defer func() { restconfCapabilities = rcCaps }() + + restconfCapabilities.depth = true + + // run depth test cases for GET and HEAD + testGetQuery(t, "testfield1", "fields=description", &translibArgs{fields: []string{"description"}}) + testGetQuery(t, "testfield2", "fields=description;mtu", &translibArgs{fields: []string{"description", "mtu"}}) + testGetQuery(t, "testfield3", "fields=description,mtu", &translibArgs{fields: []string{"description,mtu"}}) + testGetQuery(t, "testfield4", "fields=config/description;mtu", &translibArgs{fields: []string{"config/description", "mtu"}}) + testGetQuery(t, "testfield4", "fields=config/description,mtu", &translibArgs{fields: []string{"config/description,mtu"}}) + testGetQuery(t, "testfield5", "fields=config(description;mtu)", &translibArgs{fields: []string{"config/description", "config/mtu"}}) + testGetQuery(t, "testfield6", "fields=config(description;mtu);state", &translibArgs{fields: []string{"config/description", "config/mtu", "state"}}) + testGetQuery(t, "testfield7", "fields=config(description;mtu),state", &translibArgs{fields: []string{"config/description", "config/mtu", ",state"}}) + testGetQuery(t, "testfield8", "fields=config(description,mtu),state", &translibArgs{fields: []string{"config/description,mtu", ",state"}}) + testGetQuery(t, "testfield9", "fields=config(description;mtu);state/mtu", &translibArgs{fields: []string{"config/description", "config/mtu", "state/mtu"}}) + testGetQuery(t, "testfield10", "fields=config(description;mtu);state(mtu)", &translibArgs{fields: []string{"config/description", "config/mtu", "state/mtu"}}) + testGetQuery(t, "testfield11", "fields=config(description;mtu);state(mtu;counters)", &translibArgs{fields: []string{"config/description", "config/mtu", "state/mtu", "state/counters"}}) + testGetQuery(t, "testfield12", "fields=config(description;mtu)&state", nil) + testGetQuery(t, "testfield13", "fields=config(description,mtu)&state=test", nil) + testGetQuery(t, "testfield14", "fields=config/mtu@state", &translibArgs{fields: []string{"config/mtu@state"}}) + testGetQuery(t, "testfield15", "fields=config(description,mtu)@state", &translibArgs{fields: []string{"config/description,mtu", "@state"}}) + testGetQuery(t, "testfield16", "fields=mtu&depth=2", nil) + testGetQuery(t, "testfield17", "fields=mtu&content=all", nil) + testGetQuery(t, "testfield18", "fields=mtu&depth=2&content=all", nil) + + // check for other methods + t.Run("OPTIONS", testQuery("OPTIONS", "fields=mtu", nil)) + t.Run("PUT", testQuery("PUT", "fields=mtu", nil)) + t.Run("POST", testQuery("POST", "fields=mtu", nil)) + t.Run("PATCH", testQuery("PATCH", "fields=mtu", nil)) + t.Run("DELETE", testQuery("DELETE", "fields=mtu", nil)) +} + +func TestQuery_fields_disabled(t *testing.T) { + rcCaps := restconfCapabilities + defer func() { restconfCapabilities = rcCaps }() + + restconfCapabilities.fields = false + + testGetQuery(t, "100", "fields=100", nil) + restconfCapabilities.fields = true +} + +func TestQuery_DepthContent(t *testing.T) { + rcCaps := restconfCapabilities + defer func() { restconfCapabilities = rcCaps }() + + restconfCapabilities.content = true + restconfCapabilities.depth = true + + // run content and depth query test cases for GET and HEAD + testGetQuery(t, "testDepConQuery1", "depth=unbounded&content=all", &translibArgs{content: "all", depth: 0}) + testGetQuery(t, "testDepConQuery2", "depth=1&content=all", &translibArgs{content: "all", depth: 1}) + testGetQuery(t, "testDepConQuery3", "depth=3&content=config", &translibArgs{content: "config", depth: 3}) + testGetQuery(t, "testDepConQuery4", "depth=4&content=nonconfig", &translibArgs{content: "nonconfig", depth: 4}) + testGetQuery(t, "testDepConQuery5", "depth=65535&content=nonconfig", &translibArgs{content: "nonconfig", depth: 65535}) + testGetQuery(t, "testDepConQuery6", "depth=65536&content=all", nil) + testGetQuery(t, "testDepConQuery7", "depth=5&content=ALL", nil) + testGetQuery(t, "testDepConQuery8", "depth=5&content=9", nil) + testGetQuery(t, "testDepConQuery9", "depth=%$&content=state", nil) + testGetQuery(t, "testDepConQuery10", "depth=3$&depth=2&content=all", nil) + testGetQuery(t, "testDepConQuery11", "depth=3$&content=all&content=config", nil) + testGetQuery(t, "testDepConQuery12", "depth=3$&content=all&content=config", nil) + testGetQuery(t, "testDepConQuery13", "depth=3;content=all", nil) + + // check for other methods + t.Run("OPTIONS", testQuery("OPTIONS", "depth=3&content=config", nil)) + t.Run("PUT", testQuery("PUT", "depth=3&content=config", nil)) + t.Run("POST", testQuery("POST", "depth=3&content=config", nil)) + t.Run("PATCH", testQuery("PATCH", "depth=3&content=config", nil)) + t.Run("DELETE", testQuery("DELETE", "depth=3&content=config", nil)) +} + +func TestQuery_depth_content_disabled(t *testing.T) { + rcCaps := restconfCapabilities + defer func() { restconfCapabilities = rcCaps }() + + restconfCapabilities.content = false + restconfCapabilities.depth = true + testGetQuery(t, "config", "depth=3&content=config", nil) + + restconfCapabilities.content = true + restconfCapabilities.depth = false + testGetQuery(t, "config", "depth=3&content=config", nil) + + restconfCapabilities.content = false + restconfCapabilities.depth = false + testGetQuery(t, "config", "depth=3&content=config", nil) + +} diff --git a/rest/server/restconf.go b/rest/server/restconf.go index 5f698a5786..a62f297d88 100644 --- a/rest/server/restconf.go +++ b/rest/server/restconf.go @@ -39,7 +39,9 @@ const ( // restconfCapabilities defines server capabilities var restconfCapabilities struct { - depth bool // depth query parameter + depth bool // depth query parameter + content bool // content query parameter + fields bool // fields query parameter } func init() { @@ -51,6 +53,9 @@ func init() { AddRoute("yanglibVersionHandler", "GET", "/restconf/yang-library-version", yanglibVersionHandler) // RESTCONF capability handler + restconfCapabilities.depth = true + restconfCapabilities.content = true + restconfCapabilities.fields = true AddRoute("capabilityHandler", "GET", "/restconf/data/ietf-restconf-monitoring:restconf-state/capabilities", capabilityHandler) AddRoute("capabilityHandler", "GET", @@ -111,7 +116,14 @@ func capabilityHandler(w http.ResponseWriter, r *http.Request) { c.Capabilities.Capability = append(c.Capabilities.Capability, "urn:ietf:params:restconf:capability:depth:1.0") } - + if restconfCapabilities.content { + c.Capabilities.Capability = append(c.Capabilities.Capability, + "urn:ietf:params:restconf:capability:content:1.0") + } + if restconfCapabilities.fields { + c.Capabilities.Capability = append(c.Capabilities.Capability, + "urn:ietf:params:restconf:capability:fields:1.0") + } var data []byte if strings.HasSuffix(r.URL.Path, "/capabilities") { data, _ = json.Marshal(&c) diff --git a/rest/server/restconf_test.go b/rest/server/restconf_test.go index 7cc3686ec8..907f77ba63 100644 --- a/rest/server/restconf_test.go +++ b/rest/server/restconf_test.go @@ -165,6 +165,17 @@ func testCapability(t *testing.T, path string) { if c, ok := cap.([]interface{}); !ok || len(c) == 0 { log.Fatalf("Could not parse capability info: %s", w.Body.String()) } + + var curCap []interface{} + curCap = append(curCap, "urn:ietf:params:restconf:capability:defaults:1.0?basic-mode=report-all", + "urn:ietf:params:restconf:capability:depth:1.0", + "urn:ietf:params:restconf:capability:content:1.0", + "urn:ietf:params:restconf:capability:fields:1.0") + + if !reflect.DeepEqual(cap.([]interface{}), curCap) { + t.Fatalf("Response does not include expected capabilities \n"+ + "expected: %v\nfound: %v", curCap, cap) + } } func TestOpsDiscovery_none(t *testing.T) {