Refactor apigen package and check in generated protos

.github/linters/.protolintrc.yml

@@ -0,0 +1,5 @@
+lint:
+  directories:
+    # The SAI generated proto doesn't follow all the proto naming conventions.
+    exclude:
+      - dataplane/standalone/proto

dataplane/standalone/apigen/BUILD

@@ -5,14 +5,13 @@ go_library(
     srcs = [
         "apigen.go",
         "ccgen.go",
-        "protogen.go",
-        "xml.go",
     ],
     importpath = "github.com/openconfig/lemming/dataplane/standalone/apigen",
     visibility = ["//visibility:private"],
     deps = [
-        "@com_github_golang_glog//:glog",
-        "@com_github_stoewer_go_strcase//:go-strcase",
+        "//dataplane/standalone/apigen/docparser",
+        "//dataplane/standalone/apigen/protogen",
+        "//dataplane/standalone/apigen/saiast",
         "@org_modernc_cc_v4//:cc",
     ],
 )

dataplane/standalone/apigen/README.md

@@ -0,0 +1,8 @@
+# APIGen
+
+## How to Use
+
+1. Checkout the [SAI API](https://github.com/opencomputeproject/SAI)
+2. Install doxygen and run `make doc`
+3. Copy the xml files to dataplane/standalone/apigen/xml
+4. Run the apigen `go run ./dataplane/standalone/apigen`

dataplane/standalone/apigen/apigen.go

@@ -20,123 +20,16 @@ import (
 	"log"
 	"os"
 	"path/filepath"
-	"regexp"
 	"runtime"
 	"strings"
 
+	"github.com/openconfig/lemming/dataplane/standalone/apigen/docparser"
+	"github.com/openconfig/lemming/dataplane/standalone/apigen/protogen"
+	"github.com/openconfig/lemming/dataplane/standalone/apigen/saiast"
+
 	cc "modernc.org/cc/v4"
 )
 
-type saiAPI struct {
-	ifaces []*saiInterface
-	funcs  map[string]*saiFunc
-}
-
-type saiInterface struct {
-	name  string
-	funcs []typeDecl
-}
-
-type saiFunc struct {
-	name       string
-	returnType string
-	params     []typeDecl
-}
-
-type typeDecl struct {
-	name string
-	typ  string
-}
-
-func handleFunc(name string, decl *cc.Declaration, directDecl *cc.DirectDeclarator) *saiFunc {
-	sf := &saiFunc{
-		name:       name,
-		returnType: decl.DeclarationSpecifiers.Type().String(),
-	}
-	if directDecl.ParameterTypeList == nil {
-		return nil
-	}
-	for paramList := directDecl.ParameterTypeList.ParameterList; paramList != nil; paramList = paramList.ParameterList {
-		if paramList.ParameterDeclaration.Declarator == nil { // When the parameter is void.
-			return nil
-		}
-		pd := typeDecl{
-			name: paramList.ParameterDeclaration.Declarator.Name(),
-			typ:  paramList.ParameterDeclaration.Declarator.Type().String(),
-		}
-		if ptr, ok := paramList.ParameterDeclaration.Declarator.Type().(*cc.PointerType); ok {
-			pd.typ = ptr.Elem().String()
-			pd.name = fmt.Sprintf("*%s", pd.name)
-			if ptr2, ok := ptr.Elem().(*cc.PointerType); ok {
-				pd.typ = ptr2.Elem().String()
-				pd.name = fmt.Sprintf("*%s", pd.name)
-			}
-		}
-
-		if paramList.ParameterDeclaration.DeclarationSpecifiers.TypeQualifier != nil && paramList.ParameterDeclaration.DeclarationSpecifiers.TypeQualifier.Case == cc.TypeQualifierConst {
-			pd.typ = fmt.Sprintf("const %s", pd.typ)
-		}
-		sf.params = append(sf.params, pd)
-	}
-	return sf
-}
-
-func handleIfaces(name string, decl *cc.Declaration) *saiInterface {
-	ts := decl.DeclarationSpecifiers.DeclarationSpecifiers.TypeSpecifier
-	if ts.StructOrUnionSpecifier == nil || ts.StructOrUnionSpecifier.StructDeclarationList == nil {
-		return nil
-	}
-	if !strings.Contains(name, "api_t") {
-		return nil
-	}
-	si := &saiInterface{
-		name: name,
-	}
-
-	structSpec := ts.StructOrUnionSpecifier.StructDeclarationList
-	for sd := structSpec; sd != nil; sd = sd.StructDeclarationList {
-		si.funcs = append(si.funcs, typeDecl{
-			name: sd.StructDeclaration.StructDeclaratorList.StructDeclarator.Declarator.Name(),
-			typ:  sd.StructDeclaration.StructDeclaratorList.StructDeclarator.Declarator.Type().String(),
-		})
-	}
-	return si
-}
-
-func getFuncAndTypes(ast *cc.AST) *saiAPI {
-	sa := saiAPI{
-		funcs: map[string]*saiFunc{},
-	}
-	for unit := ast.TranslationUnit; unit != nil; unit = unit.TranslationUnit {
-		decl := unit.ExternalDeclaration.Declaration
-		if decl == nil {
-			continue
-		}
-		if decl.InitDeclaratorList == nil {
-			continue
-		}
-
-		name := decl.InitDeclaratorList.InitDeclarator.Declarator.Name()
-		if !strings.Contains(name, "sai") {
-			continue
-		}
-
-		directDecl := decl.InitDeclaratorList.InitDeclarator.Declarator.DirectDeclarator
-		if directDecl != nil { // Possible func declaration.
-			if fn := handleFunc(name, decl, directDecl); fn != nil {
-				sa.funcs[fn.name] = fn
-			}
-		}
-		// Possible struct type declaration.
-		if decl.DeclarationSpecifiers != nil && decl.DeclarationSpecifiers.DeclarationSpecifiers != nil && decl.DeclarationSpecifiers.DeclarationSpecifiers.TypeSpecifier != nil {
-			if si := handleIfaces(name, decl); si != nil {
-				sa.ifaces = append(sa.ifaces, si)
-			}
-		}
-	}
-	return &sa
-}
-
 func parse(header string, includePaths ...string) (*cc.AST, error) {
 	cfg, err := cc.NewConfig(runtime.GOOS, runtime.GOARCH)
 	if err != nil {
@@ -160,19 +53,6 @@ const (
 	protoOutDir = "dataplane/standalone/proto"
 )
 
-var (
-	supportedOperation = map[string]bool{
-		"create":        true,
-		"remove":        true,
-		"get_attribute": true,
-		"set_attribute": true,
-		"clear_stats":   true,
-		"get_stats":     true,
-		"get_stats_ext": true,
-	}
-	funcExpr = regexp.MustCompile(`^([a-z]*_)(\w*)_(attribute|stats_ext|stats)|([a-z]*)_(\w*)$`)
-)
-
 func generate() error {
 	headerFile, err := filepath.Abs(filepath.Join(saiPath, "inc/sai.h"))
 	if err != nil {
@@ -190,36 +70,30 @@ func generate() error {
 	if err != nil {
 		return err
 	}
-	sai := getFuncAndTypes(ast)
-	xmlInfo, err := parseSAIXMLDir()
+	sai := saiast.Get(ast)
+	xmlInfo, err := docparser.ParseSAIXMLDir()
 	if err != nil {
 		return err
 	}
 
-	apis := make(map[string]*protoAPITmplData)
-	common, err := populateCommonTypes(xmlInfo)
+	protos, err := protogen.Generate(xmlInfo, sai)
 	if err != nil {
 		return err
 	}
 
-	for _, iface := range sai.ifaces {
-		nameTrimmed := strings.TrimSuffix(strings.TrimPrefix(iface.name, "sai_"), "_api_t")
+	for _, iface := range sai.Ifaces {
+		nameTrimmed := strings.TrimSuffix(strings.TrimPrefix(iface.Name, "sai_"), "_api_t")
 		ccData := ccTemplateData{
 			IncludeGuard: fmt.Sprintf("DATAPLANE_STANDALONE_SAI_%s_H_", strings.ToUpper(nameTrimmed)),
 			Header:       fmt.Sprintf("%s.h", nameTrimmed),
-			APIType:      iface.name,
+			APIType:      iface.Name,
 			APIName:      nameTrimmed,
 		}
-		for _, fn := range iface.funcs {
-			tf, isSwitchScoped, entry := createCCData(sai, fn)
+		for _, fn := range iface.Funcs {
+			meta := sai.GetFuncMeta(fn)
+			tf := createCCData(meta, sai, fn)
 			ccData.Funcs = append(ccData.Funcs, *tf)
-
-			err := populateTmplDataFromFunc(apis, xmlInfo, tf.Name, entry, tf.Operation, tf.TypeName, iface.name, isSwitchScoped)
-			if err != nil {
-				return err
-			}
 		}
-
 		header, err := os.Create(filepath.Join(ccOutDir, ccData.Header))
 		if err != nil {
 			return err
@@ -228,26 +102,21 @@ func generate() error {
 		if err != nil {
 			return err
 		}
-		proto, err := os.Create(filepath.Join(protoOutDir, fmt.Sprintf("%s.proto", nameTrimmed)))
-		if err != nil {
-			return err
-		}
+
 		if err := headerTmpl.Execute(header, ccData); err != nil {
 			return err
 		}
 		if err := ccTmpl.Execute(impl, ccData); err != nil {
 			return err
 		}
-		if err := protoTmpl.Execute(proto, apis[iface.name]); err != nil {
+	}
+	for file, content := range protos {
+		if err := os.WriteFile(filepath.Join(protoOutDir, file), []byte(content), 0600); err != nil {
 			return err
 		}
 	}
-	protoCommonFile, err := os.Create(filepath.Join(protoOutDir, "common.proto"))
-	if err != nil {
-		return err
-	}
 
-	return protoCommonTmpl.Execute(protoCommonFile, common)
+	return nil
 }
 
 func main() {

dataplane/standalone/apigen/ccgen.go

@@ -18,58 +18,56 @@ import (
 	"fmt"
 	"strings"
 	"text/template"
+
+	"github.com/openconfig/lemming/dataplane/standalone/apigen/saiast"
 )
 
-func createCCData(sai *saiAPI, fn typeDecl) (*templateFunc, bool, string) {
-	name := strings.TrimSuffix(strings.TrimPrefix(fn.name, "sai_"), "_fn")
+func createCCData(meta *saiast.FuncMetadata, sai *saiast.SAIAPI, fn *saiast.TypeDecl) *templateFunc {
+	name := meta.Name
 	tf := &templateFunc{
-		ReturnType: sai.funcs[fn.typ].returnType,
+		ReturnType: sai.Funcs[fn.Typ].ReturnType,
 		Name:       name,
+		Operation:  meta.Operation,
+		TypeName:   meta.TypeName,
 	}
 
-	isSwitchScoped := false
-	entryType := ""
 	var paramDefs []string
 	var paramVars []string
-	for i, param := range sai.funcs[fn.typ].params {
-		paramDefs = append(paramDefs, fmt.Sprintf("%s %s", param.typ, param.name))
-		name := strings.ReplaceAll(param.name, "*", "")
+	for _, param := range sai.Funcs[fn.Typ].Params {
+		paramDefs = append(paramDefs, fmt.Sprintf("%s %s", param.Typ, param.Name))
+		name := strings.ReplaceAll(param.Name, "*", "")
 		// Functions that operator on entries take some entry type instead of an object id as argument.
 		// Generate a entry union with the pointer to entry instead.
-		if strings.Contains(param.typ, "entry") {
+		if strings.Contains(param.Typ, "entry") {
 			tf.Entry = fmt.Sprintf("common_entry_t entry = {.%s = %s};", name, name)
 			name = "entry"
-			entryType = trimSAIName(strings.TrimPrefix(param.typ, "const "), true, false)
-		}
-		if i == 1 && param.name == "switch_id" {
-			isSwitchScoped = true
 		}
 		paramVars = append(paramVars, name)
 	}
 	tf.Args = strings.Join(paramDefs, ", ")
 	tf.Vars = strings.Join(paramVars, ", ")
 
-	matches := funcExpr.FindStringSubmatch(name)
-	tf.Operation = matches[1] + matches[4] + matches[3]
-
 	tf.UseCommonAPI = supportedOperation[tf.Operation]
-	tf.TypeName = strings.ToUpper(matches[2]) + strings.ToUpper(matches[5])
-
-	// Handle plural types using the bulk API.
-	if strings.HasSuffix(tf.TypeName, "PORTS") || strings.HasSuffix(tf.TypeName, "ENTRIES") || strings.HasSuffix(tf.TypeName, "MEMBERS") || strings.HasSuffix(tf.TypeName, "LISTS") {
-		tf.Operation += "_bulk"
-		tf.TypeName = strings.TrimSuffix(tf.TypeName, "S")
-		if strings.HasSuffix(tf.TypeName, "IE") {
-			tf.TypeName = strings.TrimSuffix(tf.TypeName, "IE")
-			tf.TypeName += "Y"
-		}
-	}
 
 	// Function or types that don't follow standard naming.
 	if strings.Contains(tf.TypeName, "PORT_ALL") || strings.Contains(tf.TypeName, "ALL_NEIGHBOR") {
 		tf.UseCommonAPI = false
 	}
-	return tf, isSwitchScoped, entryType
+	return tf
+}
+
+var supportedOperation = map[string]bool{
+	"create":             true,
+	"remove":             true,
+	"get_attribute":      true,
+	"set_attribute":      true,
+	"clear_stats":        true,
+	"get_stats":          true,
+	"get_stats_ext":      true,
+	"create_bulk":        true,
+	"remove_bulk":        true,
+	"set_attribute_bulk": true,
+	"get_attribute_bulk": true,
 }
 
 var (

dataplane/standalone/apigen/docparser/BUILD

@@ -0,0 +1,8 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_library")
+
+go_library(
+    name = "docparser",
+    srcs = ["docparser.go"],
+    importpath = "github.com/openconfig/lemming/dataplane/standalone/apigen/docparser",
+    visibility = ["//visibility:public"],
+)