Merge pull request #550 from meshery/models_registraion

Central Model Registration Logic

models/registration/dir.go

@@ -0,0 +1,93 @@
+package registration
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"reflect"
+
+	"github.com/layer5io/meshkit/models/meshmodel/core/v1alpha2"
+	"github.com/layer5io/meshkit/models/meshmodel/core/v1beta1"
+	"github.com/layer5io/meshkit/models/meshmodel/entity"
+	"github.com/layer5io/meshkit/utils"
+)
+
+type Dir struct {
+	dirpath string
+}
+
+/*
+	The directory should contain one and only one `model`.
+	A directory containing multiple `model` will be invalid.
+*/
+func NewDir(path string) Dir {
+	return Dir{dirpath: path}
+}
+
+
+/* 
+    PkgUnit parses all the files inside the directory and finds out if they are any valid meshery definitions. Valid meshery definitions are added to the packagingUnit struct.
+    Invalid definitions are stored in the regErrStore with error data.
+*/
+func (d Dir) PkgUnit(regErrStore RegistrationErrorStore) (_ packagingUnit, err error){
+	pkg := packagingUnit{}
+	// check if the given is a directory
+	_, err = os.ReadDir(d.dirpath)
+	if(err != nil){
+		return pkg, ErrDirPkgUnitParseFail(d.dirpath, fmt.Errorf("Could not read the directory: %e", err))
+	}
+	err = filepath.Walk(d.dirpath, func (path string, f os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+		if(f.IsDir()){
+				return nil
+		}
+		byt, _ := os.ReadFile(path)
+		if(byt == nil){
+			 return nil
+		}
+
+		var e entity.Entity
+		e, err = getEntity(byt)
+		if err != nil {
+			regErrStore.AddInvalidDefinition(path, err)
+			return nil
+		}
+		// set it to pkgunit
+		switch e.Type() {
+			case entity.Model:
+				if !reflect.ValueOf(pkg.model).IsZero() {
+					// currently models inside models are not handled
+					return nil
+				}
+				model, err := utils.Cast[*v1beta1.Model](e)
+				if(err != nil){
+					regErrStore.AddInvalidDefinition(path, ErrGetEntity(err))
+				}
+				pkg.model = *model
+			case entity.ComponentDefinition:
+				comp , err := utils.Cast[*v1beta1.ComponentDefinition](e)
+				if(err != nil){
+					regErrStore.AddInvalidDefinition(path, ErrGetEntity(err))
+				}
+				pkg.components = append(pkg.components, *comp)
+			case entity.RelationshipDefinition:
+				rel, err := utils.Cast[*v1alpha2.RelationshipDefinition](e)
+				if(err != nil){
+					regErrStore.AddInvalidDefinition(path, ErrGetEntity(err))
+				}
+				pkg.relationships = append(pkg.relationships, *rel)
+		}
+		return nil
+	})
+	if err != nil {
+		return pkg, ErrDirPkgUnitParseFail(d.dirpath, fmt.Errorf("Could not completely walk the file tree: %e", err))
+	}
+	if (reflect.ValueOf(pkg.model).IsZero()){
+		err := fmt.Errorf("Model definition not found in imported package. Model definitions often use the filename `model.json`, but are not required to have this filename. One and exactly one entity containing schema: model.core....... ...... must be present, otherwise the model package is considered malformed..")
+        regErrStore.AddInvalidDefinition(d.dirpath, err)
+		return pkg, err
+	}
+	return pkg, err
+}

models/registration/error.go

@@ -0,0 +1,85 @@
+package registration
+
+import (
+	"fmt"
+
+	"github.com/layer5io/meshkit/errors"
+)
+
+const (
+	ErrDirPkgUnitParseFailCode = "replace_me"
+	ErrGetEntityCode = "replace_me"
+	ErrRegisterEntityCode = "replace_me"
+	ErrImportFailureCode = "replace_me"
+	ErrMissingRegistrantCode = "replace_me"
+	ErrSeedingComponentsCode = "replace-me"
+)
+
+
+
+func ErrSeedingComponents(err error) error {
+	return errors.New(
+		ErrSeedingComponentsCode,
+		errors.Alert,
+		[]string{"Failed to register the given models into meshery's registry"},
+		[]string{err.Error()},
+		[]string{"Given models may not be in accordance with Meshery's schema", "Internal(OS level) error while reading files" },
+		[]string{"Make sure the models being seeded are valid in accordance with Meshery's schema", "If it is an internal error, please try again after some time"},
+	)
+}
+
+func ErrMissingRegistrant(modelName string) error {
+	return errors.New(
+		ErrMissingRegistrantCode,
+		errors.Alert,
+		[]string{fmt.Sprintf("Model with name: %s does not have registrant information", modelName)},
+		[]string{"Meshery models are always registered in context of a registrant."},
+		// there is only one cause for this error
+		[]string{""},
+		[]string{"Make sure that the registrant information is present in the model definition"},
+	)
+}
+
+func ErrRegisterEntity(err error, name, entity string) error {
+	return errors.New(
+		ErrRegisterEntityCode,
+		errors.Alert,
+		[]string{fmt.Sprintf("Failed to register an entity of name: %s and type: %s into Meshery's registry", name, entity)},
+		[]string{err.Error()},
+		[]string{fmt.Sprintf("%s definition might be violating the definition schema", entity), fmt.Sprintf("%s might be missing model details", entity)},
+		[]string{fmt.Sprintf("ensure the %s definition follows the correct schema", entity), fmt.Sprintf("ensure %s definition belongs to correct model", entity)},
+	)
+}
+
+func ErrGetEntity(err error) error {
+	return errors.New(
+		ErrGetEntityCode,
+		errors.Alert,
+		[]string{"Could not parse the given data into any Meshery entity"},
+		[]string{err.Error()},
+		[]string{"Entity definition might not be in accordance with it's corresponding schema", "Might be an invalid/unsupported schemaVersion"},
+		[]string{"Ensure that the definition given is in accordance with it's schema", "Ensure that the schemaVersion used is valid and is supported by Meshery"},
+	)
+}
+
+func ErrDirPkgUnitParseFail(dirpath string, err error) error {
+	return errors.New(
+		ErrDirPkgUnitParseFailCode,
+		errors.Alert,
+		[]string{fmt.Sprintf("Directory at path: %s cannot be registered into Meshery", dirpath)},
+		[]string{fmt.Sprint(err.Error())},
+		[]string{"The directory might not have a valid model definition", "Might be some internal issues while walking the file tree"},
+		[]string{"Make sure that there is a valid model definition present in the directory. Meshery's registration pipeline currently does not support nested models, therefore the behaviour might be unexpected if it contains nested models.", "If there is an internal error, please try again after some time"},
+	)
+}
+
+func ErrImportFailure(hostname string, failedMsg string) error {
+	return errors.New(
+		ErrImportFailureCode,
+		errors.Alert,
+		[]string{fmt.Sprintf("Errors while registering entities for registrant: %s", hostname)},
+		[]string{failedMsg},
+		[]string{"Entity definition might not be in accordance with schema", "Entity version might not be supported by Meshery"},
+		[]string{"See the registration logs (found at $HOME/.meshery/logs/registry/registry-logs.log) to find out which Entity failed to be imported with more specific error information."},
+	)
+}

models/registration/interface.go

@@ -0,0 +1,24 @@
+package registration
+
+import (
+	"github.com/layer5io/meshkit/models/meshmodel/entity"
+)
+
+/* 
+    RegistrationErrorStore stores all the errors that does not break the registration process, but have to be reported nevertheless.
+*/
+type RegistrationErrorStore interface {
+    AddInvalidDefinition(string,error)
+    InsertEntityRegError(hostname string, modelName string, entityType entity.EntityType, entityName string, err error)
+}
+
+// Anything that can be parsed into a packagingUnit is a RegisterableEntity in Meshery server
+type RegisterableEntity interface {
+   /*
+		1. `err` - this is a breaking error, which signifies that the given entity is invalid
+		2. Errors encountered while parsing items into meshmodel entites are stored in the RegistrationErrorStore
+	*/
+	PkgUnit(RegistrationErrorStore) (packagingUnit, error)
+}
+
+

models/registration/oci.go

@@ -0,0 +1,16 @@
+package registration
+
+
+import(
+	gcrv1 "github.com/google/go-containerregistry/pkg/v1"
+
+)
+
+type OCIImage struct {
+    _ gcrv1.Image
+}
+
+func (o OCIImage) PkgUnit(regErrStore RegistrationErrorStore) (packagingUnit, error){
+	pkg := packagingUnit{}
+	return pkg, nil
+}

models/registration/register.go

@@ -0,0 +1,95 @@
+package registration
+
+import (
+	"github.com/layer5io/meshkit/models/meshmodel/core/v1alpha2"
+	"github.com/layer5io/meshkit/models/meshmodel/core/v1beta1"
+	"github.com/layer5io/meshkit/models/meshmodel/entity"
+	meshmodel "github.com/layer5io/meshkit/models/meshmodel/registry"
+)
+
+// packaingUnit is the representation of the atomic unit that can be registered into the capabilities registry
+type packagingUnit struct {
+	model v1beta1.Model
+	components []v1beta1.ComponentDefinition
+	relationships []v1alpha2.RelationshipDefinition
+	_ []v1beta1.PolicyDefinition
+}
+
+type RegistrationHelper struct {
+	regManager       *meshmodel.RegistryManager
+    regErrStore      RegistrationErrorStore
+}
+
+func NewRegistrationHelper(regm *meshmodel.RegistryManager, regErrStore RegistrationErrorStore) RegistrationHelper {
+	return RegistrationHelper{ regManager: regm, regErrStore: regErrStore}
+}
+
+/*
+	Register will accept a RegisterableEntity (dir, tar or oci for now).
+*/
+func (rh *RegistrationHelper) Register(entity RegisterableEntity) error {
+	// get the packaging units
+	pu, err := entity.PkgUnit(rh.regErrStore)
+	if(err != nil){
+		// given input is not a valid model, or could not walk the directory
+		return err
+	}
+	// fmt.Printf("Packaging Unit: Model name: %s, comps: %d, rels: %d\n", pu.model.Name, len(pu.components), len(pu.relationships))
+	return rh.register(pu)
+}
+
+
+/*
+	register will return an error if it is not able to register the `model`.
+	If there are errors when registering other entities, they are handled properly but does not stop the registration process.
+*/
+func (rh *RegistrationHelper)register(pkg packagingUnit) error {
+	// 1. Register the model
+	model := pkg.model
+
+	// Dont register anything else if registrant is not there
+	if(model.Registrant.Hostname == ""){
+		err := ErrMissingRegistrant(model.Name)
+		rh.regErrStore.InsertEntityRegError(model.Registrant.Hostname, "",entity.Model, model.Name, err)
+		return err
+	}
+	_, _, err := rh.regManager.RegisterEntity(
+		v1beta1.Host{Hostname: model.Registrant.Hostname,},
+		&model,
+		)
+
+	// If model cannot be registered, don't register anything else
+	if err != nil {
+		err = ErrRegisterEntity(err, string(model.Type()), model.DisplayName)
+		rh.regErrStore.InsertEntityRegError(model.Registrant.Hostname, "",entity.Model, model.Name, err)
+		return err
+	}
+
+	hostname := model.Registrant.Hostname
+	modelName := model.Name
+	// 2. Register components
+	for _, comp := range pkg.components {
+		comp.Model = model
+		_, _, err := rh.regManager.RegisterEntity(
+		v1beta1.Host{Hostname: hostname,},
+		&comp,
+		)
+	if err != nil {
+		err = ErrRegisterEntity(err, string(comp.Type()), comp.DisplayName)
+		rh.regErrStore.InsertEntityRegError(hostname, modelName ,entity.ComponentDefinition, comp.DisplayName, err)
+	}
+	}
+
+	// 3. Register relationships
+	for _, rel := range pkg.relationships {
+		rel.Model = model
+		_, _, err := rh.regManager.RegisterEntity(v1beta1.Host{
+			Hostname: hostname,
+		}, &rel)
+		if err != nil {
+			err = ErrRegisterEntity(err, string(rel.Type()), rel.Kind)
+			rh.regErrStore.InsertEntityRegError(hostname, modelName ,entity.RelationshipDefinition, rel.ID.String(), err)
+		}
+	}
+    return nil
+}

models/registration/tar.go

@@ -0,0 +1,10 @@
+package registration
+
+type Tar struct {
+    _ string
+}
+
+func (t Tar) PkgUnit(regErrStore RegistrationErrorStore) (packagingUnit, error){
+	pkg := packagingUnit{}
+	return pkg, nil
+}

models/registration/utils.go

@@ -0,0 +1,57 @@
+package registration
+
+import (
+	"encoding/json"
+	"fmt"
+
+	"github.com/layer5io/meshkit/models/meshmodel/core/v1alpha2"
+	"github.com/layer5io/meshkit/models/meshmodel/core/v1beta1"
+	"github.com/layer5io/meshkit/models/meshmodel/entity"
+	"gopkg.in/yaml.v2"
+)
+
+func unmarshal(byt []byte, out interface{}) error {
+	err := json.Unmarshal(byt, out)
+	if(err != nil){
+		err = yaml.Unmarshal(byt, out)
+		if(err != nil){
+			return fmt.Errorf("Not a valid YAML or JSON")
+		}
+	}
+	return nil
+}
+
+// TODO: refactor this and use CUE
+func getEntity(byt []byte) (et entity.Entity, _ error) {
+    var versionMeta v1beta1.VersionMeta
+	err  := unmarshal(byt, &versionMeta)
+	if err != nil || versionMeta.SchemaVersion == "" {
+		return nil, ErrGetEntity(fmt.Errorf("Does not contain versionmeta"))
+	}
+	switch (versionMeta.SchemaVersion) {
+		case v1beta1.ComponentSchemaVersion:
+			var compDef v1beta1.ComponentDefinition
+			err  := unmarshal(byt, &compDef)
+			if err != nil {
+				return nil, ErrGetEntity(fmt.Errorf("Invalid component definition: %s", err.Error()))
+			}
+			et = &compDef
+		case v1beta1.ModelSchemaVersion:
+			var model v1beta1.Model
+			err  := unmarshal(byt,&model)
+			if err != nil {
+				return nil, ErrGetEntity(fmt.Errorf("Invalid model definition: %s", err.Error()))
+			}
+			et = &model
+		case v1alpha2.RelationshipSchemaVersion:
+			var rel v1alpha2.RelationshipDefinition
+			err  := unmarshal(byt,&rel)
+			if err != nil {
+				return nil, ErrGetEntity(fmt.Errorf("Invalid relationship definition: %s", err.Error()))
+			}
+			et = &rel
+		default:
+			return nil, ErrGetEntity(fmt.Errorf("Not a valid component definition, model definition, or relationship definition"))
+	}
+	return et, nil
+}