update registration logic to support import command w/signoff

models/meshmodel/registry/v1beta1/model_filter.go

@@ -102,7 +102,7 @@ func (mf *ModelFilter) Get(db *database.Handler) ([]entity.Entity, int64, int, e
 
 	if mf.Greedy {
 		if mf.Id != "" {
-			finder = finder.First("model_dbs.id = ?", mf.Id)
+			finder = finder.Where("model_dbs.id = ?", mf.Id)
 		}
 		if mf.Name != "" && mf.DisplayName != "" {
 			finder = finder.Where("model_dbs.name LIKE ? OR model_dbs.display_name LIKE ?", "%"+mf.Name+"%", "%"+mf.DisplayName+"%")

models/registration/dir.go

@@ -7,7 +7,10 @@ import (
 	"reflect"
 
 	"github.com/layer5io/meshkit/models/meshmodel/entity"
+	"github.com/layer5io/meshkit/models/oci"
+
 	"github.com/layer5io/meshkit/utils"
+
 	"github.com/meshery/schemas/models/v1alpha3/relationship"
 	"github.com/meshery/schemas/models/v1beta1/component"
 	"github.com/meshery/schemas/models/v1beta1/model"
@@ -26,69 +29,185 @@ func NewDir(path string) Dir {
 }
 
 /*
-PkgUnit parses all the files inside the directory and finds out if they are any valid meshery definitions. Valid meshery definitions are added to the packagingUnit struct.
+PkgUnit parses all the files inside the directory and finds out if they are any valid meshery definitions. Valid meshery definitions are added to the PackagingUnit struct.
 Invalid definitions are stored in the regErrStore with error data.
 */
-func (d Dir) PkgUnit(regErrStore RegistrationErrorStore) (_ packagingUnit, err error) {
-	pkg := packagingUnit{}
-	// check if the given is a directory
-	_, err = os.ReadDir(d.dirpath)
+func (d Dir) PkgUnit(regErrStore RegistrationErrorStore) (_ PackagingUnit, err error) {
+	pkg := PackagingUnit{}
+
+	// Extract the filename to use as entityName in case of errors
+	filename := filepath.Base(d.dirpath)
+
+	// Check if the given path is accessible
+	_, err = os.Stat(d.dirpath)
 	if err != nil {
-		return pkg, ErrDirPkgUnitParseFail(d.dirpath, fmt.Errorf("Could not read the directory: %e", err))
+		regErrStore.InsertEntityRegError("", "", entity.EntityType("unknown"), filename, ErrDirPkgUnitParseFail(d.dirpath, fmt.Errorf("could not access the path: %w", err)))
+		return pkg, ErrDirPkgUnitParseFail(d.dirpath, fmt.Errorf("could not access the path: %w", err))
+	}
+
+	// Process the path (file or directory)
+	err = processDir(d.dirpath, &pkg, regErrStore)
+	if err != nil {
+		modelName := ""
+		if !reflect.ValueOf(pkg.Model).IsZero() {
+			modelName = pkg.Model.Name
+		}
+		regErrStore.InsertEntityRegError("", modelName, entity.EntityType("unknown"), filename, ErrDirPkgUnitParseFail(d.dirpath, fmt.Errorf("could not process the path: %w", err)))
+		return pkg, ErrDirPkgUnitParseFail(d.dirpath, fmt.Errorf("could not process the path: %w", err))
 	}
-	err = filepath.Walk(d.dirpath, func(path string, f os.FileInfo, err error) error {
+
+	if reflect.ValueOf(pkg.Model).IsZero() {
+		errMsg := fmt.Errorf("model definition not found in imported package. Model definitions often use the filename `model.json`, but are not required to have this filename. One and exactly one entity containing schema: model.core must be present, otherwise the model package is considered malformed")
+		regErrStore.InsertEntityRegError("", "", entity.Model, filename, errMsg)
+		return pkg, errMsg
+	}
+
+	return pkg, nil
+}
+
+func processDir(dirPath string, pkg *PackagingUnit, regErrStore RegistrationErrorStore) error {
+	var tempDirs []string
+	defer func() {
+		for _, tempDir := range tempDirs {
+			os.RemoveAll(tempDir)
+		}
+	}()
+
+	return filepath.Walk(dirPath, func(path string, info os.FileInfo, err error) error {
 		if err != nil {
-			return err
+			err = utils.ErrFileWalkDir(fmt.Errorf("error accessing path: %w", err), path)
+			regErrStore.InsertEntityRegError("", "", entity.EntityType("unknown"), filepath.Base(path), err)
+			regErrStore.AddInvalidDefinition(path, err)
+			return nil
 		}
-		if f.IsDir() {
+		if info.IsDir() {
 			return nil
 		}
-		byt, _ := os.ReadFile(path)
-		if byt == nil {
+
+		// Read the file content
+		data, err := os.ReadFile(path)
+		if err != nil {
+			err = oci.ErrReadingFile(err)
+			regErrStore.InsertEntityRegError("", "", entity.EntityType("unknown"), filepath.Base(path), err)
+			regErrStore.AddInvalidDefinition(path, err)
 			return nil
 		}
 
-		var e entity.Entity
-		e, err = getEntity(byt)
+		// Check if the file is an OCI artifact
+		if oci.IsOCIArtifact(data) {
+			// Extract the OCI artifact
+			tempDir, err := oci.CreateTempOCIContentDir()
+			if err != nil {
+				regErrStore.InsertEntityRegError("", "", entity.EntityType("unknown"), filepath.Base(path), err)
+				regErrStore.AddInvalidDefinition(path, err)
+				return nil
+			}
+			tempDirs = append(tempDirs, tempDir)
+			err = oci.UnCompressOCIArtifact(path, tempDir)
+			if err != nil {
+				regErrStore.InsertEntityRegError("", "", entity.EntityType("unknown"), filepath.Base(path), err)
+				regErrStore.AddInvalidDefinition(path, err)
+				return nil
+			}
+			// Recursively process the extracted directory
+			if err := processDir(tempDir, pkg, regErrStore); err != nil {
+				return err
+			}
+			return nil
+		}
+
+		// Check if the file is a zip or tar file
+		if utils.IsZip(path) || utils.IsTarGz(path) {
+			tempDir, err := os.MkdirTemp("", "nested-extract-")
+			if err != nil {
+				err = utils.ErrCreateDir(fmt.Errorf("error creating temp directory for nested archive extraction: %w", err), tempDir)
+				regErrStore.InsertEntityRegError("", "", entity.EntityType("unknown"), filepath.Base(path), err)
+				regErrStore.AddInvalidDefinition(path, err)
+				return nil
+			}
+			tempDirs = append(tempDirs, tempDir)
+			if err := utils.ExtractFile(path, tempDir); err != nil {
+				regErrStore.InsertEntityRegError("", "", entity.EntityType("unknown"), filepath.Base(path), err)
+				regErrStore.AddInvalidDefinition(path, err)
+				return nil
+			}
+			// Recursively process the extracted directory
+			if err := processDir(tempDir, pkg, regErrStore); err != nil {
+				return err
+			}
+			return nil
+		}
+
+		content := data
+		content, err = utils.YAMLToJSON(content)
 		if err != nil {
+			regErrStore.InsertEntityRegError("", "", entity.EntityType("unknown"), filepath.Base(path), err)
+			return nil
+		}
+		// Determine the entity type
+		entityType, err := utils.FindEntityType(content)
+		if err != nil {
+			regErrStore.InsertEntityRegError("", "", entity.EntityType("unknown"), filepath.Base(path), err)
 			regErrStore.AddInvalidDefinition(path, err)
 			return nil
 		}
 
-		// set it to pkgunit
+		if entityType == "" {
+			// Not an entity we care about
+			return nil
+		}
+
+		// Get the entity
+		var e entity.Entity
+		e, err = getEntity(content)
+		if err != nil {
+			regErrStore.InsertEntityRegError("", "", entityType, filepath.Base(path), fmt.Errorf("could not get entity: %w", err))
+			regErrStore.AddInvalidDefinition(path, fmt.Errorf("could not get entity: %w", err))
+			return nil
+		}
+
+		// Add the entity to the packaging unit
 		switch e.Type() {
 		case entity.Model:
-			if !reflect.ValueOf(pkg.model).IsZero() {
-				// currently models inside models are not handled
-				return nil
-			}
 			model, err := utils.Cast[*model.ModelDefinition](e)
 			if err != nil {
+				modelName := ""
+				if model != nil {
+					modelName = model.Name
+				}
+				regErrStore.InsertEntityRegError("", modelName, entityType, modelName, ErrGetEntity(err))
 				regErrStore.AddInvalidDefinition(path, ErrGetEntity(err))
+				return nil
 			}
-			pkg.model = *model
+			pkg.Model = *model
 		case entity.ComponentDefinition:
 			comp, err := utils.Cast[*component.ComponentDefinition](e)
 			if err != nil {
+				componentName := ""
+				if comp != nil {
+					componentName = comp.Component.Kind
+				}
+				regErrStore.InsertEntityRegError("", "", entityType, componentName, ErrGetEntity(err))
 				regErrStore.AddInvalidDefinition(path, ErrGetEntity(err))
+				return nil
 			}
-			pkg.components = append(pkg.components, *comp)
+			pkg.Components = append(pkg.Components, *comp)
 		case entity.RelationshipDefinition:
 			rel, err := utils.Cast[*relationship.RelationshipDefinition](e)
 			if err != nil {
+				relationshipName := ""
+				if rel != nil {
+					relationshipName = rel.Model.Name
+				}
+				regErrStore.InsertEntityRegError("", "", entityType, relationshipName, ErrGetEntity(err))
 				regErrStore.AddInvalidDefinition(path, ErrGetEntity(err))
+				return nil
 			}
-			pkg.relationships = append(pkg.relationships, *rel)
+			pkg.Relationships = append(pkg.Relationships, *rel)
+		default:
+			// Unhandled entity type
+			return nil
 		}
 		return nil
 	})
-	if err != nil {
-		return pkg, ErrDirPkgUnitParseFail(d.dirpath, fmt.Errorf("Could not completely walk the file tree: %e", err))
-	}
-	if reflect.ValueOf(pkg.model).IsZero() {
-		err := fmt.Errorf("Model definition not found in imported package. Model definitions often use the filename `model.json`, but are not required to have this filename. One and exactly one entity containing schema: model.core....... ...... must be present, otherwise the model package is considered malformed..")
-		regErrStore.AddInvalidDefinition(d.dirpath, err)
-		return pkg, err
-	}
-	return pkg, err
 }

models/registration/interface.go

@@ -12,11 +12,11 @@ type RegistrationErrorStore interface {
 	InsertEntityRegError(hostname string, modelName string, entityType entity.EntityType, entityName string, err error)
 }
 
-// Anything that can be parsed into a packagingUnit is a RegisterableEntity in Meshery server
+// Anything that can be parsed into a PackagingUnit is a RegisterableEntity in Meshery server
 type RegisterableEntity interface {
 	/*
 		1. `err` - this is a breaking error, which signifies that the given entity is invalid and cannot be registered
 		2. Errors encountered while parsing items into meshmodel entites are stored in the RegistrationErrorStore
 	*/
-	PkgUnit(RegistrationErrorStore) (packagingUnit, error)
+	PkgUnit(RegistrationErrorStore) (PackagingUnit, error)
 }

models/registration/oci.go

@@ -8,7 +8,7 @@ type OCIImage struct {
 	_ gcrv1.Image
 }
 
-func (o OCIImage) PkgUnit(regErrStore RegistrationErrorStore) (packagingUnit, error) {
-	pkg := packagingUnit{}
+func (o OCIImage) PkgUnit(regErrStore RegistrationErrorStore) (PackagingUnit, error) {
+	pkg := PackagingUnit{}
 	return pkg, nil
 }

models/registration/register.go

@@ -10,22 +10,23 @@ import (
 	"github.com/meshery/schemas/models/v1beta1/model"
 )
 
-// packaingUnit is the representation of the atomic unit that can be registered into the capabilities registry
-type packagingUnit struct {
-	model         model.ModelDefinition
-	components    []component.ComponentDefinition
-	relationships []relationship.RelationshipDefinition
+// PackagingUnit is the representation of the atomic unit that can be registered into the capabilities registry
+type PackagingUnit struct {
+	Model         model.ModelDefinition
+	Components    []component.ComponentDefinition
+	Relationships []relationship.RelationshipDefinition
 	_             []v1beta1.PolicyDefinition
 }
 
 type RegistrationHelper struct {
 	regManager  *meshmodel.RegistryManager
 	regErrStore RegistrationErrorStore
 	svgBaseDir  string
+	PkgUnits    []PackagingUnit // Store successfully registered packagingUnits
 }
 
 func NewRegistrationHelper(svgBaseDir string, regm *meshmodel.RegistryManager, regErrStore RegistrationErrorStore) RegistrationHelper {
-	return RegistrationHelper{svgBaseDir: svgBaseDir, regManager: regm, regErrStore: regErrStore}
+	return RegistrationHelper{svgBaseDir: svgBaseDir, regManager: regm, regErrStore: regErrStore, PkgUnits: []PackagingUnit{}}
 }
 
 /*
@@ -45,11 +46,11 @@ func (rh *RegistrationHelper) Register(entity RegisterableEntity) {
 register will return an error if it is not able to register the `model`.
 If there are errors when registering other entities, they are handled properly but does not stop the registration process.
 */
-func (rh *RegistrationHelper) register(pkg packagingUnit) {
+func (rh *RegistrationHelper) register(pkg PackagingUnit) {
 	// 1. Register the model
-	model := pkg.model
+	model := pkg.Model
 
-	// Dont register anything else if registrant is not there
+	// Don't register anything else if registrant is not there
 	if model.Registrant.Kind == "" {
 		err := ErrMissingRegistrant(model.Name)
 		rh.regErrStore.InsertEntityRegError(model.Registrant.Kind, "", entity.Model, model.Name, err)
@@ -64,24 +65,22 @@ func (rh *RegistrationHelper) register(pkg packagingUnit) {
 
 		var svgCompletePath string
 
-		//Write SVG for models
-		model.Metadata.SvgColor, model.Metadata.SvgWhite, svgCompletePath = WriteAndReplaceSVGWithFileSystemPath(model.Metadata.SvgColor,
+		// Write SVG for models
+		model.Metadata.SvgColor, model.Metadata.SvgWhite, svgCompletePath = WriteAndReplaceSVGWithFileSystemPath(
+			model.Metadata.SvgColor,
 			model.Metadata.SvgWhite,
-			svgComplete, rh.svgBaseDir,
+			svgComplete,
+			rh.svgBaseDir,
 			model.Name,
 			model.Name,
 		)
 		if svgCompletePath != "" {
 			model.Metadata.SvgComplete = &svgCompletePath
 		}
-
 	}
 
 	model.Registrant.Status = connection.Registered
-	_, _, err := rh.regManager.RegisterEntity(
-		model.Registrant,
-		&model,
-	)
+	_, _, err := rh.regManager.RegisterEntity(model.Registrant, &model)
 
 	// If model cannot be registered, don't register anything else
 	if err != nil {
@@ -91,13 +90,16 @@ func (rh *RegistrationHelper) register(pkg packagingUnit) {
 	}
 
 	hostname := model.Registrant.Kind
-	modelName := model.Name
+
+	// Prepare slices to hold successfully registered components and relationships
+	var registeredComponents []component.ComponentDefinition
+	var registeredRelationships []relationship.RelationshipDefinition
 	// 2. Register components
-	for _, comp := range pkg.components {
+	for _, comp := range pkg.Components {
 		comp.Model = model
 
 		if comp.Styles != nil {
-			//Write SVG for components
+			// Write SVG for components
 			comp.Styles.SvgColor, comp.Styles.SvgWhite, comp.Styles.SvgComplete = WriteAndReplaceSVGWithFileSystemPath(
 				comp.Styles.SvgColor,
 				comp.Styles.SvgWhite,
@@ -108,23 +110,33 @@ func (rh *RegistrationHelper) register(pkg packagingUnit) {
 			)
 		}
 
-		_, _, err := rh.regManager.RegisterEntity(
-			model.Registrant,
-			&comp,
-		)
+		_, _, err := rh.regManager.RegisterEntity(model.Registrant, &comp)
 		if err != nil {
 			err = ErrRegisterEntity(err, string(comp.Type()), comp.DisplayName)
-			rh.regErrStore.InsertEntityRegError(hostname, modelName, entity.ComponentDefinition, comp.DisplayName, err)
+			rh.regErrStore.InsertEntityRegError(hostname, model.DisplayName, entity.ComponentDefinition, comp.DisplayName, err)
+		} else {
+			// Successful registration, add to successfulComponents
+			registeredComponents = append(registeredComponents, comp)
 		}
 	}
 
 	// 3. Register relationships
-	for _, rel := range pkg.relationships {
+	for _, rel := range pkg.Relationships {
 		rel.Model = model
 		_, _, err := rh.regManager.RegisterEntity(model.Registrant, &rel)
 		if err != nil {
 			err = ErrRegisterEntity(err, string(rel.Type()), string(rel.Kind))
-			rh.regErrStore.InsertEntityRegError(hostname, modelName, entity.RelationshipDefinition, rel.Id.String(), err)
+			rh.regErrStore.InsertEntityRegError(hostname, model.DisplayName, entity.RelationshipDefinition, rel.Id.String(), err)
+		} else {
+			// Successful registration, add to successfulRelationships
+			registeredRelationships = append(registeredRelationships, rel)
 		}
 	}
+
+	// Update pkg with only successfully registered components and relationships
+	pkg.Components = registeredComponents
+	pkg.Relationships = registeredRelationships
+	pkg.Model = model
+	// Store the successfully registered PackagingUnit
+	rh.PkgUnits = append(rh.PkgUnits, pkg)
 }