Merge pull request #539 from Jougan-0/modelImportUpdate

Model import update

models/meshmodel/core/v1alpha2/relationship.go

@@ -16,9 +16,9 @@ import (
 const RelationshipSchemaVersion = "relationships.meshery.io/v1alpha2"
 
 type RelationshipDefinition struct {
-	ID uuid.UUID `json:"id"`
-	v1beta1.VersionMeta
-	Kind string `json:"kind,omitempty" yaml:"kind"`
+	ID                  uuid.UUID `json:"id"`
+	v1beta1.VersionMeta `json:",inline" yaml:",inline"`
+	Kind                string `json:"kind,omitempty" yaml:"kind"`
 	// The property has been named RelationshipType instead of Type to avoid collision from Type() function, which enables support for dynamic type.
 	// Though, the column name and the json representation is "type".
 	RelationshipType string                   `json:"type" yaml:"type" gorm:"type"`

models/meshmodel/core/v1beta1/component.go

@@ -13,8 +13,8 @@ import (
 )
 
 type VersionMeta struct {
-	SchemaVersion string `json:"schemaVersion,omitempty" yaml:"schemaVersion"`
-	Version       string `json:"version,omitempty" yaml:"version"`
+	SchemaVersion string `json:"schemaVersion,omitempty" yaml:"schemaVersion" gorm:"column:schema_version"`
+	Version       string `json:"version,omitempty" yaml:"version" gorm:"column:version"`
 }
 
 type TypeMeta struct {
@@ -40,10 +40,10 @@ type ComponentEntity struct {
 
 // swagger:response ComponentDefinition
 type ComponentDefinition struct {
-	ID uuid.UUID `json:"id"`
-	VersionMeta
-	DisplayName string                 `json:"displayName" gorm:"displayName"`
-	Description string                 `json:"description" gorm:"description"`
+	ID          uuid.UUID `json:"id"`
+	VersionMeta `json:",inline" yaml:",inline"`
+	DisplayName string                 `json:"displayName" gorm:"column:display_name"`
+	Description string                 `json:"description" gorm:"column:description"`
 	Format      ComponentFormat        `json:"format" yaml:"format"`
 	ModelID     uuid.UUID              `json:"-" gorm:"index:idx_component_definition_dbs_model_id,column:model_id"`
 	Model       Model                  `json:"model" gorm:"foreignKey:ModelID;references:ID"`

models/meshmodel/core/v1beta1/models.go

@@ -24,8 +24,9 @@ type ModelEntity struct {
 
 // swagger:response Model
 type Model struct {
-	ID uuid.UUID `json:"id"`
-	VersionMeta
+	ID          uuid.UUID `json:"id"`
+	VersionMeta `yaml:",inline"`
+
 	Name          string                 `json:"name" gorm:"modelName"`
 	DisplayName   string                 `json:"displayName"`
 	Description   string                 `json:"description" gorm:"description"`

models/meshmodel/registry/v1beta1/model_filter.go

@@ -10,7 +10,7 @@ import (
 )
 
 type ModelFilter struct {
-    Id          string
+	Id          string
 	Name        string
 	Registrant  string //name of the registrant for a given model
 	DisplayName string //If Name is already passed, avoid passing Display name unless greedy=true, else the filter will translate to an AND returning only the models where name and display name match exactly. Ignore, if this behavior is expected.
@@ -49,18 +49,42 @@ func countUniqueModels(models []v1beta1.Model) int {
 }
 
 func (mf *ModelFilter) GetById(db *database.Handler) (entity.Entity, error) {
-    m := &v1beta1.Model{}
-    err := db.First(m, "id = ?", mf.Id).Error
+	m := &v1beta1.Model{}
 
+	// Retrieve the model by ID
+	err := db.First(m, "id = ?", mf.Id).Error
 	if err != nil {
 		return nil, registry.ErrGetById(err, mf.Id)
 	}
-    return  m, err
-}
 
+	// Include components if requested
+	if mf.Components {
+		var components []v1beta1.ComponentDefinition
+		componentFinder := db.Model(&v1beta1.ComponentDefinition{}).
+			Select("component_definition_dbs.id, component_definition_dbs.component, component_definition_dbs.display_name, component_definition_dbs.metadata, component_definition_dbs.schema_version, component_definition_dbs.version").
+			Where("component_definition_dbs.model_id = ?", m.ID)
+		if err := componentFinder.Scan(&components).Error; err != nil {
+			return nil, err
+		}
+		m.Components = components
+	}
 
-func (mf *ModelFilter) Get(db *database.Handler) ([]entity.Entity, int64, int, error) {
+	// Include relationships if requested
+	if mf.Relationships {
+		var relationships []v1alpha2.RelationshipDefinition
+		relationshipFinder := db.Model(&v1alpha2.RelationshipDefinition{}).
+			Select("relationship_definition_dbs.*").
+			Where("relationship_definition_dbs.model_id = ?", m.ID)
+		if err := relationshipFinder.Scan(&relationships).Error; err != nil {
+			return nil, err
+		}
+		m.Relationships = relationships
+	}
+
+	return m, nil
+}
 
+func (mf *ModelFilter) Get(db *database.Handler) ([]entity.Entity, int64, int, error) {
 	var modelWithCategories []v1beta1.Model
 
 	finder := db.Model(&v1beta1.Model{}).Preload("Category").Preload("Registrant").
@@ -129,8 +153,8 @@ func (mf *ModelFilter) Get(db *database.Handler) ([]entity.Entity, int64, int, e
 	}
 
 	status := "enabled"
-	
-	if mf.Status != ""  {
+
+	if mf.Status != "" {
 		status = mf.Status
 	}
 
@@ -153,7 +177,7 @@ func (mf *ModelFilter) Get(db *database.Handler) ([]entity.Entity, int64, int, e
 		if includeComponents {
 			var components []v1beta1.ComponentDefinition
 			finder := db.Model(&v1beta1.ComponentDefinition{}).
-				Select("component_definition_dbs.id, component_definition_dbs.component, component_definition_dbs.display_name, component_definition_dbs.metadata").
+				Select("component_definition_dbs.id, component_definition_dbs.component, component_definition_dbs.display_name, component_definition_dbs.metadata, component_definition_dbs.schema_version, component_definition_dbs.version").
 				Where("component_definition_dbs.model_id = ?", _modelDB.ID)
 			if err := finder.Scan(&components).Error; err != nil {
 				return nil, 0, 0, err

models/oci/error.go

@@ -22,12 +22,24 @@ var (
 	ErrAddLayerCode                 = "meshkit-11247"
 	ErrTaggingPackageCode           = "meshkit-11248"
 	ErrPushingPackageCode           = "meshkit-11249"
+	ErrSeekFailedCode               = "replace_me"
+	ErrCreateLayerCode              = "replace_me"
+	ErrSavingImageCode              = "replace_me"
 )
 
 func ErrAppendingLayer(err error) error {
 	return errors.New(ErrAppendingLayerCode, errors.Alert, []string{"appending content to artifact failed"}, []string{err.Error()}, []string{"layer is not compatible with the base image"}, []string{"Try using a different base image", "use a different media type for the layer"})
 }
-
+func ErrSavingImage(err error) error {
+	return errors.New(
+		ErrSavingImageCode,
+		errors.Alert,
+		[]string{"Failed to save image"},
+		[]string{"An error occurred while saving the image."},
+		[]string{"Possible issues with file system, insufficient disk space, , other IO errors."},
+		[]string{"Check the file system permissions and available disk space.", "Ensure the file path is correct and accessible.", "Check for any underlying IO errors."},
+	)
+}
 func ErrReadingFile(err error) error {
 	return errors.New(ErrReadingFileCode, errors.Alert, []string{"reading file failed"}, []string{err.Error()}, []string{"failed to read the file", "Insufficient permissions"}, []string{"Try using a different file", "check if appropriate read permissions are given to the file"})
 }
@@ -43,7 +55,16 @@ func ErrGettingLayer(err error) error {
 func ErrCompressingLayer(err error) error {
 	return errors.New(ErrCompressingLayerCode, errors.Alert, []string{"compressing layer failed"}, []string{err.Error()}, []string{"failed to compress the layer"}, []string{"Try using a different layer", "check if layers are compatible with the base image"})
 }
-
+func ErrCreateLayer(err error) error {
+	return errors.New(
+		ErrCreateLayerCode,
+		errors.Alert,
+		[]string{"Failed to create layer"},
+		[]string{"An error occurred while creating the layer for the image."},
+		[]string{"Possible issues with file system, incorrect layer type, or other IO errors."},
+		[]string{"Check the file system permissions and paths.", "Ensure the layer type is correct.", "Check for any underlying IO errors."},
+	)
+}
 func ErrUnTaringLayer(err error) error {
 	return errors.New(ErrUnTaringLayerCode, errors.Alert, []string{"untaring layer failed"}, []string{err.Error()}, []string{"failed to untar the layer"}, []string{"Try using a different layer", "check if image is not malformed"})
 }
@@ -83,3 +104,6 @@ func ErrTaggingPackage(err error) error {
 func ErrPushingPackage(err error) error {
 	return errors.New(ErrPushingPackageCode, errors.Alert, []string{"pushing package failed"}, []string{err.Error()}, []string{"failed to push the package"}, []string{"Try using a different tag", "check if package is not malformed"})
 }
+func ErrSeekFailed(err error) error {
+	return errors.New(ErrSeekFailedCode, errors.Alert, []string{"Unable to reset the position within the OCI data."}, []string{err.Error()}, []string{"The function attempted to move to the start of the data but failed. This could happen if the data is corrupted or not in the expected format."}, []string{"Ensure the input data is a valid OCI archive and try again. Check if the data is compressed correctly and is not corrupted."})
+}

models/oci/oci.go

@@ -64,7 +64,7 @@ func BuildImage(sourcePath string, opts ...BuildOption) (gcrv1.Image, error) {
 
 	layer, err := createLayer(sourcePath, o.layerType, o.layerOpts)
 	if err != nil {
-		return nil, err
+		return nil, ErrCreateLayer(err)
 	}
 
 	if o.meta.Created == "" {

models/oci/utils.go

@@ -1,10 +1,16 @@
 package oci
 
 import (
+	"bytes"
+	"compress/gzip"
+	"encoding/json"
 	"fmt"
+	"io"
 	"os"
 	"path/filepath"
 
+	archiveTar "archive/tar"
+
 	"github.com/fluxcd/pkg/tar"
 	"github.com/google/go-containerregistry/pkg/crane"
 	gcrv1 "github.com/google/go-containerregistry/pkg/v1"
@@ -28,7 +34,7 @@ func SaveOCIArtifact(img gcrv1.Image, artifactPath, name string) error {
 	repoWithTag := fmt.Sprintf("%s:%s", name, "latest") // TODO: Add support to make this dynamic from user input
 	err := crane.Save(img, repoWithTag, artifactPath)
 	if err != nil {
-		return err
+		return ErrSavingImage(err)
 	}
 
 	return nil
@@ -63,3 +69,83 @@ func UnCompressOCIArtifact(source, destination string) error {
 
 	return nil
 }
+
+// ValidateOCIArtifact validates the OCI artifact tarball using go-containerregistry's validate function
+func ValidateOCIArtifact(tarballPath string) error {
+	img, err := tarball.ImageFromPath(tarballPath, nil)
+	if err != nil {
+		return err
+	}
+
+	return validate.Image(img)
+}
+
+// IsOCIArtifact checks if the tarball is an OCI artifact by looking for manifest.json or index.json
+func IsOCIArtifact(data []byte) bool {
+	reader := bytes.NewReader(data)
+	var tr *archiveTar.Reader
+
+	if gzr, err := gzip.NewReader(reader); err == nil {
+		defer gzr.Close()
+		tr = archiveTar.NewReader(gzr)
+	} else {
+		_, err := reader.Seek(0, io.SeekStart)
+		if err != nil {
+			return false
+		}
+		tr = archiveTar.NewReader(reader)
+	}
+
+	for {
+		header, err := tr.Next()
+		if err == io.EOF {
+			break
+		}
+		if err != nil {
+			return false
+		}
+
+		if header.Name == "manifest.json" || header.Name == "index.json" {
+			return true
+		}
+	}
+
+	return false
+}
+
+func ExtractAndValidateManifest(data []byte) error {
+	reader := bytes.NewReader(data)
+	var tr *archiveTar.Reader
+
+	if gzr, err := gzip.NewReader(reader); err == nil {
+		defer gzr.Close()
+		tr = archiveTar.NewReader(gzr)
+	} else {
+		_, err := reader.Seek(0, io.SeekStart)
+		if err != nil {
+			return ErrSeekFailed(err)
+		}
+		tr = archiveTar.NewReader(reader)
+	}
+
+	for {
+		header, err := tr.Next()
+		if err == io.EOF {
+			break
+		}
+		if err != nil {
+			return err
+		}
+
+		if header.Name == "manifest.json" {
+			var manifest []map[string]interface{}
+			if err := json.NewDecoder(tr).Decode(&manifest); err != nil {
+				return fmt.Errorf("failed to decode manifest.json: %w", err)
+			}
+			fmt.Println("manifest.json is valid:", manifest)
+			return nil
+		}
+	}
+
+	return fmt.Errorf("manifest.json not found in tarball")
+}

schemas/configuration/modelImport.json

@@ -0,0 +1,65 @@
+{
+  "type": "object",
+  "properties": {
+    "uploadType": {
+      "title": "Upload method",
+      "enum": [
+        "File Upload",
+        "URL Import"
+      ],
+      "default": "Select the Upload Method",
+      "x-rjsf-grid-area": "12",
+      "description": "Choose the method you prefer to upload your model file. Select 'File Upload' if you have the file on your local system or 'URL Import' if you have the file hosted online."
+    }
+  },
+  "allOf": [
+    {
+      "if": {
+        "properties": {
+          "uploadType": {
+            "const": "File Upload"
+          }
+        }
+      },
+      "then": {
+        "properties": {
+          "file": {
+            "type": "string",
+            "format": "file",
+            "description": "Browse the filter file from your file system",
+            "x-rjsf-grid-area": "12"
+          }
+        },
+        "required": [
+          "file"
+        ]
+      }
+    },
+    {
+      "if": {
+        "properties": {
+          "uploadType": {
+            "const": "URL Import"
+          }
+        }
+      },
+      "then": {
+        "properties": {
+          "url": {
+            "type": "string",
+            "format": "uri",
+            "title": "URL",
+            "description": "Provide the URL of the design file you want to import. This should be a direct URL to the file, for example: https://raw.github.com/your-design-file.yaml",
+            "x-rjsf-grid-area": "12"
+          }
+        },
+        "required": [
+          "url"
+        ]
+      }
+    }
+  ],
+  "required": [
+    "uploadType"
+  ]
+}

schemas/configuration/uiSchemaModelImport.json

@@ -0,0 +1,6 @@
+{
+    "uploadType": {
+      "ui:widget": "radio"
+    },
+    "ui:order" : [ "uploadType", "file", "url"]
+  }

schemas/schemaProvider.go

@@ -13,6 +13,7 @@ func getSchemaMap() map[string]string {
 		"helmRepo":    "connections/helmConnection/helmRepoConnection.json",
 		"environment": "configuration/environment.json",
 		"workspace":   "configuration/workspace.json",
+		"model":       "configuration/modelImport.json",
 	}
 }
 
@@ -25,6 +26,7 @@ func getUiSchemaMap() map[string]string {
 		"helmRepo":    "connections/helmConnection/uiHelmRepoConnection.json",
 		"environment": "configuration/uiSchemaEnvironment.json",
 		"workspace":   "configuration/uiSchemaWorkspace.json",
+		"model":       "configuration/uiSchemaModelImport.json",
 	}
 }
 

utils/error.go

@@ -60,9 +60,9 @@ var (
 		ErrInvalidSchemaVersionCode,
 		errors.Alert,
 		[]string{"Invalid schema version"},
-		[]string{"The `schemaVersion` key in the JSON file is either empty or has an incorrect value."},
-		[]string{"The JSON file schema is not of type 'relationship' or 'component'.", "The `schemaVersion` key in the JSON should be either `relationships.meshery.io` or `component.meshery.io`."},
-		[]string{"Verify that the `schemaVersion` key in the JSON has the correct value."},
+		[]string{"The `schemaVersion` key is either empty or has an incorrect value."},
+		[]string{"The schema is not of type 'relationship', 'component', 'model' , 'policy'."},
+		[]string{"Verify that `schemaVersion` key should be either `relationships.meshery.io`, `component.meshery.io`, `model.meshery.io` or `policy.meshery.io`."},
 	)
 )