From e98053eaa0231cbe385d0c998b7ab7cf2e3aee09 Mon Sep 17 00:00:00 2001 From: MUzairS15 Date: Thu, 21 Mar 2024 15:50:31 +0530 Subject: [PATCH 01/32] change location for types package Signed-off-by: MUzairS15 --- models/meshmodel/core/policies/data_models.go | 2 ++ models/meshmodel/core/v1alpha1/component.go | 2 +- models/meshmodel/core/v1alpha1/models.go | 2 +- models/meshmodel/core/v1alpha1/policy.go | 2 +- models/meshmodel/core/v1alpha1/relationship.go | 2 +- models/meshmodel/{core/types => entity}/types.go | 2 +- models/meshmodel/registry/registry.go | 2 +- 7 files changed, 8 insertions(+), 6 deletions(-) rename models/meshmodel/{core/types => entity}/types.go (96%) diff --git a/models/meshmodel/core/policies/data_models.go b/models/meshmodel/core/policies/data_models.go index 350207a1..a563cb04 100644 --- a/models/meshmodel/core/policies/data_models.go +++ b/models/meshmodel/core/policies/data_models.go @@ -17,3 +17,5 @@ type NetworkPolicyRegoResponse struct { ServicePodRelationships []RelationObject `json:"service_pod_relationships,omitempty"` ServiceDeploymentRelationships []RelationObject `json:"service_deployment_relationships,omitempty"` } + +// Add response struct based on schema for all relationships evaluations. binding, inventory, network... \ No newline at end of file diff --git a/models/meshmodel/core/v1alpha1/component.go b/models/meshmodel/core/v1alpha1/component.go index 203516a2..b8d28140 100644 --- a/models/meshmodel/core/v1alpha1/component.go +++ b/models/meshmodel/core/v1alpha1/component.go @@ -8,7 +8,7 @@ import ( "github.com/google/uuid" "github.com/layer5io/meshkit/database" - "github.com/layer5io/meshkit/models/meshmodel/core/types" + types "github.com/layer5io/meshkit/models/meshmodel/entity" "github.com/layer5io/meshkit/utils" "gorm.io/gorm/clause" ) diff --git a/models/meshmodel/core/v1alpha1/models.go b/models/meshmodel/core/v1alpha1/models.go index 3e3b58ad..1a1e4598 100644 --- a/models/meshmodel/core/v1alpha1/models.go +++ b/models/meshmodel/core/v1alpha1/models.go @@ -8,7 +8,7 @@ import ( "github.com/google/uuid" "github.com/layer5io/meshkit/database" - "github.com/layer5io/meshkit/models/meshmodel/core/types" + types "github.com/layer5io/meshkit/models/meshmodel/entity" "github.com/layer5io/meshkit/utils" "gorm.io/gorm" ) diff --git a/models/meshmodel/core/v1alpha1/policy.go b/models/meshmodel/core/v1alpha1/policy.go index b85e7446..722720c7 100644 --- a/models/meshmodel/core/v1alpha1/policy.go +++ b/models/meshmodel/core/v1alpha1/policy.go @@ -7,7 +7,7 @@ import ( "github.com/google/uuid" "github.com/layer5io/meshkit/database" - "github.com/layer5io/meshkit/models/meshmodel/core/types" + types "github.com/layer5io/meshkit/models/meshmodel/entity" ) type PolicyDefinition struct { diff --git a/models/meshmodel/core/v1alpha1/relationship.go b/models/meshmodel/core/v1alpha1/relationship.go index 1999fa30..b4b8e60c 100644 --- a/models/meshmodel/core/v1alpha1/relationship.go +++ b/models/meshmodel/core/v1alpha1/relationship.go @@ -7,7 +7,7 @@ import ( "github.com/google/uuid" "github.com/layer5io/meshkit/database" - "github.com/layer5io/meshkit/models/meshmodel/core/types" + types "github.com/layer5io/meshkit/models/meshmodel/entity" "gorm.io/gorm/clause" ) diff --git a/models/meshmodel/core/types/types.go b/models/meshmodel/entity/types.go similarity index 96% rename from models/meshmodel/core/types/types.go rename to models/meshmodel/entity/types.go index ba2a639d..3ea908ea 100644 --- a/models/meshmodel/core/types/types.go +++ b/models/meshmodel/entity/types.go @@ -1,4 +1,4 @@ -package types +package entity type EntityType string diff --git a/models/meshmodel/registry/registry.go b/models/meshmodel/registry/registry.go index cee24460..889f3bd5 100644 --- a/models/meshmodel/registry/registry.go +++ b/models/meshmodel/registry/registry.go @@ -8,7 +8,7 @@ import ( "github.com/google/uuid" "github.com/layer5io/meshkit/database" - "github.com/layer5io/meshkit/models/meshmodel/core/types" + types "github.com/layer5io/meshkit/models/meshmodel/entity" "github.com/layer5io/meshkit/models/meshmodel/core/v1alpha1" "golang.org/x/text/cases" "golang.org/x/text/language" From deff7f670cf99f51263bac14d7d171b3c588d969 Mon Sep 17 00:00:00 2001 From: MUzairS15 Date: Thu, 21 Mar 2024 16:45:59 +0530 Subject: [PATCH 02/32] implement Create function for entities Signed-off-by: MUzairS15 --- models/meshmodel/core/v1beta1/category.go | 80 +++++++++++ models/meshmodel/core/v1beta1/component.go | 100 ++++++++++++++ models/meshmodel/core/v1beta1/host.go | 1 + models/meshmodel/core/v1beta1/models.go | 127 ++++++++++++++++++ models/meshmodel/core/v1beta1/policy.go | 1 + models/meshmodel/core/v1beta1/relationship.go | 114 ++++++++++++++++ models/meshmodel/entity/types.go | 7 + models/meshmodel/registry/registry.go | 1 + utils/cue.go | 2 +- utils/utils.go | 13 ++ 10 files changed, 445 insertions(+), 1 deletion(-) create mode 100644 models/meshmodel/core/v1beta1/category.go create mode 100644 models/meshmodel/core/v1beta1/component.go create mode 100644 models/meshmodel/core/v1beta1/host.go create mode 100644 models/meshmodel/core/v1beta1/models.go create mode 100644 models/meshmodel/core/v1beta1/policy.go create mode 100644 models/meshmodel/core/v1beta1/relationship.go diff --git a/models/meshmodel/core/v1beta1/category.go b/models/meshmodel/core/v1beta1/category.go new file mode 100644 index 00000000..aed56851 --- /dev/null +++ b/models/meshmodel/core/v1beta1/category.go @@ -0,0 +1,80 @@ +package v1beta1 + +import ( + "encoding/json" + "sync" + + "github.com/google/uuid" + "github.com/layer5io/meshkit/database" + "gorm.io/gorm" +) + +var categoryCreationLock sync.Mutex //Each model will perform a check and if the category already doesn't exist, it will create a category. This lock will make sure that there are no race conditions. + +// swagger:response Category +type Category struct { + ID uuid.UUID `json:"-" yaml:"-"` + Name string `json:"name"` + Metadata map[string]interface{} `json:"metadata" yaml:"metadata"` +} + +type CategoryDB struct { + ID uuid.UUID `json:"-"` + Name string `json:"categoryName" gorm:"categoryName"` + Metadata []byte `json:"categoryMetadata" gorm:"categoryMetadata"` +} + +type CategoryFilter struct { + Name string + OrderOn string + Greedy bool + Sort string //asc or desc. Default behavior is asc + Limit int //If 0 or unspecified then all records are returned and limit is not used + Offset int +} +// "Uncategorized" is assigned when Category is empty in the component definitions. +const DefaultCategory = "Uncategorized" + + +func(cat *Category) Create(db *database.Handler) (uuid.UUID, error) { + if cat.Name == "" { + cat.Name = DefaultCategory + } + byt, err := json.Marshal(cat) + if err != nil { + return uuid.UUID{}, err + } + catID := uuid.NewSHA1(uuid.UUID{}, byt) + var category CategoryDB + categoryCreationLock.Lock() + defer categoryCreationLock.Unlock() + err = db.First(&category, "id = ?", catID).Error + if err != nil && err != gorm.ErrRecordNotFound { + return uuid.UUID{}, err + } + if err == gorm.ErrRecordNotFound { //The category is already not present and needs to be inserted + cat.ID = catID + catdb := cat.GetCategoryDB(db) + err = db.Create(&catdb).Error + if err != nil { + return uuid.UUID{}, err + } + return catdb.ID, nil + } + return category.ID, nil +} + +func (c *Category) GetCategoryDB(db *database.Handler) (catdb CategoryDB) { + catdb.ID = c.ID + catdb.Name = c.Name + catdb.Metadata, _ = json.Marshal(c.Metadata) + return +} + +func (cdb *CategoryDB) GetCategory(db *database.Handler) (cat Category) { + cat.ID = cdb.ID + cat.Name = cdb.Name + _ = json.Unmarshal(cdb.Metadata, &cat.Metadata) + return +} + diff --git a/models/meshmodel/core/v1beta1/component.go b/models/meshmodel/core/v1beta1/component.go new file mode 100644 index 00000000..7a09ff31 --- /dev/null +++ b/models/meshmodel/core/v1beta1/component.go @@ -0,0 +1,100 @@ +package v1beta1 + +import ( + "encoding/json" + "time" + + "github.com/layer5io/meshkit/database" + types "github.com/layer5io/meshkit/models/meshmodel/entity" + "github.com/layer5io/meshkit/utils" + + "github.com/google/uuid" +) + +type TypeMeta struct { + Kind string `json:"kind,omitempty" yaml:"kind"` + APIVersion string `json:"apiVersion,omitempty" yaml:"apiVersion"` +} +type ComponentFormat string + +const ( + JSON ComponentFormat = "JSON" + YAML ComponentFormat = "YAML" + CUE ComponentFormat = "CUE" +) + +// swagger:response ComponentDefinition +// use NewComponent function for instantiating +type ComponentDefinition struct { + ID uuid.UUID `json:"id,omitempty"` + TypeMeta + DisplayName string `json:"displayName" gorm:"displayName"` + Format ComponentFormat `json:"format" yaml:"format"` + HostName string `json:"hostname,omitempty"` + HostID uuid.UUID `json:"hostID,omitempty"` + DisplayHostName string `json:"displayhostname,omitempty"` + Metadata map[string]interface{} `json:"metadata" yaml:"metadata"` + Model Model `json:"model"` + Schema string `json:"schema,omitempty" yaml:"schema"` + CreatedAt time.Time `json:"-"` + UpdatedAt time.Time `json:"-"` +} +type ComponentDefinitionDB struct { + ID uuid.UUID `json:"id"` + ModelID uuid.UUID `json:"-" gorm:"index:idx_component_definition_dbs_model_id,column:modelID"` + TypeMeta + DisplayName string `json:"displayName" gorm:"displayName"` + Format ComponentFormat `json:"format" yaml:"format"` + Metadata []byte `json:"metadata" yaml:"metadata"` + Schema string `json:"schema,omitempty" yaml:"schema"` + CreatedAt time.Time `json:"-"` + UpdatedAt time.Time `json:"-"` +} + +func (c ComponentDefinition) Type() types.EntityType { + return types.ComponentDefinition +} + +func (c ComponentDefinition) GetID() uuid.UUID { + return c.ID +} + +func (c *ComponentDefinition) Create(db *database.Handler) (uuid.UUID, error) { + c.ID = uuid.New() + mid, err := c.Model.Create(db) + if err != nil { + return uuid.UUID{}, err + } + + if !utils.IsSchemaEmpty(c.Schema) { + c.Metadata["hasInvalidSchema"] = true + } + cdb := c.GetComponentDefinitionDB() + cdb.ModelID = mid + err = db.Create(&cdb).Error + return c.ID, err +} + +func (c *ComponentDefinition) GetComponentDefinitionDB() (cmd ComponentDefinitionDB) { + cmd.ID = c.ID + cmd.TypeMeta = c.TypeMeta + cmd.Format = c.Format + cmd.Metadata, _ = json.Marshal(c.Metadata) + cmd.DisplayName = c.DisplayName + cmd.Schema = c.Schema + return +} + +func (cmd *ComponentDefinitionDB) GetComponentDefinition(model Model) (c ComponentDefinition) { + c.ID = cmd.ID + c.TypeMeta = cmd.TypeMeta + c.Format = cmd.Format + c.DisplayName = cmd.DisplayName + if c.Metadata == nil { + c.Metadata = make(map[string]interface{}) + } + _ = json.Unmarshal(cmd.Metadata, &c.Metadata) + c.Schema = cmd.Schema + c.Model = model + return +} diff --git a/models/meshmodel/core/v1beta1/host.go b/models/meshmodel/core/v1beta1/host.go new file mode 100644 index 00000000..fd8416ce --- /dev/null +++ b/models/meshmodel/core/v1beta1/host.go @@ -0,0 +1 @@ +package v1beta1 \ No newline at end of file diff --git a/models/meshmodel/core/v1beta1/models.go b/models/meshmodel/core/v1beta1/models.go new file mode 100644 index 00000000..21b95421 --- /dev/null +++ b/models/meshmodel/core/v1beta1/models.go @@ -0,0 +1,127 @@ +package v1beta1 + +import ( + "encoding/json" + "fmt" + "sync" + + "github.com/google/uuid" + "github.com/layer5io/meshkit/database" + types "github.com/layer5io/meshkit/models/meshmodel/entity" + "gorm.io/gorm" +) + +var modelCreationLock sync.Mutex //Each component/relationship will perform a check and if the model already doesn't exist, it will create a model. This lock will make sure that there are no race conditions. + +type ModelStatus string + +type ModelFilter struct { + Name string + Registrant string //name of the registrant for a given model + DisplayName string //If Name is already passed, avoid passing Display name unless greedy=true, else the filter will translate to an AND returning only the models where name and display name match exactly. Ignore, if this behavior is expected. + Greedy bool //when set to true - instead of an exact match, name will be prefix matched. Also an OR will be performed of name and display_name + Version string + Category string + OrderOn string + Sort string //asc or desc. Default behavior is asc + Limit int //If 0 or unspecified then all records are returned and limit is not used + Offset int + Annotations string //When this query parameter is "true", only models with the "isAnnotation" property set to true are returned. When this query parameter is "false", all models except those considered to be annotation models are returned. Any other value of the query parameter results in both annoations as well as non-annotation models being returned. + + // When these are set to true, we also retrieve components/relationships associated with the model. + Components bool + Relationships bool + Status string +} + +// swagger:response Model +type Model struct { + ID uuid.UUID `json:"id,omitempty" yaml:"-"` + Name string `json:"name"` + Version string `json:"version"` + DisplayName string `json:"displayName" gorm:"modelDisplayName"` + Status ModelStatus `json:"status" gorm:"status"` + HostName string `json:"hostname,omitempty"` + HostID uuid.UUID `json:"hostID,omitempty"` + DisplayHostName string `json:"displayhostname,omitempty"` + Category Category `json:"category"` + Metadata map[string]interface{} `json:"metadata" yaml:"modelMetadata"` + // Components []ComponentDefinitionDB `json:"components"` + // Relationships []RelationshipDefinitionDB `json:"relationships"` +} + +type ModelDB struct { + ID uuid.UUID `json:"id"` + CategoryID uuid.UUID `json:"-" gorm:"categoryID"` + Name string `json:"modelName" gorm:"modelName"` + Version string `json:"version"` + DisplayName string `json:"modelDisplayName" gorm:"modelDisplayName"` + SubCategory string `json:"subCategory" gorm:"subCategory"` + Metadata []byte `json:"modelMetadata" gorm:"modelMetadata"` + Status ModelStatus `json:"status" gorm:"status"` +} + +func (m Model) Type() types.EntityType { + return types.Model +} +func (m Model) GetID() uuid.UUID { + return m.ID +} + +func (m *Model) Create(db *database.Handler) (uuid.UUID, error) { + byt, err := json.Marshal(m) + if err != nil { + return uuid.UUID{}, err + } + modelID := uuid.NewSHA1(uuid.UUID{}, byt) + var model ModelDB + if m.Name == "" { + return uuid.UUID{}, fmt.Errorf("empty or invalid model name passed") + } + modelCreationLock.Lock() + defer modelCreationLock.Unlock() + err = db.First(&model, "id = ?", modelID).Error + if err != nil && err != gorm.ErrRecordNotFound { + return uuid.UUID{}, err + } + if err == gorm.ErrRecordNotFound { //The model is already not present and needs to be inserted + id, err := m.Category.Create(db) + if err != nil { + return uuid.UUID{}, err + } + m.ID = modelID + mdb := m.GetModelDB() + mdb.CategoryID = id + mdb.Status = "registered" + err = db.Create(&mdb).Error + if err != nil { + return uuid.UUID{}, err + } + return mdb.ID, nil + } + return model.ID, nil +} + + +func (c *Model) GetModelDB() (cmd ModelDB) { + cmd.ID = c.ID + cmd.DisplayName = c.DisplayName + cmd.Status = c.Status + cmd.Name = c.Name + cmd.Version = c.Version + cmd.Metadata, _ = json.Marshal(c.Metadata) + return +} + +func (cmd *ModelDB) GetModel(cat Category) (c Model) { + c.ID = cmd.ID + c.Category = cat + c.DisplayName = cmd.DisplayName + c.Status = cmd.Status + c.Name = cmd.Name + c.Version = cmd.Version + // c.Components = make([]ComponentDefinitionDB, 0) + // c.Relationships = make([]RelationshipDefinitionDB, 0) + _ = json.Unmarshal(cmd.Metadata, &c.Metadata) + return +} diff --git a/models/meshmodel/core/v1beta1/policy.go b/models/meshmodel/core/v1beta1/policy.go new file mode 100644 index 00000000..fd8416ce --- /dev/null +++ b/models/meshmodel/core/v1beta1/policy.go @@ -0,0 +1 @@ +package v1beta1 \ No newline at end of file diff --git a/models/meshmodel/core/v1beta1/relationship.go b/models/meshmodel/core/v1beta1/relationship.go new file mode 100644 index 00000000..c54c7541 --- /dev/null +++ b/models/meshmodel/core/v1beta1/relationship.go @@ -0,0 +1,114 @@ +package v1beta1 + +import ( + "encoding/json" + "time" + + "github.com/google/uuid" + "github.com/layer5io/meshkit/database" + types "github.com/layer5io/meshkit/models/meshmodel/entity" +) + +// Remove additional DB structs from all entites they are used just for marshaling & unmarshlling and tarcking modelID vs model(modelID when referenced in DB and full model when sending data, use foregin key references tag of gorm for that as used in MeshSync tables) when saving retrivng from database, insted use gorm' serailization tag as used in events. +type RelationshipDefinition struct { + ID uuid.UUID `json:"id"` + TypeMeta + Model Model `json:"model"` + HostName string `json:"hostname"` + HostID uuid.UUID `json:"hostID"` + DisplayHostName string `json:"displayhostname"` + Metadata map[string]interface{} `json:"metadata" yaml:"metadata"` + // The property has been named RelationshipType instead of Type to avoid collision from Type() function, which enables support for dynamic type. + // Though, the column name and the json representation is "type". + RelationshipType string `json:"type" yaml:"type" gorm:"type"` + SubType string `json:"subType" yaml:"subType" gorm:"subType"` + EvaluationQuery string `json:"evaluationQuery" yaml:"evaluationQuery" gorm:"evaluationQuery"` + Selectors []map[string]interface{} `json:"selectors" yaml:"selectors"` + CreatedAt time.Time `json:"-"` + UpdatedAt time.Time `json:"-"` +} + +type RelationshipDefinitionDB struct { + ID uuid.UUID `json:"id"` + ModelID uuid.UUID `json:"-" gorm:"index:idx_relationship_definition_dbs_model_id,column:modelID"` + TypeMeta + Metadata []byte `json:"metadata" yaml:"metadata"` + // The property has been named RelationshipType instead of Type to avoid collision from Type() function, which enables support for dynamic type. + // Though, the column name and the json representation is "type". + RelationshipType string `json:"type" yaml:"type" gorm:"type"` + SubType string `json:"subType" yaml:"subType"` + EvaluationQuery string `json:"evaluationQuery" yaml:"evaluationQuery" gorm:"evaluationQuery"` + Selectors []byte `json:"selectors" yaml:"selectors"` + CreatedAt time.Time `json:"-"` + UpdatedAt time.Time `json:"-"` +} + +// For now, only filtering by Kind and SubType are allowed. +// In the future, we will add support to query using `selectors` (using CUE) +// TODO: Add support for Model +type RelationshipFilter struct { + Kind string + Greedy bool //when set to true - instead of an exact match, kind will be prefix matched + SubType string + RelationshipType string + Version string + ModelName string + OrderOn string + Sort string //asc or desc. Default behavior is asc + Limit int //If 0 or unspecified then all records are returned and limit is not used + Offset int +} + +func (r RelationshipDefinition) Type() types.EntityType { + return types.RelationshipDefinition +} +func (r RelationshipDefinition) GetID() uuid.UUID { + return r.ID +} + +func (r *RelationshipDefinition) Create(db *database.Handler) (uuid.UUID, error) { + r.ID = uuid.New() + mid, err := r.Model.Create(db) + if err != nil { + return uuid.UUID{}, err + } + rdb := r.GetRelationshipDefinitionDB() + rdb.ModelID = mid + err = db.Create(&rdb).Error + if err != nil { + return uuid.UUID{}, err + } + return r.ID, err +} + +func (r *RelationshipDefinition) GetRelationshipDefinitionDB() (rdb RelationshipDefinitionDB) { + rdb.ID = r.ID + rdb.TypeMeta = r.TypeMeta + rdb.Metadata, _ = json.Marshal(r.Metadata) + rdb.Selectors, _ = json.Marshal(r.Selectors) + rdb.Kind = r.Kind + rdb.RelationshipType = r.RelationshipType + rdb.SubType = r.SubType + rdb.ModelID = r.Model.ID + rdb.EvaluationQuery = r.EvaluationQuery + return +} + +func (rdb *RelationshipDefinitionDB) GetRelationshipDefinition(m Model) (r RelationshipDefinition) { + r.ID = rdb.ID + r.TypeMeta = rdb.TypeMeta + if r.Metadata == nil { + r.Metadata = make(map[string]interface{}) + } + _ = json.Unmarshal(rdb.Metadata, &r.Metadata) + if r.Selectors == nil { + r.Selectors = []map[string]interface{}{} + } + _ = json.Unmarshal(rdb.Selectors, &r.Selectors) + r.RelationshipType = rdb.RelationshipType + r.SubType = rdb.SubType + r.Kind = rdb.Kind + r.Model = m + r.EvaluationQuery = rdb.EvaluationQuery + return +} diff --git a/models/meshmodel/entity/types.go b/models/meshmodel/entity/types.go index 3ea908ea..8764c2ca 100644 --- a/models/meshmodel/entity/types.go +++ b/models/meshmodel/entity/types.go @@ -1,5 +1,7 @@ package entity +import "github.com/google/uuid" + type EntityType string const ( @@ -13,3 +15,8 @@ const ( type Filter interface { Create(map[string]interface{}) } + +type Entity interface { + Create() (entityID uuid.UUID, err error) + Get() (entityID uuid.UUID, err error) +} diff --git a/models/meshmodel/registry/registry.go b/models/meshmodel/registry/registry.go index 889f3bd5..ebdf49c6 100644 --- a/models/meshmodel/registry/registry.go +++ b/models/meshmodel/registry/registry.go @@ -281,6 +281,7 @@ func (rm *RegistryManager) GetRegistrants(f *v1alpha1.HostFilter) ([]v1alpha1.Me return response, totalcount, nil } + func (rm *RegistryManager) GetEntities(f types.Filter) ([]Entity, *int64, *int) { switch filter := f.(type) { case *v1alpha1.ComponentFilter: diff --git a/utils/cue.go b/utils/cue.go index 64f0db4d..2e90f863 100644 --- a/utils/cue.go +++ b/utils/cue.go @@ -112,4 +112,4 @@ func Lookup(rootVal cue.Value, path string) (cue.Value, error) { } return res.Value(), nil -} +} \ No newline at end of file diff --git a/utils/utils.go b/utils/utils.go index c960a5fc..855f9874 100644 --- a/utils/utils.go +++ b/utils/utils.go @@ -397,3 +397,16 @@ func IsInterfaceNil(val interface{}) bool { return false } + +func IsSchemaEmpty(schema string) (valid bool) { + if schema == "" { + return + } + m := make(map[string]interface{}) + _ = json.Unmarshal([]byte(schema), &m) + if m["properties"] == nil { + return + } + valid = true + return +} \ No newline at end of file From 1735863aff81095444683505620f7819be21fbef Mon Sep 17 00:00:00 2001 From: MUzairS15 Date: Thu, 21 Mar 2024 17:00:25 +0530 Subject: [PATCH 03/32] add registrant/host as first class attribute for models Signed-off-by: MUzairS15 --- models/meshmodel/core/v1beta1/models.go | 34 ++++--- models/meshmodel/registry/hostv1beta1.go | 121 +++++++++++++++++++++++ 2 files changed, 141 insertions(+), 14 deletions(-) create mode 100644 models/meshmodel/registry/hostv1beta1.go diff --git a/models/meshmodel/core/v1beta1/models.go b/models/meshmodel/core/v1beta1/models.go index 21b95421..32e662d7 100644 --- a/models/meshmodel/core/v1beta1/models.go +++ b/models/meshmodel/core/v1beta1/models.go @@ -8,6 +8,7 @@ import ( "github.com/google/uuid" "github.com/layer5io/meshkit/database" types "github.com/layer5io/meshkit/models/meshmodel/entity" + "github.com/layer5io/meshkit/models/meshmodel/registry" "gorm.io/gorm" ) @@ -38,12 +39,10 @@ type ModelFilter struct { type Model struct { ID uuid.UUID `json:"id,omitempty" yaml:"-"` Name string `json:"name"` - Version string `json:"version"` DisplayName string `json:"displayName" gorm:"modelDisplayName"` + Version string `json:"version"` Status ModelStatus `json:"status" gorm:"status"` - HostName string `json:"hostname,omitempty"` - HostID uuid.UUID `json:"hostID,omitempty"` - DisplayHostName string `json:"displayhostname,omitempty"` + Registrant registry.Hostv1beta1 `json:"registrant" gorm:"registrant"` // to be Connection Category Category `json:"category"` Metadata map[string]interface{} `json:"metadata" yaml:"modelMetadata"` // Components []ComponentDefinitionDB `json:"components"` @@ -51,14 +50,15 @@ type Model struct { } type ModelDB struct { - ID uuid.UUID `json:"id"` - CategoryID uuid.UUID `json:"-" gorm:"categoryID"` - Name string `json:"modelName" gorm:"modelName"` - Version string `json:"version"` - DisplayName string `json:"modelDisplayName" gorm:"modelDisplayName"` - SubCategory string `json:"subCategory" gorm:"subCategory"` - Metadata []byte `json:"modelMetadata" gorm:"modelMetadata"` - Status ModelStatus `json:"status" gorm:"status"` + ID uuid.UUID `json:"id"` + CategoryID uuid.UUID `json:"-" gorm:"categoryID"` + RegistrantID uuid.UUID `json:"hostID" gorm:"hostID"` + Name string `json:"modelName" gorm:"modelName"` + Version string `json:"version"` + DisplayName string `json:"modelDisplayName" gorm:"modelDisplayName"` + SubCategory string `json:"subCategory" gorm:"subCategory"` + Metadata []byte `json:"modelMetadata" gorm:"modelMetadata"` + Status ModelStatus `json:"status" gorm:"status"` } func (m Model) Type() types.EntityType { @@ -69,6 +69,12 @@ func (m Model) GetID() uuid.UUID { } func (m *Model) Create(db *database.Handler) (uuid.UUID, error) { + + hostID, err := m.Registrant.Create(db) + if err != nil { + return uuid.UUID{}, err + } + byt, err := json.Marshal(m) if err != nil { return uuid.UUID{}, err @@ -80,7 +86,7 @@ func (m *Model) Create(db *database.Handler) (uuid.UUID, error) { } modelCreationLock.Lock() defer modelCreationLock.Unlock() - err = db.First(&model, "id = ?", modelID).Error + err = db.First(&model, "id = ? and hostID = ?", modelID, hostID).Error if err != nil && err != gorm.ErrRecordNotFound { return uuid.UUID{}, err } @@ -92,6 +98,7 @@ func (m *Model) Create(db *database.Handler) (uuid.UUID, error) { m.ID = modelID mdb := m.GetModelDB() mdb.CategoryID = id + mdb.RegistrantID = hostID mdb.Status = "registered" err = db.Create(&mdb).Error if err != nil { @@ -102,7 +109,6 @@ func (m *Model) Create(db *database.Handler) (uuid.UUID, error) { return model.ID, nil } - func (c *Model) GetModelDB() (cmd ModelDB) { cmd.ID = c.ID cmd.DisplayName = c.DisplayName diff --git a/models/meshmodel/registry/hostv1beta1.go b/models/meshmodel/registry/hostv1beta1.go new file mode 100644 index 00000000..dde23631 --- /dev/null +++ b/models/meshmodel/registry/hostv1beta1.go @@ -0,0 +1,121 @@ +package registry + +import ( + "encoding/json" + "fmt" + "sync" + "time" + + "github.com/google/uuid" + "github.com/layer5io/meshkit/database" + "github.com/layer5io/meshkit/models/oam/core/v1alpha1" + "github.com/layer5io/meshkit/utils/kubernetes" + "gorm.io/gorm" +) + +var hostv1beta1CreationLock sync.Mutex //Each entity will perform a check and if the host already doesn't exist, it will create a host. This lock will make sure that there are no race conditions. + +type Hostv1beta1 struct { + ID uuid.UUID `json:"-"` + Hostname string + Port int + Metadata string + CreatedAt time.Time `json:"-"` + UpdatedAt time.Time `json:"-"` + IHost IHost `gorm:"-"` +} + +func(h *Hostv1beta1) Create(db *database.Handler) (uuid.UUID, error) { + byt, err := json.Marshal(h) + if err != nil { + return uuid.UUID{}, err + } + hID := uuid.NewSHA1(uuid.UUID{}, byt) + var host Host + hostv1beta1CreationLock.Lock() + defer hostv1beta1CreationLock.Unlock() + err = db.First(&host, "id = ?", hID).Error // check if the host already exists + if err != nil && err != gorm.ErrRecordNotFound { + return uuid.UUID{}, err + } + + // if not exists then create a new host and return the id + if err == gorm.ErrRecordNotFound { + h.ID = hID + err = db.Create(&h).Error + if err != nil { + return uuid.UUID{}, err + } + return h.ID, nil + } + + // else return the id of the existing host + return host.ID, nil +} + +func (h *Hostv1beta1) AfterFind(tx *gorm.DB) error { + switch h.Hostname { + case "artifacthub": + h.IHost = ArtifactHub{} + case "kubernetes": + h.IHost = Kubernetes{} + default: // do nothing if the host is not pre-unknown. Currently adapters fall into this case. + return nil + } + return nil +} + +// Each host from where meshmodels can be generated needs to implement this interface +// HandleDependents, contains host specific logic for provisioning required CRDs/operators for corresponding components. +type _IHost interface { + HandleDependents(comp v1alpha1.Component, kc *kubernetes.Client, isDeploy bool) (string, error) + String() string +} + +type _ArtifactHub struct{} + +func (ah _ArtifactHub) HandleDependents(comp v1alpha1.Component, kc *kubernetes.Client, isDeploy bool) (summary string, err error) { + source_uri := comp.Annotations[fmt.Sprintf("%s.model.source_uri", v1alpha1.MesheryAnnotationPrefix)] + act := kubernetes.UNINSTALL + if isDeploy { + act = kubernetes.INSTALL + } + + if source_uri != "" { + err = kc.ApplyHelmChart(kubernetes.ApplyHelmChartConfig{ + URL: source_uri, + Namespace: comp.Namespace, + CreateNamespace: true, + Action: act, + SkipUpgradeIfInstalled: true, + }) + if err != nil { + if !isDeploy { + summary = fmt.Sprintf("error undeploying dependent helm chart for %s, please proceed with manual uninstall or try again", comp.Name) + } else { + summary = fmt.Sprintf("error deploying dependent helm chart for %s, please procced with manual install or try again", comp.Name) + } + } else { + if !isDeploy { + summary = fmt.Sprintf("Undeployed dependent helm chart for %s", comp.Name) + } else { + summary = fmt.Sprintf("Deployed dependent helm chart for %s", comp.Name) + } + } + } + return summary, err +} + +func (ah _ArtifactHub) String() string { + return "artifacthub" +} + +type _Kubernetes struct{} + +func (k _Kubernetes) HandleDependents(comp v1alpha1.Component, kc *kubernetes.Client, isDeploy bool) (summary string, err error) { + return summary, err +} + +func (k _Kubernetes) String() string { + return "kubernetes" +} From 02ec85d0709077e5aacf1af81bf6cfdb420a6497 Mon Sep 17 00:00:00 2001 From: MUzairS15 Date: Thu, 21 Mar 2024 17:04:40 +0530 Subject: [PATCH 04/32] complete v1beta1 for model Signed-off-by: MUzairS15 --- models/meshmodel/core/v1beta1/models.go | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/models/meshmodel/core/v1beta1/models.go b/models/meshmodel/core/v1beta1/models.go index 32e662d7..ef270f0e 100644 --- a/models/meshmodel/core/v1beta1/models.go +++ b/models/meshmodel/core/v1beta1/models.go @@ -37,14 +37,15 @@ type ModelFilter struct { // swagger:response Model type Model struct { - ID uuid.UUID `json:"id,omitempty" yaml:"-"` - Name string `json:"name"` - DisplayName string `json:"displayName" gorm:"modelDisplayName"` - Version string `json:"version"` - Status ModelStatus `json:"status" gorm:"status"` - Registrant registry.Hostv1beta1 `json:"registrant" gorm:"registrant"` // to be Connection - Category Category `json:"category"` - Metadata map[string]interface{} `json:"metadata" yaml:"modelMetadata"` + ID uuid.UUID `json:"id,omitempty" yaml:"-"` + Name string `json:"name"` + DisplayName string `json:"displayName" gorm:"modelDisplayName"` + Version string `json:"version"` + Status ModelStatus `json:"status" gorm:"status"` + Registrant registry.Hostv1beta1 `json:"registrant" gorm:"registrant"` // to be Connection + Category Category `json:"category"` + SubCategory string `json:"subCategory" gorm:"subCategory"` + Metadata map[string]interface{} `json:"metadata" yaml:"modelMetadata"` // Components []ComponentDefinitionDB `json:"components"` // Relationships []RelationshipDefinitionDB `json:"relationships"` } From 21b9b467cd5665d68b6e50a1dcfbeeb6d4f5cd64 Mon Sep 17 00:00:00 2001 From: MUzairS15 Date: Thu, 21 Mar 2024 17:13:47 +0530 Subject: [PATCH 05/32] complete v1beta1 for component Signed-off-by: MUzairS15 --- models/meshmodel/core/v1beta1/component.go | 57 ++++++++++++---------- 1 file changed, 30 insertions(+), 27 deletions(-) diff --git a/models/meshmodel/core/v1beta1/component.go b/models/meshmodel/core/v1beta1/component.go index 7a09ff31..4e5a140b 100644 --- a/models/meshmodel/core/v1beta1/component.go +++ b/models/meshmodel/core/v1beta1/component.go @@ -2,7 +2,6 @@ package v1beta1 import ( "encoding/json" - "time" "github.com/layer5io/meshkit/database" types "github.com/layer5io/meshkit/models/meshmodel/entity" @@ -23,32 +22,36 @@ const ( CUE ComponentFormat = "CUE" ) +type entity struct { + TypeMeta + Type string `json:"type,omitempty" yaml:"type"` + SubType string `json:"subType,omitempty" yaml:"subType"` + Schema string `json:"schema,omitempty" yaml:"schema"` +} + // swagger:response ComponentDefinition // use NewComponent function for instantiating type ComponentDefinition struct { - ID uuid.UUID `json:"id,omitempty"` - TypeMeta - DisplayName string `json:"displayName" gorm:"displayName"` - Format ComponentFormat `json:"format" yaml:"format"` - HostName string `json:"hostname,omitempty"` - HostID uuid.UUID `json:"hostID,omitempty"` - DisplayHostName string `json:"displayhostname,omitempty"` - Metadata map[string]interface{} `json:"metadata" yaml:"metadata"` - Model Model `json:"model"` - Schema string `json:"schema,omitempty" yaml:"schema"` - CreatedAt time.Time `json:"-"` - UpdatedAt time.Time `json:"-"` + ID uuid.UUID `json:"id,omitempty"` + SchemaVersion string `json:"schemaVersion,omitempty" yaml:"schemaVersion"` + DefinitionVersion string `json:"definitionVersion,omitempty" yaml:"definitionVersion"` + DisplayName string `json:"displayName" gorm:"displayName"` + Format ComponentFormat `json:"format" yaml:"format"` + Model Model `json:"model"` + Metadata map[string]interface{} `json:"metadata" yaml:"metadata"` + // Rename this here and in schema as well, it creates confustion b/w entities like comps. relationship. model, here entity means Pod/Deployment.... + Entity entity `json:"entity,omitempty" yaml:"entity"` } + type ComponentDefinitionDB struct { - ID uuid.UUID `json:"id"` - ModelID uuid.UUID `json:"-" gorm:"index:idx_component_definition_dbs_model_id,column:modelID"` - TypeMeta - DisplayName string `json:"displayName" gorm:"displayName"` - Format ComponentFormat `json:"format" yaml:"format"` - Metadata []byte `json:"metadata" yaml:"metadata"` - Schema string `json:"schema,omitempty" yaml:"schema"` - CreatedAt time.Time `json:"-"` - UpdatedAt time.Time `json:"-"` + ID uuid.UUID `json:"id"` + SchemaVersion string `json:"schemaVersion,omitempty" yaml:"schemaVersion"` + DefinitionVersion string `json:"definitionVersion,omitempty" yaml:"definitionVersion"` + DisplayName string `json:"displayName" gorm:"displayName"` + Format ComponentFormat `json:"format" yaml:"format"` + ModelID uuid.UUID `json:"-" gorm:"index:idx_component_definition_dbs_model_id,column:modelID"` + Metadata []byte `json:"metadata" yaml:"metadata"` + Entity entity `json:"entity,omitempty" yaml:"entity" gorm:"entity"` } func (c ComponentDefinition) Type() types.EntityType { @@ -66,7 +69,7 @@ func (c *ComponentDefinition) Create(db *database.Handler) (uuid.UUID, error) { return uuid.UUID{}, err } - if !utils.IsSchemaEmpty(c.Schema) { + if !utils.IsSchemaEmpty(c.Entity.Schema) { c.Metadata["hasInvalidSchema"] = true } cdb := c.GetComponentDefinitionDB() @@ -77,24 +80,24 @@ func (c *ComponentDefinition) Create(db *database.Handler) (uuid.UUID, error) { func (c *ComponentDefinition) GetComponentDefinitionDB() (cmd ComponentDefinitionDB) { cmd.ID = c.ID - cmd.TypeMeta = c.TypeMeta + cmd.Entity.TypeMeta = c.Entity.TypeMeta cmd.Format = c.Format cmd.Metadata, _ = json.Marshal(c.Metadata) cmd.DisplayName = c.DisplayName - cmd.Schema = c.Schema + cmd.Entity.Schema = c.Entity.Schema return } func (cmd *ComponentDefinitionDB) GetComponentDefinition(model Model) (c ComponentDefinition) { c.ID = cmd.ID - c.TypeMeta = cmd.TypeMeta + c.Entity.TypeMeta = cmd.Entity.TypeMeta c.Format = cmd.Format c.DisplayName = cmd.DisplayName if c.Metadata == nil { c.Metadata = make(map[string]interface{}) } _ = json.Unmarshal(cmd.Metadata, &c.Metadata) - c.Schema = cmd.Schema + c.Entity.Schema = cmd.Entity.Schema c.Model = model return } From 35d7e3eb398b18ec3691e783ea5c2dc952e384da Mon Sep 17 00:00:00 2001 From: MUzairS15 Date: Fri, 22 Mar 2024 17:04:15 +0530 Subject: [PATCH 06/32] update as per latest changes to schema Signed-off-by: MUzairS15 --- models/meshmodel/core/v1beta1/component.go | 48 ++++++++++--------- models/meshmodel/core/v1beta1/models.go | 16 +++++-- models/meshmodel/core/v1beta1/relationship.go | 43 ++++++++--------- 3 files changed, 59 insertions(+), 48 deletions(-) diff --git a/models/meshmodel/core/v1beta1/component.go b/models/meshmodel/core/v1beta1/component.go index 4e5a140b..61bccced 100644 --- a/models/meshmodel/core/v1beta1/component.go +++ b/models/meshmodel/core/v1beta1/component.go @@ -10,10 +10,16 @@ import ( "github.com/google/uuid" ) +type VersionMeta struct { + SchemaVersion string `json:"schemaVersion,omitempty" yaml:"schemaVersion"` + Version string `json:"version,omitempty" yaml:"version"` +} + type TypeMeta struct { - Kind string `json:"kind,omitempty" yaml:"kind"` - APIVersion string `json:"apiVersion,omitempty" yaml:"apiVersion"` + Kind string `json:"kind,omitempty" yaml:"kind"` + Version string `json:"version,omitempty" yaml:"version"` } + type ComponentFormat string const ( @@ -22,36 +28,32 @@ const ( CUE ComponentFormat = "CUE" ) -type entity struct { +type component struct { TypeMeta - Type string `json:"type,omitempty" yaml:"type"` - SubType string `json:"subType,omitempty" yaml:"subType"` - Schema string `json:"schema,omitempty" yaml:"schema"` + Schema string `json:"schema,omitempty" yaml:"schema"` } // swagger:response ComponentDefinition // use NewComponent function for instantiating type ComponentDefinition struct { - ID uuid.UUID `json:"id,omitempty"` - SchemaVersion string `json:"schemaVersion,omitempty" yaml:"schemaVersion"` - DefinitionVersion string `json:"definitionVersion,omitempty" yaml:"definitionVersion"` - DisplayName string `json:"displayName" gorm:"displayName"` - Format ComponentFormat `json:"format" yaml:"format"` - Model Model `json:"model"` - Metadata map[string]interface{} `json:"metadata" yaml:"metadata"` + ID uuid.UUID `json:"id,omitempty"` + VersionMeta + DisplayName string `json:"displayName" gorm:"displayName"` + Format ComponentFormat `json:"format" yaml:"format"` + Model Model `json:"model"` + Metadata map[string]interface{} `json:"metadata" yaml:"metadata"` // Rename this here and in schema as well, it creates confustion b/w entities like comps. relationship. model, here entity means Pod/Deployment.... - Entity entity `json:"entity,omitempty" yaml:"entity"` + Entity component `json:"entity,omitempty" yaml:"entity"` } type ComponentDefinitionDB struct { - ID uuid.UUID `json:"id"` - SchemaVersion string `json:"schemaVersion,omitempty" yaml:"schemaVersion"` - DefinitionVersion string `json:"definitionVersion,omitempty" yaml:"definitionVersion"` - DisplayName string `json:"displayName" gorm:"displayName"` - Format ComponentFormat `json:"format" yaml:"format"` - ModelID uuid.UUID `json:"-" gorm:"index:idx_component_definition_dbs_model_id,column:modelID"` - Metadata []byte `json:"metadata" yaml:"metadata"` - Entity entity `json:"entity,omitempty" yaml:"entity" gorm:"entity"` + ID uuid.UUID `json:"id"` + VersionMeta + DisplayName string `json:"displayName" gorm:"displayName"` + Format ComponentFormat `json:"format" yaml:"format"` + ModelID uuid.UUID `json:"-" gorm:"index:idx_component_definition_dbs_model_id,column:modelID"` + Metadata []byte `json:"metadata" yaml:"metadata"` + Entity component `json:"entity,omitempty" yaml:"entity" gorm:"entity"` } func (c ComponentDefinition) Type() types.EntityType { @@ -80,6 +82,7 @@ func (c *ComponentDefinition) Create(db *database.Handler) (uuid.UUID, error) { func (c *ComponentDefinition) GetComponentDefinitionDB() (cmd ComponentDefinitionDB) { cmd.ID = c.ID + cmd.VersionMeta = c.VersionMeta cmd.Entity.TypeMeta = c.Entity.TypeMeta cmd.Format = c.Format cmd.Metadata, _ = json.Marshal(c.Metadata) @@ -90,6 +93,7 @@ func (c *ComponentDefinition) GetComponentDefinitionDB() (cmd ComponentDefinitio func (cmd *ComponentDefinitionDB) GetComponentDefinition(model Model) (c ComponentDefinition) { c.ID = cmd.ID + c.VersionMeta = cmd.VersionMeta c.Entity.TypeMeta = cmd.Entity.TypeMeta c.Format = cmd.Format c.DisplayName = cmd.DisplayName diff --git a/models/meshmodel/core/v1beta1/models.go b/models/meshmodel/core/v1beta1/models.go index ef270f0e..ea60be94 100644 --- a/models/meshmodel/core/v1beta1/models.go +++ b/models/meshmodel/core/v1beta1/models.go @@ -35,15 +35,21 @@ type ModelFilter struct { Status string } +type model struct { + Version string `json:"version,omitempty" yaml:"version"` +} + // swagger:response Model type Model struct { - ID uuid.UUID `json:"id,omitempty" yaml:"-"` + ID uuid.UUID `json:"id,omitempty" yaml:"-"` + VersionMeta Name string `json:"name"` DisplayName string `json:"displayName" gorm:"modelDisplayName"` Version string `json:"version"` Status ModelStatus `json:"status" gorm:"status"` Registrant registry.Hostv1beta1 `json:"registrant" gorm:"registrant"` // to be Connection Category Category `json:"category"` + Model model `json:"model,omitempty" gorm:"model"` SubCategory string `json:"subCategory" gorm:"subCategory"` Metadata map[string]interface{} `json:"metadata" yaml:"modelMetadata"` // Components []ComponentDefinitionDB `json:"components"` @@ -51,8 +57,10 @@ type Model struct { } type ModelDB struct { - ID uuid.UUID `json:"id"` - CategoryID uuid.UUID `json:"-" gorm:"categoryID"` + ID uuid.UUID `json:"id"` + VersionMeta + CategoryID uuid.UUID `json:"-" gorm:"categoryID"` + Model model `json:"model,omitempty" gorm:"model"` RegistrantID uuid.UUID `json:"hostID" gorm:"hostID"` Name string `json:"modelName" gorm:"modelName"` Version string `json:"version"` @@ -112,6 +120,7 @@ func (m *Model) Create(db *database.Handler) (uuid.UUID, error) { func (c *Model) GetModelDB() (cmd ModelDB) { cmd.ID = c.ID + cmd.VersionMeta = c.VersionMeta cmd.DisplayName = c.DisplayName cmd.Status = c.Status cmd.Name = c.Name @@ -122,6 +131,7 @@ func (c *Model) GetModelDB() (cmd ModelDB) { func (cmd *ModelDB) GetModel(cat Category) (c Model) { c.ID = cmd.ID + c.VersionMeta = cmd.VersionMeta c.Category = cat c.DisplayName = cmd.DisplayName c.Status = cmd.Status diff --git a/models/meshmodel/core/v1beta1/relationship.go b/models/meshmodel/core/v1beta1/relationship.go index c54c7541..63307db9 100644 --- a/models/meshmodel/core/v1beta1/relationship.go +++ b/models/meshmodel/core/v1beta1/relationship.go @@ -2,7 +2,6 @@ package v1beta1 import ( "encoding/json" - "time" "github.com/google/uuid" "github.com/layer5io/meshkit/database" @@ -12,35 +11,33 @@ import ( // Remove additional DB structs from all entites they are used just for marshaling & unmarshlling and tarcking modelID vs model(modelID when referenced in DB and full model when sending data, use foregin key references tag of gorm for that as used in MeshSync tables) when saving retrivng from database, insted use gorm' serailization tag as used in events. type RelationshipDefinition struct { ID uuid.UUID `json:"id"` - TypeMeta - Model Model `json:"model"` - HostName string `json:"hostname"` - HostID uuid.UUID `json:"hostID"` - DisplayHostName string `json:"displayhostname"` - Metadata map[string]interface{} `json:"metadata" yaml:"metadata"` + VersionMeta + Kind string `json:"kind,omitempty" yaml:"kind"` + RelationshipType string `json:"type" yaml:"type" gorm:"type"` + SubType string `json:"subType" yaml:"subType" gorm:"subType"` + Model Model `json:"model"` + HostName string `json:"hostname"` + HostID uuid.UUID `json:"hostID"` + DisplayHostName string `json:"displayhostname"` + Metadata map[string]interface{} `json:"metadata" yaml:"metadata"` // The property has been named RelationshipType instead of Type to avoid collision from Type() function, which enables support for dynamic type. // Though, the column name and the json representation is "type". - RelationshipType string `json:"type" yaml:"type" gorm:"type"` - SubType string `json:"subType" yaml:"subType" gorm:"subType"` - EvaluationQuery string `json:"evaluationQuery" yaml:"evaluationQuery" gorm:"evaluationQuery"` - Selectors []map[string]interface{} `json:"selectors" yaml:"selectors"` - CreatedAt time.Time `json:"-"` - UpdatedAt time.Time `json:"-"` + EvaluationQuery string `json:"evaluationQuery" yaml:"evaluationQuery" gorm:"evaluationQuery"` + Selectors []map[string]interface{} `json:"selectors" yaml:"selectors"` } type RelationshipDefinitionDB struct { ID uuid.UUID `json:"id"` ModelID uuid.UUID `json:"-" gorm:"index:idx_relationship_definition_dbs_model_id,column:modelID"` - TypeMeta - Metadata []byte `json:"metadata" yaml:"metadata"` + VersionMeta + Kind string `json:"kind,omitempty" yaml:"kind"` + RelationshipType string `json:"type" yaml:"type" gorm:"type"` + SubType string `json:"subType" yaml:"subType"` + Metadata []byte `json:"metadata" yaml:"metadata"` // The property has been named RelationshipType instead of Type to avoid collision from Type() function, which enables support for dynamic type. // Though, the column name and the json representation is "type". - RelationshipType string `json:"type" yaml:"type" gorm:"type"` - SubType string `json:"subType" yaml:"subType"` - EvaluationQuery string `json:"evaluationQuery" yaml:"evaluationQuery" gorm:"evaluationQuery"` - Selectors []byte `json:"selectors" yaml:"selectors"` - CreatedAt time.Time `json:"-"` - UpdatedAt time.Time `json:"-"` + EvaluationQuery string `json:"evaluationQuery" yaml:"evaluationQuery" gorm:"evaluationQuery"` + Selectors []byte `json:"selectors" yaml:"selectors"` } // For now, only filtering by Kind and SubType are allowed. @@ -83,7 +80,7 @@ func (r *RelationshipDefinition) Create(db *database.Handler) (uuid.UUID, error) func (r *RelationshipDefinition) GetRelationshipDefinitionDB() (rdb RelationshipDefinitionDB) { rdb.ID = r.ID - rdb.TypeMeta = r.TypeMeta + rdb.VersionMeta = r.VersionMeta rdb.Metadata, _ = json.Marshal(r.Metadata) rdb.Selectors, _ = json.Marshal(r.Selectors) rdb.Kind = r.Kind @@ -96,7 +93,7 @@ func (r *RelationshipDefinition) GetRelationshipDefinitionDB() (rdb Relationship func (rdb *RelationshipDefinitionDB) GetRelationshipDefinition(m Model) (r RelationshipDefinition) { r.ID = rdb.ID - r.TypeMeta = rdb.TypeMeta + r.VersionMeta = rdb.VersionMeta if r.Metadata == nil { r.Metadata = make(map[string]interface{}) } From 335ca45caf0016d169e926daa9c4f115e630a5ab Mon Sep 17 00:00:00 2001 From: MUzairS15 Date: Mon, 25 Mar 2024 12:43:08 +0530 Subject: [PATCH 07/32] reorder Signed-off-by: MUzairS15 --- .../{v1beta1 => v1alpha2}/relationship.go | 61 +++++++++---------- models/meshmodel/core/v1beta1/component.go | 30 +++++---- models/meshmodel/core/v1beta1/models.go | 40 +++++++----- 3 files changed, 71 insertions(+), 60 deletions(-) rename models/meshmodel/core/{v1beta1 => v1alpha2}/relationship.go (66%) diff --git a/models/meshmodel/core/v1beta1/relationship.go b/models/meshmodel/core/v1alpha2/relationship.go similarity index 66% rename from models/meshmodel/core/v1beta1/relationship.go rename to models/meshmodel/core/v1alpha2/relationship.go index 63307db9..65ce357f 100644 --- a/models/meshmodel/core/v1beta1/relationship.go +++ b/models/meshmodel/core/v1alpha2/relationship.go @@ -1,43 +1,41 @@ -package v1beta1 +package v1alpha2 import ( "encoding/json" "github.com/google/uuid" "github.com/layer5io/meshkit/database" + "github.com/layer5io/meshkit/models/meshmodel/core/v1beta1" types "github.com/layer5io/meshkit/models/meshmodel/entity" ) // Remove additional DB structs from all entites they are used just for marshaling & unmarshlling and tarcking modelID vs model(modelID when referenced in DB and full model when sending data, use foregin key references tag of gorm for that as used in MeshSync tables) when saving retrivng from database, insted use gorm' serailization tag as used in events. type RelationshipDefinition struct { ID uuid.UUID `json:"id"` - VersionMeta - Kind string `json:"kind,omitempty" yaml:"kind"` - RelationshipType string `json:"type" yaml:"type" gorm:"type"` - SubType string `json:"subType" yaml:"subType" gorm:"subType"` - Model Model `json:"model"` - HostName string `json:"hostname"` - HostID uuid.UUID `json:"hostID"` - DisplayHostName string `json:"displayhostname"` - Metadata map[string]interface{} `json:"metadata" yaml:"metadata"` + v1beta1.VersionMeta + Kind string `json:"kind,omitempty" yaml:"kind"` // The property has been named RelationshipType instead of Type to avoid collision from Type() function, which enables support for dynamic type. // Though, the column name and the json representation is "type". - EvaluationQuery string `json:"evaluationQuery" yaml:"evaluationQuery" gorm:"evaluationQuery"` - Selectors []map[string]interface{} `json:"selectors" yaml:"selectors"` + RelationshipType string `json:"type" yaml:"type" gorm:"type"` + SubType string `json:"subType" yaml:"subType" gorm:"subType"` + EvaluationQuery string `json:"evaluationQuery" yaml:"evaluationQuery" gorm:"evaluationQuery"` + Metadata map[string]interface{} `json:"metadata" yaml:"metadata"` + Model v1beta1.Model `json:"model"` + Selectors []map[string]interface{} `json:"selectors" yaml:"selectors"` } type RelationshipDefinitionDB struct { - ID uuid.UUID `json:"id"` - ModelID uuid.UUID `json:"-" gorm:"index:idx_relationship_definition_dbs_model_id,column:modelID"` - VersionMeta - Kind string `json:"kind,omitempty" yaml:"kind"` - RelationshipType string `json:"type" yaml:"type" gorm:"type"` - SubType string `json:"subType" yaml:"subType"` - Metadata []byte `json:"metadata" yaml:"metadata"` + ID uuid.UUID `json:"id"` + v1beta1.VersionMeta + Kind string `json:"kind,omitempty" yaml:"kind"` // The property has been named RelationshipType instead of Type to avoid collision from Type() function, which enables support for dynamic type. // Though, the column name and the json representation is "type". - EvaluationQuery string `json:"evaluationQuery" yaml:"evaluationQuery" gorm:"evaluationQuery"` - Selectors []byte `json:"selectors" yaml:"selectors"` + RelationshipType string `json:"type" yaml:"type" gorm:"type"` + SubType string `json:"subType" yaml:"subType"` + EvaluationQuery string `json:"evaluationQuery" yaml:"evaluationQuery" gorm:"evaluationQuery"` + Metadata []byte `json:"metadata" yaml:"metadata"` + ModelID uuid.UUID `json:"-" gorm:"index:idx_relationship_definition_dbs_model_id,column:modelID"` + Selectors []byte `json:"selectors" yaml:"selectors"` } // For now, only filtering by Kind and SubType are allowed. @@ -79,33 +77,34 @@ func (r *RelationshipDefinition) Create(db *database.Handler) (uuid.UUID, error) } func (r *RelationshipDefinition) GetRelationshipDefinitionDB() (rdb RelationshipDefinitionDB) { - rdb.ID = r.ID + // rdb.ID = r.ID id will be assigned by the database itself don't use this, as it will be always uuid.nil, because id is not known when comp gets generated. + // While database creates an entry with valid primary key but to avoid confusion, it is disabled and accidental assignment of custom id. rdb.VersionMeta = r.VersionMeta - rdb.Metadata, _ = json.Marshal(r.Metadata) - rdb.Selectors, _ = json.Marshal(r.Selectors) rdb.Kind = r.Kind rdb.RelationshipType = r.RelationshipType rdb.SubType = r.SubType - rdb.ModelID = r.Model.ID rdb.EvaluationQuery = r.EvaluationQuery + rdb.Metadata, _ = json.Marshal(r.Metadata) + rdb.ModelID = r.Model.ID + rdb.Selectors, _ = json.Marshal(r.Selectors) return } -func (rdb *RelationshipDefinitionDB) GetRelationshipDefinition(m Model) (r RelationshipDefinition) { +func (rdb *RelationshipDefinitionDB) GetRelationshipDefinition(m v1beta1.Model) (r RelationshipDefinition) { r.ID = rdb.ID r.VersionMeta = rdb.VersionMeta + r.Kind = rdb.Kind + r.RelationshipType = rdb.RelationshipType + r.SubType = rdb.SubType + r.EvaluationQuery = rdb.EvaluationQuery if r.Metadata == nil { r.Metadata = make(map[string]interface{}) } _ = json.Unmarshal(rdb.Metadata, &r.Metadata) + r.Model = m if r.Selectors == nil { r.Selectors = []map[string]interface{}{} } _ = json.Unmarshal(rdb.Selectors, &r.Selectors) - r.RelationshipType = rdb.RelationshipType - r.SubType = rdb.SubType - r.Kind = rdb.Kind - r.Model = m - r.EvaluationQuery = rdb.EvaluationQuery return } diff --git a/models/meshmodel/core/v1beta1/component.go b/models/meshmodel/core/v1beta1/component.go index 61bccced..f4b415f0 100644 --- a/models/meshmodel/core/v1beta1/component.go +++ b/models/meshmodel/core/v1beta1/component.go @@ -39,21 +39,23 @@ type ComponentDefinition struct { ID uuid.UUID `json:"id,omitempty"` VersionMeta DisplayName string `json:"displayName" gorm:"displayName"` + Description string `json:"description" gorm:"description"` Format ComponentFormat `json:"format" yaml:"format"` Model Model `json:"model"` Metadata map[string]interface{} `json:"metadata" yaml:"metadata"` - // Rename this here and in schema as well, it creates confustion b/w entities like comps. relationship. model, here entity means Pod/Deployment.... - Entity component `json:"entity,omitempty" yaml:"entity"` + // component corresponds to the specifications of underlying entity eg: Pod/Deployment.... + Component component `json:"component,omitempty" yaml:"component"` } type ComponentDefinitionDB struct { ID uuid.UUID `json:"id"` VersionMeta DisplayName string `json:"displayName" gorm:"displayName"` + Description string `json:"description" gorm:"description"` Format ComponentFormat `json:"format" yaml:"format"` ModelID uuid.UUID `json:"-" gorm:"index:idx_component_definition_dbs_model_id,column:modelID"` Metadata []byte `json:"metadata" yaml:"metadata"` - Entity component `json:"entity,omitempty" yaml:"entity" gorm:"entity"` + Component component `json:"component,omitempty" yaml:"component" gorm:"component"` } func (c ComponentDefinition) Type() types.EntityType { @@ -71,7 +73,7 @@ func (c *ComponentDefinition) Create(db *database.Handler) (uuid.UUID, error) { return uuid.UUID{}, err } - if !utils.IsSchemaEmpty(c.Entity.Schema) { + if !utils.IsSchemaEmpty(c.Component.Schema) { c.Metadata["hasInvalidSchema"] = true } cdb := c.GetComponentDefinitionDB() @@ -81,27 +83,29 @@ func (c *ComponentDefinition) Create(db *database.Handler) (uuid.UUID, error) { } func (c *ComponentDefinition) GetComponentDefinitionDB() (cmd ComponentDefinitionDB) { - cmd.ID = c.ID + // cmd.ID = c.ID id will be assigned by the database itself don't use this, as it will be always uuid.nil, because id is not known when comp gets generated. + // While database creates an entry with valid primary key but to avoid confusion, it is disabled and accidental assignment of custom id. cmd.VersionMeta = c.VersionMeta - cmd.Entity.TypeMeta = c.Entity.TypeMeta - cmd.Format = c.Format - cmd.Metadata, _ = json.Marshal(c.Metadata) cmd.DisplayName = c.DisplayName - cmd.Entity.Schema = c.Entity.Schema + cmd.Description = c.Description + cmd.Format = c.Format + cmd.ModelID = c.Model.ID + cmd.Metadata, _ = json.Marshal(c.Metadata) + cmd.Component = c.Component return } func (cmd *ComponentDefinitionDB) GetComponentDefinition(model Model) (c ComponentDefinition) { c.ID = cmd.ID c.VersionMeta = cmd.VersionMeta - c.Entity.TypeMeta = cmd.Entity.TypeMeta - c.Format = cmd.Format c.DisplayName = cmd.DisplayName + c.Description = cmd.Description + c.Format = cmd.Format + c.Model = model if c.Metadata == nil { c.Metadata = make(map[string]interface{}) } _ = json.Unmarshal(cmd.Metadata, &c.Metadata) - c.Entity.Schema = cmd.Entity.Schema - c.Model = model + c.Component = cmd.Component return } diff --git a/models/meshmodel/core/v1beta1/models.go b/models/meshmodel/core/v1beta1/models.go index ea60be94..fe8f2ec0 100644 --- a/models/meshmodel/core/v1beta1/models.go +++ b/models/meshmodel/core/v1beta1/models.go @@ -45,13 +45,13 @@ type Model struct { VersionMeta Name string `json:"name"` DisplayName string `json:"displayName" gorm:"modelDisplayName"` - Version string `json:"version"` + Description string `json:"description" gorm:"description"` Status ModelStatus `json:"status" gorm:"status"` Registrant registry.Hostv1beta1 `json:"registrant" gorm:"registrant"` // to be Connection Category Category `json:"category"` - Model model `json:"model,omitempty" gorm:"model"` SubCategory string `json:"subCategory" gorm:"subCategory"` Metadata map[string]interface{} `json:"metadata" yaml:"modelMetadata"` + Model model `json:"model,omitempty" gorm:"model"` // Components []ComponentDefinitionDB `json:"components"` // Relationships []RelationshipDefinitionDB `json:"relationships"` } @@ -59,15 +59,15 @@ type Model struct { type ModelDB struct { ID uuid.UUID `json:"id"` VersionMeta - CategoryID uuid.UUID `json:"-" gorm:"categoryID"` - Model model `json:"model,omitempty" gorm:"model"` - RegistrantID uuid.UUID `json:"hostID" gorm:"hostID"` Name string `json:"modelName" gorm:"modelName"` - Version string `json:"version"` DisplayName string `json:"modelDisplayName" gorm:"modelDisplayName"` + Description string `json:"description" gorm:"description"` + Status ModelStatus `json:"status" gorm:"status"` + RegistrantID uuid.UUID `json:"hostID" gorm:"hostID"` + CategoryID uuid.UUID `json:"-" gorm:"categoryID"` SubCategory string `json:"subCategory" gorm:"subCategory"` Metadata []byte `json:"modelMetadata" gorm:"modelMetadata"` - Status ModelStatus `json:"status" gorm:"status"` + Model model `json:"model,omitempty" gorm:"model"` } func (m Model) Type() types.EntityType { @@ -119,26 +119,34 @@ func (m *Model) Create(db *database.Handler) (uuid.UUID, error) { } func (c *Model) GetModelDB() (cmd ModelDB) { - cmd.ID = c.ID + // cmd.ID = c.ID id will be assigned by the database itself don't use this, as it will be always uuid.nil, because id is not known when comp gets generated. + // While database creates an entry with valid primary key but to avoid confusion, it is disabled and accidental assignment of custom id. cmd.VersionMeta = c.VersionMeta + cmd.Name = c.Name cmd.DisplayName = c.DisplayName + cmd.Description = c.Description cmd.Status = c.Status - cmd.Name = c.Name - cmd.Version = c.Version + cmd.RegistrantID = c.Registrant.ID + cmd.CategoryID = c.Category.ID + cmd.SubCategory = c.SubCategory cmd.Metadata, _ = json.Marshal(c.Metadata) + cmd.Model = c.Model return } - -func (cmd *ModelDB) GetModel(cat Category) (c Model) { +// is reg should be passed as param? +func (cmd *ModelDB) GetModel(cat Category, reg registry.Hostv1beta1) (c Model) { c.ID = cmd.ID c.VersionMeta = cmd.VersionMeta - c.Category = cat + c.Name = cmd.Name c.DisplayName = cmd.DisplayName + c.Description = cmd.Description c.Status = cmd.Status - c.Name = cmd.Name - c.Version = cmd.Version + c.Registrant = reg + c.Category = cat + c.SubCategory = cmd.SubCategory + _ = json.Unmarshal(cmd.Metadata, &c.Metadata) + c.Model = cmd.Model // c.Components = make([]ComponentDefinitionDB, 0) // c.Relationships = make([]RelationshipDefinitionDB, 0) - _ = json.Unmarshal(cmd.Metadata, &c.Metadata) return } From 2b9672b740037f8031d3365c7e39681a8363d0b4 Mon Sep 17 00:00:00 2001 From: MUzairS15 Date: Mon, 25 Mar 2024 15:07:30 +0530 Subject: [PATCH 08/32] implement updateStatus for all entites Signed-off-by: MUzairS15 --- .../meshmodel/core/v1alpha2/relationship.go | 10 ++++-- models/meshmodel/core/v1beta1/category.go | 10 ++++-- models/meshmodel/core/v1beta1/component.go | 12 ++++--- models/meshmodel/core/v1beta1/models.go | 31 +++++++++++++++++-- models/meshmodel/entity/error.go | 15 +++++++++ models/meshmodel/entity/status.go | 29 +++++++++++++++++ models/meshmodel/entity/types.go | 1 + 7 files changed, 95 insertions(+), 13 deletions(-) create mode 100644 models/meshmodel/entity/error.go create mode 100644 models/meshmodel/entity/status.go diff --git a/models/meshmodel/core/v1alpha2/relationship.go b/models/meshmodel/core/v1alpha2/relationship.go index 65ce357f..dc287f50 100644 --- a/models/meshmodel/core/v1alpha2/relationship.go +++ b/models/meshmodel/core/v1alpha2/relationship.go @@ -6,7 +6,7 @@ import ( "github.com/google/uuid" "github.com/layer5io/meshkit/database" "github.com/layer5io/meshkit/models/meshmodel/core/v1beta1" - types "github.com/layer5io/meshkit/models/meshmodel/entity" + "github.com/layer5io/meshkit/models/meshmodel/entity" ) // Remove additional DB structs from all entites they are used just for marshaling & unmarshlling and tarcking modelID vs model(modelID when referenced in DB and full model when sending data, use foregin key references tag of gorm for that as used in MeshSync tables) when saving retrivng from database, insted use gorm' serailization tag as used in events. @@ -54,8 +54,8 @@ type RelationshipFilter struct { Offset int } -func (r RelationshipDefinition) Type() types.EntityType { - return types.RelationshipDefinition +func (r RelationshipDefinition) Type() entity.EntityType { + return entity.RelationshipDefinition } func (r RelationshipDefinition) GetID() uuid.UUID { return r.ID @@ -76,6 +76,10 @@ func (r *RelationshipDefinition) Create(db *database.Handler) (uuid.UUID, error) return r.ID, err } +func (m *RelationshipDefinition) UpdateStatus(db database.Handler, status entity.EntityStatus) error { + return nil +} + func (r *RelationshipDefinition) GetRelationshipDefinitionDB() (rdb RelationshipDefinitionDB) { // rdb.ID = r.ID id will be assigned by the database itself don't use this, as it will be always uuid.nil, because id is not known when comp gets generated. // While database creates an entry with valid primary key but to avoid confusion, it is disabled and accidental assignment of custom id. diff --git a/models/meshmodel/core/v1beta1/category.go b/models/meshmodel/core/v1beta1/category.go index aed56851..32f9dc02 100644 --- a/models/meshmodel/core/v1beta1/category.go +++ b/models/meshmodel/core/v1beta1/category.go @@ -6,6 +6,7 @@ import ( "github.com/google/uuid" "github.com/layer5io/meshkit/database" + "github.com/layer5io/meshkit/models/meshmodel/entity" "gorm.io/gorm" ) @@ -32,11 +33,11 @@ type CategoryFilter struct { Limit int //If 0 or unspecified then all records are returned and limit is not used Offset int } + // "Uncategorized" is assigned when Category is empty in the component definitions. const DefaultCategory = "Uncategorized" - -func(cat *Category) Create(db *database.Handler) (uuid.UUID, error) { +func (cat *Category) Create(db *database.Handler) (uuid.UUID, error) { if cat.Name == "" { cat.Name = DefaultCategory } @@ -64,6 +65,10 @@ func(cat *Category) Create(db *database.Handler) (uuid.UUID, error) { return category.ID, nil } +func (m *Category) UpdateStatus(db database.Handler, status entity.EntityStatus) error { + return nil +} + func (c *Category) GetCategoryDB(db *database.Handler) (catdb CategoryDB) { catdb.ID = c.ID catdb.Name = c.Name @@ -77,4 +82,3 @@ func (cdb *CategoryDB) GetCategory(db *database.Handler) (cat Category) { _ = json.Unmarshal(cdb.Metadata, &cat.Metadata) return } - diff --git a/models/meshmodel/core/v1beta1/component.go b/models/meshmodel/core/v1beta1/component.go index f4b415f0..f8d8c43f 100644 --- a/models/meshmodel/core/v1beta1/component.go +++ b/models/meshmodel/core/v1beta1/component.go @@ -4,7 +4,7 @@ import ( "encoding/json" "github.com/layer5io/meshkit/database" - types "github.com/layer5io/meshkit/models/meshmodel/entity" + "github.com/layer5io/meshkit/models/meshmodel/entity" "github.com/layer5io/meshkit/utils" "github.com/google/uuid" @@ -58,8 +58,8 @@ type ComponentDefinitionDB struct { Component component `json:"component,omitempty" yaml:"component" gorm:"component"` } -func (c ComponentDefinition) Type() types.EntityType { - return types.ComponentDefinition +func (c ComponentDefinition) Type() entity.EntityType { + return entity.ComponentDefinition } func (c ComponentDefinition) GetID() uuid.UUID { @@ -82,13 +82,17 @@ func (c *ComponentDefinition) Create(db *database.Handler) (uuid.UUID, error) { return c.ID, err } +func (m *ComponentDefinition) UpdateStatus(db database.Handler, status entity.EntityStatus) error { + return nil +} + func (c *ComponentDefinition) GetComponentDefinitionDB() (cmd ComponentDefinitionDB) { // cmd.ID = c.ID id will be assigned by the database itself don't use this, as it will be always uuid.nil, because id is not known when comp gets generated. // While database creates an entry with valid primary key but to avoid confusion, it is disabled and accidental assignment of custom id. cmd.VersionMeta = c.VersionMeta cmd.DisplayName = c.DisplayName cmd.Description = c.Description - cmd.Format = c.Format + cmd.Format = c.Format cmd.ModelID = c.Model.ID cmd.Metadata, _ = json.Marshal(c.Metadata) cmd.Component = c.Component diff --git a/models/meshmodel/core/v1beta1/models.go b/models/meshmodel/core/v1beta1/models.go index fe8f2ec0..eb39875b 100644 --- a/models/meshmodel/core/v1beta1/models.go +++ b/models/meshmodel/core/v1beta1/models.go @@ -3,12 +3,14 @@ package v1beta1 import ( "encoding/json" "fmt" + "path/filepath" "sync" "github.com/google/uuid" "github.com/layer5io/meshkit/database" - types "github.com/layer5io/meshkit/models/meshmodel/entity" + "github.com/layer5io/meshkit/models/meshmodel/entity" "github.com/layer5io/meshkit/models/meshmodel/registry" + "github.com/layer5io/meshkit/utils" "gorm.io/gorm" ) @@ -70,8 +72,8 @@ type ModelDB struct { Model model `json:"model,omitempty" gorm:"model"` } -func (m Model) Type() types.EntityType { - return types.Model +func (m Model) Type() entity.EntityType { + return entity.Model } func (m Model) GetID() uuid.UUID { return m.ID @@ -118,6 +120,14 @@ func (m *Model) Create(db *database.Handler) (uuid.UUID, error) { return model.ID, nil } +func (m *Model) UpdateStatus(db database.Handler, status entity.EntityStatus) error { + err := db.Model(&ModelDB{}).Where("id = ?", m.ID).Update("status", status).Error + if err != nil { + return entity.ErrUpdateEntityStatus(err, string(m.Type()), status.String()) + } + return nil +} + func (c *Model) GetModelDB() (cmd ModelDB) { // cmd.ID = c.ID id will be assigned by the database itself don't use this, as it will be always uuid.nil, because id is not known when comp gets generated. // While database creates an entry with valid primary key but to avoid confusion, it is disabled and accidental assignment of custom id. @@ -133,6 +143,7 @@ func (c *Model) GetModelDB() (cmd ModelDB) { cmd.Model = c.Model return } + // is reg should be passed as param? func (cmd *ModelDB) GetModel(cat Category, reg registry.Hostv1beta1) (c Model) { c.ID = cmd.ID @@ -150,3 +161,17 @@ func (cmd *ModelDB) GetModel(cat Category, reg registry.Hostv1beta1) (c Model) { // c.Relationships = make([]RelationshipDefinitionDB, 0) return } + +func (c Model) WriteModelDefinition(modelDefPath string) error { + err := utils.CreateDirectory(modelDefPath) + if err != nil { + return err + } + + modelFilePath := filepath.Join(modelDefPath, "model.json") + err = utils.WriteJSONToFile[Model](modelFilePath, c) + if err != nil { + return err + } + return nil +} diff --git a/models/meshmodel/entity/error.go b/models/meshmodel/entity/error.go new file mode 100644 index 00000000..f5f61fc6 --- /dev/null +++ b/models/meshmodel/entity/error.go @@ -0,0 +1,15 @@ +package entity + +import ( + "fmt" + + "github.com/layer5io/meshkit/errors" +) + +const ( + ErrUpdateEntityStatusCode = "" +) + +func ErrUpdateEntityStatus(err error, entity, status string) error { + return errors.New(ErrUpdateEntityStatusCode, errors.Alert, []string{fmt.Sprintf("unable to update %s to %s", entity, status)}, []string{err.Error()}, []string{}, []string{}) +} diff --git a/models/meshmodel/entity/status.go b/models/meshmodel/entity/status.go new file mode 100644 index 00000000..e12dd7a8 --- /dev/null +++ b/models/meshmodel/entity/status.go @@ -0,0 +1,29 @@ +package entity + +import "github.com/layer5io/meshkit/database" + +type EntityStatus int + +const ( + Ignored EntityStatus = iota + Enabled + Duplicate +) + +func(e EntityStatus) String() string { + switch e { + case Ignored: + return "ignored" + case Duplicate: + return "duplicate" + case Enabled: + fallthrough + default: + return "enabled" + } + +} + +type Status interface { + UpdateStatus(db *database.Handler, status EntityStatus) error +} \ No newline at end of file diff --git a/models/meshmodel/entity/types.go b/models/meshmodel/entity/types.go index 8764c2ca..425f7970 100644 --- a/models/meshmodel/entity/types.go +++ b/models/meshmodel/entity/types.go @@ -17,6 +17,7 @@ type Filter interface { } type Entity interface { + Status Create() (entityID uuid.UUID, err error) Get() (entityID uuid.UUID, err error) } From 75a08acb64526d9124a2169feb47887417e9f968 Mon Sep 17 00:00:00 2001 From: MUzairS15 Date: Mon, 25 Mar 2024 15:36:35 +0530 Subject: [PATCH 09/32] implement Get function for comp Signed-off-by: MUzairS15 --- models/meshmodel/core/v1alpha1/component.go | 5 +- models/meshmodel/core/v1beta1/component.go | 119 ++++++++++++++++++++ models/meshmodel/entity/types.go | 9 +- 3 files changed, 128 insertions(+), 5 deletions(-) diff --git a/models/meshmodel/core/v1alpha1/component.go b/models/meshmodel/core/v1alpha1/component.go index b8d28140..58ec227c 100644 --- a/models/meshmodel/core/v1alpha1/component.go +++ b/models/meshmodel/core/v1alpha1/component.go @@ -30,7 +30,7 @@ const ( type ComponentDefinition struct { ID uuid.UUID `json:"id,omitempty"` TypeMeta - DisplayName string `json:"displayName" gorm:"displayName"` + DisplayName string `json:"displayName"` Format ComponentFormat `json:"format" yaml:"format"` HostName string `json:"hostname,omitempty"` HostID uuid.UUID `json:"hostID,omitempty"` @@ -45,7 +45,7 @@ type ComponentDefinitionDB struct { ID uuid.UUID `json:"id"` ModelID uuid.UUID `json:"-" gorm:"index:idx_component_definition_dbs_model_id,column:modelID"` TypeMeta - DisplayName string `json:"displayName" gorm:"displayName"` + DisplayName string `json:"displayName"` Format ComponentFormat `json:"format" yaml:"format"` Metadata []byte `json:"metadata" yaml:"metadata"` Schema string `json:"schema,omitempty" yaml:"schema"` @@ -86,6 +86,7 @@ func CreateComponent(db *database.Handler, c ComponentDefinition) (uuid.UUID, uu err = db.Create(&cdb).Error return c.ID, mid, err } + func GetMeshModelComponents(db *database.Handler, f ComponentFilter) (c []ComponentDefinition, count int64, unique int) { type componentDefinitionWithModel struct { diff --git a/models/meshmodel/core/v1beta1/component.go b/models/meshmodel/core/v1beta1/component.go index f8d8c43f..bf5bd199 100644 --- a/models/meshmodel/core/v1beta1/component.go +++ b/models/meshmodel/core/v1beta1/component.go @@ -2,9 +2,12 @@ package v1beta1 import ( "encoding/json" + "fmt" + "path/filepath" "github.com/layer5io/meshkit/database" "github.com/layer5io/meshkit/models/meshmodel/entity" + "github.com/layer5io/meshkit/models/meshmodel/registry" "github.com/layer5io/meshkit/utils" "github.com/google/uuid" @@ -33,6 +36,12 @@ type component struct { Schema string `json:"schema,omitempty" yaml:"schema"` } +type componentDefinitionWithModel struct { + ComponentDefinitionDB + ModelDB //nolint + CategoryDB +} + // swagger:response ComponentDefinition // use NewComponent function for instantiating type ComponentDefinition struct { @@ -58,6 +67,22 @@ type ComponentDefinitionDB struct { Component component `json:"component,omitempty" yaml:"component" gorm:"component"` } +type ComponentFilter struct { + Name string + APIVersion string + Greedy bool //when set to true - instead of an exact match, name will be matched as a substring + Trim bool //when set to true - the schema is not returned + DisplayName string + ModelName string + CategoryName string + Version string + Sort string //asc or desc. Default behavior is asc + OrderOn string + Limit int //If 0 or unspecified then all records are returned and limit is not used + Offset int + Annotations string //When this query parameter is "true", only components with the "isAnnotation" property set to true are returned. When this query parameter is "false", all components except those considered to be annotation components are returned. Any other value of the query parameter results in both annotations as well as non-annotation models being returned. +} + func (c ComponentDefinition) Type() entity.EntityType { return entity.ComponentDefinition } @@ -66,6 +91,94 @@ func (c ComponentDefinition) GetID() uuid.UUID { return c.ID } +func (c *ComponentDefinition) Get(db *database.Handler, f entity.Filter) ([]ComponentDefinition, int64, int, error) { + componentFilter, err := utils.Cast[ComponentFilter](f) + if err != nil { + return nil, 0, 0, err + } + + countUniqueComponents := func(components []componentDefinitionWithModel) int { + set := make(map[string]struct{}) + for _, compWithModel := range components { + key := compWithModel.Component.Kind + "@" + compWithModel.Component.Version + "@" + compWithModel.ModelDB.Name + "@" + compWithModel.ModelDB.Version + if _, ok := set[key]; !ok { + set[key] = struct{}{} + } + } + return len(set) + } + var componentDefinitionsWithModel []componentDefinitionWithModel + finder := db.Model(&ComponentDefinitionDB{}). + Select("component_definition_dbs.*, model_dbs.*,category_dbs.*"). + Joins("JOIN model_dbs ON component_definition_dbs.model_id = model_dbs.id"). + Joins("JOIN category_dbs ON model_dbs.category_id = category_dbs.id") // + + if componentFilter.Greedy { + if componentFilter.Name != "" && componentFilter.DisplayName != "" { + finder = finder.Where("component_definition_dbs.component->>'kind' LIKE ? OR display_name LIKE ?", "%"+componentFilter.Name+"%", componentFilter.DisplayName+"%") + } else if componentFilter.Name != "" { + finder = finder.Where("component_definition_dbs.component->>'kind' LIKE ?", "%"+componentFilter.Name+"%") + } else if componentFilter.DisplayName != "" { + finder = finder.Where("component_definition_dbs.display_name LIKE ?", "%"+componentFilter.DisplayName+"%") + } + } else { + if componentFilter.Name != "" { + finder = finder.Where("component_definition_dbs.component->>'kind' = ?", componentFilter.Name) + } + if componentFilter.DisplayName != "" { + finder = finder.Where("component_definition_dbs.display_name = ?", componentFilter.DisplayName) + } + } + + if componentFilter.ModelName != "" && componentFilter.ModelName != "all" { + finder = finder.Where("model_dbs.name = ?", componentFilter.ModelName) + } + + if componentFilter.Annotations == "true" { + finder = finder.Where("component_definition_dbs.metadata->>'isAnnotation' = true") + } else if componentFilter.Annotations == "false" { + finder = finder.Where("component_definition_dbs.metadata->>'isAnnotation' = false") + } + + if componentFilter.APIVersion != "" { + finder = finder.Where("component_definition_dbs.component->>'version' = ?", componentFilter.APIVersion) + } + if componentFilter.CategoryName != "" { + finder = finder.Where("category_dbs.name = ?", componentFilter.CategoryName) + } + if componentFilter.Version != "" { + finder = finder.Where("model_dbs.version = ?", componentFilter.Version) + } + + var count int64 + finder.Count(&count) + finder = finder.Offset(componentFilter.Offset) + if componentFilter.Limit != 0 { + finder = finder.Limit(componentFilter.Limit) + } + err = finder. + Scan(&componentDefinitionsWithModel).Error + if err != nil { + fmt.Println(err.Error()) //for debugging + } + + defs := make([]ComponentDefinition, len(componentDefinitionsWithModel)) + // remove this when compoentdef and componetdefdb struct is consolidated. + for _, cm := range componentDefinitionsWithModel { + if componentFilter.Trim { + cm.Component.Schema = "" + } + // Ensure correct reg is passed, rn it is dummy for sake of testing. + // In the first query above where we do seelection i think there changes will be requrired, an when that two def and defDB structs are consolidated, using association and preload i think we can do. + reg := registry.Hostv1beta1{} + defs = append(defs, cm.ComponentDefinitionDB.GetComponentDefinition(cm.ModelDB.GetModel(cm.CategoryDB.GetCategory(db), reg))) + } + + unique := countUniqueComponents(componentDefinitionsWithModel) + + return defs, count, unique, nil +} + func (c *ComponentDefinition) Create(db *database.Handler) (uuid.UUID, error) { c.ID = uuid.New() mid, err := c.Model.Create(db) @@ -99,6 +212,12 @@ func (c *ComponentDefinition) GetComponentDefinitionDB() (cmd ComponentDefinitio return } +func (c ComponentDefinition) WriteComponentDefinition(componentDirPath string) error { + componentPath := filepath.Join(componentDirPath, c.Component.Kind+".json") + err := utils.WriteJSONToFile[ComponentDefinition](componentPath, c) + return err +} + func (cmd *ComponentDefinitionDB) GetComponentDefinition(model Model) (c ComponentDefinition) { c.ID = cmd.ID c.VersionMeta = cmd.VersionMeta diff --git a/models/meshmodel/entity/types.go b/models/meshmodel/entity/types.go index 425f7970..f30ad7c3 100644 --- a/models/meshmodel/entity/types.go +++ b/models/meshmodel/entity/types.go @@ -1,6 +1,9 @@ package entity -import "github.com/google/uuid" +import ( + "github.com/google/uuid" + "github.com/layer5io/meshkit/database" +) type EntityType string @@ -18,6 +21,6 @@ type Filter interface { type Entity interface { Status - Create() (entityID uuid.UUID, err error) - Get() (entityID uuid.UUID, err error) + Create(db *database.Handler) (entityID uuid.UUID, err error) + Get(db *database.Handler, f Filter) (entities []Entity, count int64, unique int,err error) } From 3446a64717ed1d33958e7467d10719f8a67e65ae Mon Sep 17 00:00:00 2001 From: MUzairS15 Date: Mon, 25 Mar 2024 15:47:50 +0530 Subject: [PATCH 10/32] implement Get function for relationships Signed-off-by: MUzairS15 --- .../meshmodel/core/v1alpha2/relationship.go | 82 +++++++++++++++++++ models/meshmodel/core/v1beta1/models.go | 8 +- 2 files changed, 85 insertions(+), 5 deletions(-) diff --git a/models/meshmodel/core/v1alpha2/relationship.go b/models/meshmodel/core/v1alpha2/relationship.go index dc287f50..ae1146eb 100644 --- a/models/meshmodel/core/v1alpha2/relationship.go +++ b/models/meshmodel/core/v1alpha2/relationship.go @@ -2,11 +2,16 @@ package v1alpha2 import ( "encoding/json" + "fmt" + "path/filepath" "github.com/google/uuid" "github.com/layer5io/meshkit/database" "github.com/layer5io/meshkit/models/meshmodel/core/v1beta1" "github.com/layer5io/meshkit/models/meshmodel/entity" + "github.com/layer5io/meshkit/models/meshmodel/registry" + "github.com/layer5io/meshkit/utils" + "gorm.io/gorm/clause" ) // Remove additional DB structs from all entites they are used just for marshaling & unmarshlling and tarcking modelID vs model(modelID when referenced in DB and full model when sending data, use foregin key references tag of gorm for that as used in MeshSync tables) when saving retrivng from database, insted use gorm' serailization tag as used in events. @@ -38,6 +43,12 @@ type RelationshipDefinitionDB struct { Selectors []byte `json:"selectors" yaml:"selectors"` } +type relationshipDefinitionWithModel struct { + RelationshipDefinitionDB + v1beta1.ModelDB //nolint + v1beta1.CategoryDB +} + // For now, only filtering by Kind and SubType are allowed. // In the future, we will add support to query using `selectors` (using CUE) // TODO: Add support for Model @@ -61,6 +72,71 @@ func (r RelationshipDefinition) GetID() uuid.UUID { return r.ID } +func (r *RelationshipDefinition) Get(db *database.Handler, f entity.Filter) ([]RelationshipDefinition, int64, int, error) { + relationshipFilter, err := utils.Cast[RelationshipFilter](f) + if err != nil { + return nil, 0, 0, err + } + + var relationshipDefinitionsWithModel []relationshipDefinitionWithModel + finder := db.Model(&RelationshipDefinitionDB{}). + Select("relationship_definition_dbs.*, model_dbs.*"). + Joins("JOIN model_dbs ON relationship_definition_dbs.model_id = model_dbs.id"). // + Joins("JOIN category_dbs ON model_dbs.category_id = category_dbs.id") // + if relationshipFilter.Kind != "" { + if relationshipFilter.Greedy { + finder = finder.Where("relationship_definition_dbs.kind LIKE ?", "%"+relationshipFilter.Kind+"%") + } else { + finder = finder.Where("relationship_definition_dbs.kind = ?", relationshipFilter.Kind) + } + } + + if relationshipFilter.RelationshipType != "" { + finder = finder.Where("relationship_definition_dbs.type = ?", relationshipFilter.RelationshipType) + } + + if relationshipFilter.SubType != "" { + finder = finder.Where("relationship_definition_dbs.sub_type = ?", relationshipFilter.SubType) + } + if relationshipFilter.ModelName != "" { + finder = finder.Where("model_dbs.name = ?", relationshipFilter.ModelName) + } + if relationshipFilter.Version != "" { + finder = finder.Where("model_dbs.version = ?", relationshipFilter.Version) + } + if relationshipFilter.OrderOn != "" { + if relationshipFilter.Sort == "desc" { + finder = finder.Order(clause.OrderByColumn{Column: clause.Column{Name: relationshipFilter.OrderOn}, Desc: true}) + } else { + finder = finder.Order(relationshipFilter.OrderOn) + } + } + + var count int64 + finder.Count(&count) + + finder = finder.Offset(relationshipFilter.Offset) + if relationshipFilter.Limit != 0 { + finder = finder.Limit(relationshipFilter.Limit) + } + err = finder. + Scan(&relationshipDefinitionsWithModel).Error + if err != nil { + fmt.Println(err.Error()) //for debugging + } + defs := make([]RelationshipDefinition, len(relationshipDefinitionsWithModel)) + + // remove this when reldef and reldefdb struct is consolidated. + for _, cm := range relationshipDefinitionsWithModel { + // Ensure correct reg is passed, rn it is dummy for sake of testing. + // In the first query above where we do seelection i think there changes will be requrired, an when that two def and defDB structs are consolidated, using association and preload i think we can do. + reg := registry.Hostv1beta1{} + defs = append(defs, cm.RelationshipDefinitionDB.GetRelationshipDefinition(cm.ModelDB.GetModel(cm.CategoryDB.GetCategory(db), reg))) + } + // Should have count unique relationships (by model version, model name, and relationship's kind, type, subtype, version) + return defs, count, int(count), nil +} + func (r *RelationshipDefinition) Create(db *database.Handler) (uuid.UUID, error) { r.ID = uuid.New() mid, err := r.Model.Create(db) @@ -94,6 +170,12 @@ func (r *RelationshipDefinition) GetRelationshipDefinitionDB() (rdb Relationship return } +func (c RelationshipDefinition) WriteComponentDefinition(relDirPath string) error { + relPath := filepath.Join(relDirPath, c.Kind, string(c.Type())+".json") + err := utils.WriteJSONToFile[RelationshipDefinition](relPath, c) + return err +} + func (rdb *RelationshipDefinitionDB) GetRelationshipDefinition(m v1beta1.Model) (r RelationshipDefinition) { r.ID = rdb.ID r.VersionMeta = rdb.VersionMeta diff --git a/models/meshmodel/core/v1beta1/models.go b/models/meshmodel/core/v1beta1/models.go index eb39875b..6e730a29 100644 --- a/models/meshmodel/core/v1beta1/models.go +++ b/models/meshmodel/core/v1beta1/models.go @@ -16,8 +16,6 @@ import ( var modelCreationLock sync.Mutex //Each component/relationship will perform a check and if the model already doesn't exist, it will create a model. This lock will make sure that there are no race conditions. -type ModelStatus string - type ModelFilter struct { Name string Registrant string //name of the registrant for a given model @@ -48,7 +46,7 @@ type Model struct { Name string `json:"name"` DisplayName string `json:"displayName" gorm:"modelDisplayName"` Description string `json:"description" gorm:"description"` - Status ModelStatus `json:"status" gorm:"status"` + Status entity.EntityStatus `json:"status" gorm:"status"` Registrant registry.Hostv1beta1 `json:"registrant" gorm:"registrant"` // to be Connection Category Category `json:"category"` SubCategory string `json:"subCategory" gorm:"subCategory"` @@ -64,7 +62,7 @@ type ModelDB struct { Name string `json:"modelName" gorm:"modelName"` DisplayName string `json:"modelDisplayName" gorm:"modelDisplayName"` Description string `json:"description" gorm:"description"` - Status ModelStatus `json:"status" gorm:"status"` + Status entity.EntityStatus `json:"status" gorm:"status"` RegistrantID uuid.UUID `json:"hostID" gorm:"hostID"` CategoryID uuid.UUID `json:"-" gorm:"categoryID"` SubCategory string `json:"subCategory" gorm:"subCategory"` @@ -110,7 +108,7 @@ func (m *Model) Create(db *database.Handler) (uuid.UUID, error) { mdb := m.GetModelDB() mdb.CategoryID = id mdb.RegistrantID = hostID - mdb.Status = "registered" + mdb.Status = entity.Enabled err = db.Create(&mdb).Error if err != nil { return uuid.UUID{}, err From c524f5aa61ca1e6c2732dd30e938b9ba5e3f2265 Mon Sep 17 00:00:00 2001 From: MUzairS15 Date: Mon, 25 Mar 2024 16:01:17 +0530 Subject: [PATCH 11/32] implement Get function for relationships Signed-off-by: MUzairS15 --- models/meshmodel/core/v1alpha2/relationship.go | 6 ++++-- models/meshmodel/core/v1beta1/component.go | 4 ++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/models/meshmodel/core/v1alpha2/relationship.go b/models/meshmodel/core/v1alpha2/relationship.go index ae1146eb..6f8ad3f2 100644 --- a/models/meshmodel/core/v1alpha2/relationship.go +++ b/models/meshmodel/core/v1alpha2/relationship.go @@ -45,8 +45,10 @@ type RelationshipDefinitionDB struct { type relationshipDefinitionWithModel struct { RelationshipDefinitionDB - v1beta1.ModelDB //nolint - v1beta1.CategoryDB + ModelDB v1beta1.ModelDB + // acoount for overridn fields + // v1beta1.ModelDB.Version `json:"modelVersion"` + CategoryDB v1beta1.CategoryDB } // For now, only filtering by Kind and SubType are allowed. diff --git a/models/meshmodel/core/v1beta1/component.go b/models/meshmodel/core/v1beta1/component.go index bf5bd199..ad0a1b81 100644 --- a/models/meshmodel/core/v1beta1/component.go +++ b/models/meshmodel/core/v1beta1/component.go @@ -38,8 +38,8 @@ type component struct { type componentDefinitionWithModel struct { ComponentDefinitionDB - ModelDB //nolint - CategoryDB + ModelDB ModelDB // acoount for overridn fields + CategoryDB CategoryDB } // swagger:response ComponentDefinition From 0486120565654cc73b5025c66eadcf72cffb0e45 Mon Sep 17 00:00:00 2001 From: MUzairS15 Date: Mon, 25 Mar 2024 16:06:48 +0530 Subject: [PATCH 12/32] delete v1alpha1 Signed-off-by: MUzairS15 --- models/meshmodel/core/v1alpha1/category.go | 83 ------- models/meshmodel/core/v1alpha1/component.go | 234 ------------------ models/meshmodel/core/v1alpha1/host.go | 37 --- models/meshmodel/core/v1alpha1/models.go | 152 ------------ models/meshmodel/core/v1alpha1/policy.go | 121 --------- .../meshmodel/core/v1alpha1/relationship.go | 182 -------------- 6 files changed, 809 deletions(-) delete mode 100644 models/meshmodel/core/v1alpha1/category.go delete mode 100644 models/meshmodel/core/v1alpha1/component.go delete mode 100644 models/meshmodel/core/v1alpha1/host.go delete mode 100644 models/meshmodel/core/v1alpha1/models.go delete mode 100644 models/meshmodel/core/v1alpha1/policy.go delete mode 100644 models/meshmodel/core/v1alpha1/relationship.go diff --git a/models/meshmodel/core/v1alpha1/category.go b/models/meshmodel/core/v1alpha1/category.go deleted file mode 100644 index 201938c1..00000000 --- a/models/meshmodel/core/v1alpha1/category.go +++ /dev/null @@ -1,83 +0,0 @@ -package v1alpha1 - -import ( - "encoding/json" - "sync" - - "github.com/google/uuid" - "github.com/layer5io/meshkit/database" - "gorm.io/gorm" -) - -var categoryCreationLock sync.Mutex //Each model will perform a check and if the category already doesn't exist, it will create a category. This lock will make sure that there are no race conditions. - -// swagger:response Category -type Category struct { - ID uuid.UUID `json:"-" yaml:"-"` - Name string `json:"name"` - Metadata map[string]interface{} `json:"metadata" yaml:"metadata"` -} -type CategoryDB struct { - ID uuid.UUID `json:"-"` - Name string `json:"categoryName" gorm:"categoryName"` - Metadata []byte `json:"categoryMetadata" gorm:"categoryMetadata"` -} -type CategoryFilter struct { - Name string - OrderOn string - Greedy bool - Sort string //asc or desc. Default behavior is asc - Limit int //If 0 or unspecified then all records are returned and limit is not used - Offset int -} - -// "Uncategorized" is assigned when Category is empty in the component definitions. -const DefaultCategory = "Uncategorized" - -// Create the filter from map[string]interface{} -func (cf *CategoryFilter) Create(m map[string]interface{}) { - if m == nil { - return - } - cf.Name = m["name"].(string) -} -func CreateCategory(db *database.Handler, cat Category) (uuid.UUID, error) { - if cat.Name == "" { - cat.Name = DefaultCategory - } - byt, err := json.Marshal(cat) - if err != nil { - return uuid.UUID{}, err - } - catID := uuid.NewSHA1(uuid.UUID{}, byt) - var category CategoryDB - categoryCreationLock.Lock() - defer categoryCreationLock.Unlock() - err = db.First(&category, "id = ?", catID).Error - if err != nil && err != gorm.ErrRecordNotFound { - return uuid.UUID{}, err - } - if err == gorm.ErrRecordNotFound { //The category is already not present and needs to be inserted - cat.ID = catID - catdb := cat.GetCategoryDB(db) - err = db.Create(&catdb).Error - if err != nil { - return uuid.UUID{}, err - } - return catdb.ID, nil - } - return category.ID, nil -} - -func (cdb *CategoryDB) GetCategory(db *database.Handler) (cat Category) { - cat.ID = cdb.ID - cat.Name = cdb.Name - _ = json.Unmarshal(cdb.Metadata, &cat.Metadata) - return -} -func (c *Category) GetCategoryDB(db *database.Handler) (catdb CategoryDB) { - catdb.ID = c.ID - catdb.Name = c.Name - catdb.Metadata, _ = json.Marshal(c.Metadata) - return -} diff --git a/models/meshmodel/core/v1alpha1/component.go b/models/meshmodel/core/v1alpha1/component.go deleted file mode 100644 index 58ec227c..00000000 --- a/models/meshmodel/core/v1alpha1/component.go +++ /dev/null @@ -1,234 +0,0 @@ -package v1alpha1 - -import ( - "encoding/json" - "fmt" - "path/filepath" - "time" - - "github.com/google/uuid" - "github.com/layer5io/meshkit/database" - types "github.com/layer5io/meshkit/models/meshmodel/entity" - "github.com/layer5io/meshkit/utils" - "gorm.io/gorm/clause" -) - -type TypeMeta struct { - Kind string `json:"kind,omitempty" yaml:"kind"` - APIVersion string `json:"apiVersion,omitempty" yaml:"apiVersion"` -} -type ComponentFormat string - -const ( - JSON ComponentFormat = "JSON" - YAML ComponentFormat = "YAML" - CUE ComponentFormat = "CUE" -) - -// swagger:response ComponentDefinition -// use NewComponent function for instantiating -type ComponentDefinition struct { - ID uuid.UUID `json:"id,omitempty"` - TypeMeta - DisplayName string `json:"displayName"` - Format ComponentFormat `json:"format" yaml:"format"` - HostName string `json:"hostname,omitempty"` - HostID uuid.UUID `json:"hostID,omitempty"` - DisplayHostName string `json:"displayhostname,omitempty"` - Metadata map[string]interface{} `json:"metadata" yaml:"metadata"` - Model Model `json:"model"` - Schema string `json:"schema,omitempty" yaml:"schema"` - CreatedAt time.Time `json:"-"` - UpdatedAt time.Time `json:"-"` -} -type ComponentDefinitionDB struct { - ID uuid.UUID `json:"id"` - ModelID uuid.UUID `json:"-" gorm:"index:idx_component_definition_dbs_model_id,column:modelID"` - TypeMeta - DisplayName string `json:"displayName"` - Format ComponentFormat `json:"format" yaml:"format"` - Metadata []byte `json:"metadata" yaml:"metadata"` - Schema string `json:"schema,omitempty" yaml:"schema"` - CreatedAt time.Time `json:"-"` - UpdatedAt time.Time `json:"-"` -} - -func (c ComponentDefinition) Type() types.EntityType { - return types.ComponentDefinition -} -func (c ComponentDefinition) GetID() uuid.UUID { - return c.ID -} -func emptySchemaCheck(schema string) (valid bool) { - if schema == "" { - return - } - m := make(map[string]interface{}) - _ = json.Unmarshal([]byte(schema), &m) - if m["properties"] == nil { - return - } - valid = true - return -} -func CreateComponent(db *database.Handler, c ComponentDefinition) (uuid.UUID, uuid.UUID, error) { - c.ID = uuid.New() - mid, err := CreateModel(db, c.Model) - if err != nil { - return uuid.UUID{}, uuid.UUID{}, err - } - - if !emptySchemaCheck(c.Schema) { - c.Metadata["hasInvalidSchema"] = true - } - cdb := c.GetComponentDefinitionDB() - cdb.ModelID = mid - err = db.Create(&cdb).Error - return c.ID, mid, err -} - -func GetMeshModelComponents(db *database.Handler, f ComponentFilter) (c []ComponentDefinition, count int64, unique int) { - - type componentDefinitionWithModel struct { - ComponentDefinitionDB - ModelDB //nolint - CategoryDB - } - - countUniqueComponents := func(components []componentDefinitionWithModel) int { - set := make(map[string]struct{}) - for _, model := range components { - key := model.ComponentDefinitionDB.Kind + "@" + model.APIVersion + "@" + model.ModelDB.Name + "@" + model.ModelDB.Version - if _, ok := set[key]; !ok { - set[key] = struct{}{} - } - } - return len(set) - } - - var componentDefinitionsWithModel []componentDefinitionWithModel - finder := db.Model(&ComponentDefinitionDB{}). - Select("component_definition_dbs.*, model_dbs.*,category_dbs.*"). - Joins("JOIN model_dbs ON component_definition_dbs.model_id = model_dbs.id"). - Joins("JOIN category_dbs ON model_dbs.category_id = category_dbs.id") // - - if f.Greedy { - if f.Name != "" && f.DisplayName != "" { - finder = finder.Where("component_definition_dbs.kind LIKE ? OR display_name LIKE ?", "%"+f.Name+"%", f.DisplayName+"%") - } else if f.Name != "" { - finder = finder.Where("component_definition_dbs.kind LIKE ?", "%"+f.Name+"%") - } else if f.DisplayName != "" { - finder = finder.Where("component_definition_dbs.display_name LIKE ?", "%"+f.DisplayName+"%") - } - } else { - if f.Name != "" { - finder = finder.Where("component_definition_dbs.kind = ?", f.Name) - } - if f.DisplayName != "" { - finder = finder.Where("component_definition_dbs.display_name = ?", f.DisplayName) - } - } - if f.ModelName != "" && f.ModelName != "all" { - finder = finder.Where("model_dbs.name = ?", f.ModelName) - } - - if f.Annotations == "true" { - finder = finder.Where("component_definition_dbs.metadata->>'isAnnotation' = true") - } else if f.Annotations == "false" { - finder = finder.Where("component_definition_dbs.metadata->>'isAnnotation' = false") - } - - if f.APIVersion != "" { - finder = finder.Where("component_definition_dbs.api_version = ?", f.APIVersion) - } - if f.CategoryName != "" { - finder = finder.Where("category_dbs.name = ?", f.CategoryName) - } - if f.Version != "" { - finder = finder.Where("model_dbs.version = ?", f.Version) - } - if f.OrderOn != "" { - if f.Sort == "desc" { - finder = finder.Order(clause.OrderByColumn{Column: clause.Column{Name: f.OrderOn}, Desc: true}) - } else { - finder = finder.Order(f.OrderOn) - } - } else { - finder = finder.Order("display_name") - } - - finder.Count(&count) - - finder = finder.Offset(f.Offset) - if f.Limit != 0 { - finder = finder.Limit(f.Limit) - } - err := finder. - Scan(&componentDefinitionsWithModel).Error - if err != nil { - fmt.Println(err.Error()) //for debugging - } - for _, cm := range componentDefinitionsWithModel { - if f.Trim { - cm.Schema = "" - } - c = append(c, cm.ComponentDefinitionDB.GetComponentDefinition(cm.ModelDB.GetModel(cm.CategoryDB.GetCategory(db)))) - } - - unique = countUniqueComponents(componentDefinitionsWithModel) - - return c, count, unique -} - -type ComponentFilter struct { - Name string - APIVersion string - Greedy bool //when set to true - instead of an exact match, name will be matched as a substring - Trim bool //when set to true - the schema is not returned - DisplayName string - ModelName string - CategoryName string - Version string - Sort string //asc or desc. Default behavior is asc - OrderOn string - Limit int //If 0 or unspecified then all records are returned and limit is not used - Offset int - Annotations string //When this query parameter is "true", only components with the "isAnnotation" property set to true are returned. When this query parameter is "false", all components except those considered to be annotation components are returned. Any other value of the query parameter results in both annotations as well as non-annotation models being returned. -} - -// Create the filter from map[string]interface{} -func (cf *ComponentFilter) Create(m map[string]interface{}) { - if m == nil { - return - } - cf.Name = m["name"].(string) -} - -func (cmd *ComponentDefinitionDB) GetComponentDefinition(model Model) (c ComponentDefinition) { - c.ID = cmd.ID - c.TypeMeta = cmd.TypeMeta - c.Format = cmd.Format - c.DisplayName = cmd.DisplayName - if c.Metadata == nil { - c.Metadata = make(map[string]interface{}) - } - _ = json.Unmarshal(cmd.Metadata, &c.Metadata) - c.Schema = cmd.Schema - c.Model = model - return -} -func (c *ComponentDefinition) GetComponentDefinitionDB() (cmd ComponentDefinitionDB) { - cmd.ID = c.ID - cmd.TypeMeta = c.TypeMeta - cmd.Format = c.Format - cmd.Metadata, _ = json.Marshal(c.Metadata) - cmd.DisplayName = c.DisplayName - cmd.Schema = c.Schema - return -} - -func (c ComponentDefinition) WriteComponentDefinition(componentDirPath string) error { - componentPath := filepath.Join(componentDirPath, c.Kind+".json") - err := utils.WriteJSONToFile[ComponentDefinition](componentPath, c) - return err -} diff --git a/models/meshmodel/core/v1alpha1/host.go b/models/meshmodel/core/v1alpha1/host.go deleted file mode 100644 index fd3ece5c..00000000 --- a/models/meshmodel/core/v1alpha1/host.go +++ /dev/null @@ -1,37 +0,0 @@ -package v1alpha1 - -import "github.com/google/uuid" - -type MeshModelHostsWithEntitySummary struct { - ID uuid.UUID `json:"id"` - Hostname string `json:"hostname"` - Port int `json:"port"` - Summary EntitySummary `json:"summary"` -} -type EntitySummary struct { - Models int64 `json:"models"` - Components int64 `json:"components"` - Relationships int64 `json:"relationships"` - Policies int64 `json:"policies"` -} -type MesheryHostSummaryDB struct { - HostID uuid.UUID `json:"-" gorm:"id"` - Hostname string `json:"-" gorm:"hostname"` - Port int `json:"-" gorm:"port"` - Models int64 `json:"-" gorm:"models"` - Components int64 `json:"-" gorm:"components"` - Relationships int64 `json:"-" gorm:"relationships"` - Policies int64 `json:"-" gorm:"policies"` -} - -type HostFilter struct { - Name string - Greedy bool //when set to true - instead of an exact match, name will be prefix matched - Trim bool //when set to true - the schema is not returned - DisplayName string - Version string - Sort string //asc or desc. Default behavior is asc - OrderOn string - Limit int //If 0 or unspecified then all records are returned and limit is not used - Offset int -} diff --git a/models/meshmodel/core/v1alpha1/models.go b/models/meshmodel/core/v1alpha1/models.go deleted file mode 100644 index 1a1e4598..00000000 --- a/models/meshmodel/core/v1alpha1/models.go +++ /dev/null @@ -1,152 +0,0 @@ -package v1alpha1 - -import ( - "encoding/json" - "fmt" - "path/filepath" - "sync" - - "github.com/google/uuid" - "github.com/layer5io/meshkit/database" - types "github.com/layer5io/meshkit/models/meshmodel/entity" - "github.com/layer5io/meshkit/utils" - "gorm.io/gorm" -) - -var modelCreationLock sync.Mutex //Each component/relationship will perform a check and if the model already doesn't exist, it will create a model. This lock will make sure that there are no race conditions. -type ModelFilter struct { - Name string - Registrant string //name of the registrant for a given model - DisplayName string //If Name is already passed, avoid passing Display name unless greedy=true, else the filter will translate to an AND returning only the models where name and display name match exactly. Ignore, if this behavior is expected. - Greedy bool //when set to true - instead of an exact match, name will be prefix matched. Also an OR will be performed of name and display_name - Version string - Category string - OrderOn string - Sort string //asc or desc. Default behavior is asc - Limit int //If 0 or unspecified then all records are returned and limit is not used - Offset int - Annotations string //When this query parameter is "true", only models with the "isAnnotation" property set to true are returned. When this query parameter is "false", all models except those considered to be annotation models are returned. Any other value of the query parameter results in both annoations as well as non-annotation models being returned. - - // When these are set to true, we also retrieve components/relationships associated with the model. - Components bool - Relationships bool - Status string -} - -// Create the filter from map[string]interface{} -func (cf *ModelFilter) Create(m map[string]interface{}) { - if m == nil { - return - } - cf.Name = m["name"].(string) -} - -type ModelStatus string - -// swagger:response Model -type Model struct { - ID uuid.UUID `json:"id,omitempty" yaml:"-"` - Name string `json:"name"` - Version string `json:"version"` - DisplayName string `json:"displayName" gorm:"modelDisplayName"` - Status ModelStatus `json:"status" gorm:"status"` - HostName string `json:"hostname,omitempty"` - HostID uuid.UUID `json:"hostID,omitempty"` - DisplayHostName string `json:"displayhostname,omitempty"` - Category Category `json:"category"` - Metadata map[string]interface{} `json:"metadata" yaml:"modelMetadata"` - Components []ComponentDefinitionDB `json:"components"` - Relationships []RelationshipDefinitionDB `json:"relationships"` -} - -type ModelDB struct { - ID uuid.UUID `json:"id"` - CategoryID uuid.UUID `json:"-" gorm:"categoryID"` - Name string `json:"modelName" gorm:"modelName"` - Version string `json:"version"` - DisplayName string `json:"modelDisplayName" gorm:"modelDisplayName"` - SubCategory string `json:"subCategory" gorm:"subCategory"` - Metadata []byte `json:"modelMetadata" gorm:"modelMetadata"` - Status ModelStatus `json:"status" gorm:"status"` -} - -func (m Model) Type() types.EntityType { - return types.Model -} -func (m Model) GetID() uuid.UUID { - return m.ID -} - -func CreateModel(db *database.Handler, cmodel Model) (uuid.UUID, error) { - byt, err := json.Marshal(cmodel) - if err != nil { - return uuid.UUID{}, err - } - modelID := uuid.NewSHA1(uuid.UUID{}, byt) - var model ModelDB - if cmodel.Name == "" { - return uuid.UUID{}, fmt.Errorf("empty or invalid model name passed") - } - modelCreationLock.Lock() - defer modelCreationLock.Unlock() - err = db.First(&model, "id = ?", modelID).Error - if err != nil && err != gorm.ErrRecordNotFound { - return uuid.UUID{}, err - } - if err == gorm.ErrRecordNotFound { //The model is already not present and needs to be inserted - id, err := CreateCategory(db, cmodel.Category) - if err != nil { - return uuid.UUID{}, err - } - cmodel.ID = modelID - mdb := cmodel.GetModelDB() - mdb.CategoryID = id - mdb.Status = "registered" - err = db.Create(&mdb).Error - if err != nil { - return uuid.UUID{}, err - } - return mdb.ID, nil - } - return model.ID, nil -} - -func UpdateModelsStatus(db *database.Handler, modelID uuid.UUID, status string) error { - return db.Model(&ModelDB{}).Where("id = ?", modelID).Update("status", status).Error -} - -func (cmd *ModelDB) GetModel(cat Category) (c Model) { - c.ID = cmd.ID - c.Category = cat - c.DisplayName = cmd.DisplayName - c.Status = cmd.Status - c.Name = cmd.Name - c.Version = cmd.Version - c.Components = make([]ComponentDefinitionDB, 0) - c.Relationships = make([]RelationshipDefinitionDB, 0) - _ = json.Unmarshal(cmd.Metadata, &c.Metadata) - return -} -func (c *Model) GetModelDB() (cmd ModelDB) { - cmd.ID = c.ID - cmd.DisplayName = c.DisplayName - cmd.Status = c.Status - cmd.Name = c.Name - cmd.Version = c.Version - cmd.Metadata, _ = json.Marshal(c.Metadata) - return -} - -func (c Model) WriteModelDefinition(modelDefPath string) error { - err := utils.CreateDirectory(modelDefPath) - if err != nil { - return err - } - - modelFilePath := filepath.Join(modelDefPath, "model.json") - err = utils.WriteJSONToFile[Model](modelFilePath, c) - if err != nil { - return err - } - return nil -} diff --git a/models/meshmodel/core/v1alpha1/policy.go b/models/meshmodel/core/v1alpha1/policy.go deleted file mode 100644 index 722720c7..00000000 --- a/models/meshmodel/core/v1alpha1/policy.go +++ /dev/null @@ -1,121 +0,0 @@ -package v1alpha1 - -import ( - "encoding/json" - "fmt" - "time" - - "github.com/google/uuid" - "github.com/layer5io/meshkit/database" - types "github.com/layer5io/meshkit/models/meshmodel/entity" -) - -type PolicyDefinition struct { - ID uuid.UUID `json:"-"` - TypeMeta - Model Model `json:"model"` - SubType string `json:"subType" yaml:"subType"` - Expression map[string]interface{} `json:"expression" yaml:"expression"` - CreatedAt time.Time `json:"-"` - UpdatedAt time.Time `json:"-"` -} - -type PolicyDefinitionDB struct { - ID uuid.UUID `json:"-"` - ModelID uuid.UUID `json:"-" gorm:"modelID"` - TypeMeta - SubType string `json:"subType" yaml:"subType"` - Expression []byte `json:"expression" yaml:"expression"` - CreatedAt time.Time `json:"-"` - UpdatedAt time.Time `json:"-"` -} - -type PolicyFilter struct { - Kind string - Greedy bool - SubType string - ModelName string - OrderOn string - Sort string - Limit int - Offset int -} - -func (pf *PolicyFilter) Create(m map[string]interface{}) { - if m == nil { - return - } -} - -func (p PolicyDefinition) GetID() uuid.UUID { - return p.ID -} - -func (p PolicyDefinition) Type() types.EntityType { - return types.PolicyDefinition -} - -func GetMeshModelPolicy(db *database.Handler, f PolicyFilter) (pl []PolicyDefinition) { - type componentDefinitionWithModel struct { - PolicyDefinitionDB - Model - } - var componentDefinitionsWithModel []componentDefinitionWithModel - finder := db.Model(&PolicyDefinitionDB{}). - Select("policy_definition_dbs.*, models_dbs.*"). - Joins("JOIN model_dbs ON model_dbs.id = policy_definition_dbs.model_id") - if f.Kind != "" { - finder = finder.Where("policy_definition_dbs.kind = ?", f.Kind) - } - if f.SubType != "" { - finder = finder.Where("policy_definition_dbs.sub_type = ?", f.SubType) - } - if f.ModelName != "" { - finder = finder.Where("model_dbs.name = ?", f.ModelName) - } - err := finder.Scan(&componentDefinitionsWithModel).Error - if err != nil { - fmt.Println(err.Error()) - } - for _, cm := range componentDefinitionsWithModel { - pl = append(pl, cm.PolicyDefinitionDB.GetPolicyDefinition(cm.Model)) - } - return pl -} - -func (pdb *PolicyDefinitionDB) GetPolicyDefinition(m Model) (p PolicyDefinition) { - p.ID = pdb.ID - p.TypeMeta = pdb.TypeMeta - p.Model = m - p.SubType = pdb.SubType - if p.Expression == nil { - p.Expression = make(map[string]interface{}) - } - _ = json.Unmarshal(pdb.Expression, &p.Expression) - - return -} - -func CreatePolicy(db *database.Handler, p PolicyDefinition) (uuid.UUID, uuid.UUID, error) { - p.ID = uuid.New() - mid, err := CreateModel(db, p.Model) - if err != nil { - return uuid.UUID{}, uuid.UUID{}, err - } - pdb := p.GetPolicyDefinitionDB() - pdb.ModelID = mid - err = db.Create(&pdb).Error - if err != nil { - return uuid.UUID{}, uuid.UUID{}, err - } - return pdb.ID, mid, nil -} - -func (p *PolicyDefinition) GetPolicyDefinitionDB() (pdb PolicyDefinitionDB) { - pdb.ID = p.ID - pdb.TypeMeta = p.TypeMeta - pdb.SubType = p.SubType - pdb.ModelID = p.Model.ID - pdb.Expression, _ = json.Marshal(p.Expression) - return -} diff --git a/models/meshmodel/core/v1alpha1/relationship.go b/models/meshmodel/core/v1alpha1/relationship.go deleted file mode 100644 index b4b8e60c..00000000 --- a/models/meshmodel/core/v1alpha1/relationship.go +++ /dev/null @@ -1,182 +0,0 @@ -package v1alpha1 - -import ( - "encoding/json" - "fmt" - "time" - - "github.com/google/uuid" - "github.com/layer5io/meshkit/database" - types "github.com/layer5io/meshkit/models/meshmodel/entity" - "gorm.io/gorm/clause" -) - -// https://docs.google.com/drawings/d/1_qzQ_YxvCWPYrOBcdqGMlMwfbsZx96SBuIkbn8TfKhU/edit?pli=1 -// see RELATIONSHIPDEFINITIONS table in the diagram -// swagger:response RelationshipDefinition -// TODO: Add support for Model -type RelationshipDefinition struct { - ID uuid.UUID `json:"id"` - TypeMeta - Model Model `json:"model"` - HostName string `json:"hostname"` - HostID uuid.UUID `json:"hostID"` - DisplayHostName string `json:"displayhostname"` - Metadata map[string]interface{} `json:"metadata" yaml:"metadata"` - // The property has been named RelationshipType instead of Type to avoid collision from Type() function, which enables support for dynamic type. - // Though, the column name and the json representation is "type". - RelationshipType string `json:"type" yaml:"type" gorm:"type"` - SubType string `json:"subType" yaml:"subType" gorm:"subType"` - EvaluationQuery string `json:"evaluationQuery" yaml:"evaluationQuery" gorm:"evaluationQuery"` - Selectors []map[string]interface{} `json:"selectors" yaml:"selectors"` - CreatedAt time.Time `json:"-"` - UpdatedAt time.Time `json:"-"` -} - -type RelationshipDefinitionDB struct { - ID uuid.UUID `json:"id"` - ModelID uuid.UUID `json:"-" gorm:"index:idx_relationship_definition_dbs_model_id,column:modelID"` - TypeMeta - Metadata []byte `json:"metadata" yaml:"metadata"` - // The property has been named RelationshipType instead of Type to avoid collision from Type() function, which enables support for dynamic type. - // Though, the column name and the json representation is "type". - RelationshipType string `json:"type" yaml:"type" gorm:"type"` - SubType string `json:"subType" yaml:"subType"` - EvaluationQuery string `json:"evaluationQuery" yaml:"evaluationQuery" gorm:"evaluationQuery"` - Selectors []byte `json:"selectors" yaml:"selectors"` - CreatedAt time.Time `json:"-"` - UpdatedAt time.Time `json:"-"` -} - -// For now, only filtering by Kind and SubType are allowed. -// In the future, we will add support to query using `selectors` (using CUE) -// TODO: Add support for Model -type RelationshipFilter struct { - Kind string - Greedy bool //when set to true - instead of an exact match, kind will be prefix matched - SubType string - RelationshipType string - Version string - ModelName string - OrderOn string - Sort string //asc or desc. Default behavior is asc - Limit int //If 0 or unspecified then all records are returned and limit is not used - Offset int -} - -// Create the filter from map[string]interface{} -func (rf *RelationshipFilter) Create(m map[string]interface{}) { - if m == nil { - return - } -} -func GetMeshModelRelationship(db *database.Handler, f RelationshipFilter) (r []RelationshipDefinition, count int64) { - type componentDefinitionWithModel struct { - RelationshipDefinitionDB - ModelDB //nolint - CategoryDB - } - var componentDefinitionsWithModel []componentDefinitionWithModel - finder := db.Model(&RelationshipDefinitionDB{}). - Select("relationship_definition_dbs.*, model_dbs.*"). - Joins("JOIN model_dbs ON relationship_definition_dbs.model_id = model_dbs.id"). // - Joins("JOIN category_dbs ON model_dbs.category_id = category_dbs.id") // - if f.Kind != "" { - if f.Greedy { - finder = finder.Where("relationship_definition_dbs.kind LIKE ?", "%"+f.Kind+"%") - } else { - finder = finder.Where("relationship_definition_dbs.kind = ?", f.Kind) - } - } - - if f.RelationshipType != "" { - finder = finder.Where("relationship_definition_dbs.type = ?", f.RelationshipType) - } - - if f.SubType != "" { - finder = finder.Where("relationship_definition_dbs.sub_type = ?", f.SubType) - } - if f.ModelName != "" { - finder = finder.Where("model_dbs.name = ?", f.ModelName) - } - if f.Version != "" { - finder = finder.Where("model_dbs.version = ?", f.Version) - } - if f.OrderOn != "" { - if f.Sort == "desc" { - finder = finder.Order(clause.OrderByColumn{Column: clause.Column{Name: f.OrderOn}, Desc: true}) - } else { - finder = finder.Order(f.OrderOn) - } - } - - finder.Count(&count) - - finder = finder.Offset(f.Offset) - if f.Limit != 0 { - finder = finder.Limit(f.Limit) - } - err := finder. - Scan(&componentDefinitionsWithModel).Error - if err != nil { - fmt.Println(err.Error()) //for debugging - } - for _, cm := range componentDefinitionsWithModel { - r = append(r, cm.RelationshipDefinitionDB.GetRelationshipDefinition(cm.ModelDB.GetModel(cm.CategoryDB.GetCategory(db)))) - } - return r, count -} - -func (rdb *RelationshipDefinitionDB) GetRelationshipDefinition(m Model) (r RelationshipDefinition) { - r.ID = rdb.ID - r.TypeMeta = rdb.TypeMeta - if r.Metadata == nil { - r.Metadata = make(map[string]interface{}) - } - _ = json.Unmarshal(rdb.Metadata, &r.Metadata) - if r.Selectors == nil { - r.Selectors = []map[string]interface{}{} - } - _ = json.Unmarshal(rdb.Selectors, &r.Selectors) - r.RelationshipType = rdb.RelationshipType - r.SubType = rdb.SubType - r.Kind = rdb.Kind - r.Model = m - r.EvaluationQuery = rdb.EvaluationQuery - return -} - -func (r RelationshipDefinition) Type() types.EntityType { - return types.RelationshipDefinition -} -func (r RelationshipDefinition) GetID() uuid.UUID { - return r.ID -} - -func CreateRelationship(db *database.Handler, r RelationshipDefinition) (uuid.UUID, uuid.UUID, error) { - r.ID = uuid.New() - mid, err := CreateModel(db, r.Model) - if err != nil { - return uuid.UUID{}, uuid.UUID{}, err - } - rdb := r.GetRelationshipDefinitionDB() - rdb.ModelID = mid - err = db.Create(&rdb).Error - if err != nil { - return uuid.UUID{}, uuid.UUID{}, err - } - return r.ID, mid, err -} - -func (r *RelationshipDefinition) GetRelationshipDefinitionDB() (rdb RelationshipDefinitionDB) { - rdb.ID = r.ID - rdb.TypeMeta = r.TypeMeta - rdb.Metadata, _ = json.Marshal(r.Metadata) - rdb.Selectors, _ = json.Marshal(r.Selectors) - rdb.Kind = r.Kind - rdb.RelationshipType = r.RelationshipType - rdb.SubType = r.SubType - rdb.ModelID = r.Model.ID - rdb.EvaluationQuery = r.EvaluationQuery - return -} From ef4981fe414dd8ec4af44ded657a8df2323879a7 Mon Sep 17 00:00:00 2001 From: MUzairS15 Date: Mon, 25 Mar 2024 16:10:36 +0530 Subject: [PATCH 13/32] add getters to entity filter Signed-off-by: MUzairS15 --- models/meshmodel/core/v1alpha2/relationship.go | 12 ++++++------ models/meshmodel/core/v1beta1/component.go | 12 ++++++------ models/meshmodel/entity/types.go | 2 +- 3 files changed, 13 insertions(+), 13 deletions(-) diff --git a/models/meshmodel/core/v1alpha2/relationship.go b/models/meshmodel/core/v1alpha2/relationship.go index 6f8ad3f2..21e3fd6e 100644 --- a/models/meshmodel/core/v1alpha2/relationship.go +++ b/models/meshmodel/core/v1alpha2/relationship.go @@ -74,11 +74,11 @@ func (r RelationshipDefinition) GetID() uuid.UUID { return r.ID } -func (r *RelationshipDefinition) Get(db *database.Handler, f entity.Filter) ([]RelationshipDefinition, int64, int, error) { - relationshipFilter, err := utils.Cast[RelationshipFilter](f) - if err != nil { - return nil, 0, 0, err - } +func (relationshipFilter *RelationshipFilter) Get(db *database.Handler, f entity.Filter) ([]RelationshipDefinition, int64, int, error) { + // relationshipFilter, err := utils.Cast[RelationshipFilter](f) + // if err != nil { + // return nil, 0, 0, err + // } var relationshipDefinitionsWithModel []relationshipDefinitionWithModel finder := db.Model(&RelationshipDefinitionDB{}). @@ -121,7 +121,7 @@ func (r *RelationshipDefinition) Get(db *database.Handler, f entity.Filter) ([]R if relationshipFilter.Limit != 0 { finder = finder.Limit(relationshipFilter.Limit) } - err = finder. + err := finder. Scan(&relationshipDefinitionsWithModel).Error if err != nil { fmt.Println(err.Error()) //for debugging diff --git a/models/meshmodel/core/v1beta1/component.go b/models/meshmodel/core/v1beta1/component.go index ad0a1b81..d4388997 100644 --- a/models/meshmodel/core/v1beta1/component.go +++ b/models/meshmodel/core/v1beta1/component.go @@ -91,11 +91,11 @@ func (c ComponentDefinition) GetID() uuid.UUID { return c.ID } -func (c *ComponentDefinition) Get(db *database.Handler, f entity.Filter) ([]ComponentDefinition, int64, int, error) { - componentFilter, err := utils.Cast[ComponentFilter](f) - if err != nil { - return nil, 0, 0, err - } +func (componentFilter *ComponentFilter) Get(db *database.Handler) ([]ComponentDefinition, int64, int, error) { + // componentFilter, err := utils.Cast[ComponentFilter](f) + // if err != nil { + // return nil, 0, 0, err + // } countUniqueComponents := func(components []componentDefinitionWithModel) int { set := make(map[string]struct{}) @@ -156,7 +156,7 @@ func (c *ComponentDefinition) Get(db *database.Handler, f entity.Filter) ([]Comp if componentFilter.Limit != 0 { finder = finder.Limit(componentFilter.Limit) } - err = finder. + err := finder. Scan(&componentDefinitionsWithModel).Error if err != nil { fmt.Println(err.Error()) //for debugging diff --git a/models/meshmodel/entity/types.go b/models/meshmodel/entity/types.go index f30ad7c3..b764a4e3 100644 --- a/models/meshmodel/entity/types.go +++ b/models/meshmodel/entity/types.go @@ -17,10 +17,10 @@ const ( // Each entity will have it's own Filter implementation via which it exposes the nobs and dials to fetch entities type Filter interface { Create(map[string]interface{}) + Get(db *database.Handler, f Filter) (entities []Entity, count int64, unique int, err error) } type Entity interface { Status Create(db *database.Handler) (entityID uuid.UUID, err error) - Get(db *database.Handler, f Filter) (entities []Entity, count int64, unique int,err error) } From d56ae45d2509c30919ceecf38a31973cd62dd877 Mon Sep 17 00:00:00 2001 From: MUzairS15 Date: Mon, 25 Mar 2024 16:22:22 +0530 Subject: [PATCH 14/32] caategory Signed-off-by: MUzairS15 --- .../core/policies/rego_policy_relationship.go | 4 +- models/meshmodel/core/v1beta1/category.go | 53 +++++++++++++++++++ models/meshmodel/core/v1beta1/component.go | 1 - models/meshmodel/entity/types.go | 4 ++ 4 files changed, 59 insertions(+), 3 deletions(-) diff --git a/models/meshmodel/core/policies/rego_policy_relationship.go b/models/meshmodel/core/policies/rego_policy_relationship.go index 91c36701..8ff09a62 100644 --- a/models/meshmodel/core/policies/rego_policy_relationship.go +++ b/models/meshmodel/core/policies/rego_policy_relationship.go @@ -4,7 +4,7 @@ import ( "context" "fmt" - "github.com/layer5io/meshkit/models/meshmodel/core/v1alpha1" + "github.com/layer5io/meshkit/models/meshmodel/core/v1alpha2" "github.com/layer5io/meshkit/models/meshmodel/registry" "github.com/layer5io/meshkit/utils" "github.com/open-policy-agent/opa/rego" @@ -26,7 +26,7 @@ func NewRegoInstance(policyDir string, regManager *registry.RegistryManager) (*R var store storage.Store ctx := context.Background() - registeredRelationships, _, _ := regManager.GetEntities(&v1alpha1.RelationshipFilter{}) + registeredRelationships, _, _ := regManager.GetEntities(&v1alpha2.RelationshipFilter{}) if len(registeredRelationships) > 0 { data := map[string]interface{}{ diff --git a/models/meshmodel/core/v1beta1/category.go b/models/meshmodel/core/v1beta1/category.go index 32f9dc02..ef32f143 100644 --- a/models/meshmodel/core/v1beta1/category.go +++ b/models/meshmodel/core/v1beta1/category.go @@ -8,6 +8,7 @@ import ( "github.com/layer5io/meshkit/database" "github.com/layer5io/meshkit/models/meshmodel/entity" "gorm.io/gorm" + "gorm.io/gorm/clause" ) var categoryCreationLock sync.Mutex //Each model will perform a check and if the category already doesn't exist, it will create a category. This lock will make sure that there are no race conditions. @@ -37,6 +38,58 @@ type CategoryFilter struct { // "Uncategorized" is assigned when Category is empty in the component definitions. const DefaultCategory = "Uncategorized" +// Create the filter from map[string]interface{} +func (cf *CategoryFilter) Create(m map[string]interface{}) { + if m == nil { + return + } + cf.Name = m["name"].(string) +} + +func (cf *CategoryFilter) Get(db *database.Handler) ([]Category, int64, int, error) { + var catdb []CategoryDB + var cat []Category + finder := db.Model(&catdb) + + // total count before pagination + var count int64 + + if cf.Name != "" { + if cf.Greedy { + finder = finder.Where("name LIKE ?", "%"+cf.Name+"%") + } else { + finder = finder.Where("name = ?", cf.Name) + } + } + if cf.OrderOn != "" { + if cf.Sort == "desc" { + finder = finder.Order(clause.OrderByColumn{Column: clause.Column{Name: cf.OrderOn}, Desc: true}) + } else { + finder = finder.Order(cf.OrderOn) + } + } + + finder.Count(&count) + + if cf.Limit != 0 { + finder = finder.Limit(cf.Limit) + } + if cf.Offset != 0 { + finder = finder.Offset(cf.Offset) + } + + if count == 0 { + finder.Count(&count) + } + + _ = finder.Find(&catdb).Error + for _, c := range catdb { + cat = append(cat, c.GetCategory(db)) + } + // duplicate category ? + return cat, count, int(count), nil +} + func (cat *Category) Create(db *database.Handler) (uuid.UUID, error) { if cat.Name == "" { cat.Name = DefaultCategory diff --git a/models/meshmodel/core/v1beta1/component.go b/models/meshmodel/core/v1beta1/component.go index d4388997..4b244f65 100644 --- a/models/meshmodel/core/v1beta1/component.go +++ b/models/meshmodel/core/v1beta1/component.go @@ -7,7 +7,6 @@ import ( "github.com/layer5io/meshkit/database" "github.com/layer5io/meshkit/models/meshmodel/entity" - "github.com/layer5io/meshkit/models/meshmodel/registry" "github.com/layer5io/meshkit/utils" "github.com/google/uuid" diff --git a/models/meshmodel/entity/types.go b/models/meshmodel/entity/types.go index b764a4e3..e79f22de 100644 --- a/models/meshmodel/entity/types.go +++ b/models/meshmodel/entity/types.go @@ -21,6 +21,10 @@ type Filter interface { } type Entity interface { +// Entity is referred as any type of schema managed by the registry +// ComponentDefinitions and PolicyDefinitions are examples of entities + Type() EntityType + GetID() uuid.UUID Status Create(db *database.Handler) (entityID uuid.UUID, err error) } From 510b514f5da168dfddba5ed325cb6673986e1ee5 Mon Sep 17 00:00:00 2001 From: MUzairS15 Date: Tue, 26 Mar 2024 16:22:15 +0530 Subject: [PATCH 15/32] finalize changes and package structure: Signed-off-by: MUzairS15 --- generators/artifacthub/package.go | 8 +- generators/artifacthub/package_manager.go | 2 +- generators/artifacthub/package_test.go | 2 +- generators/generator.go | 2 +- generators/github/git_repo.go | 3 +- generators/github/package.go | 8 +- generators/github/package_manager.go | 2 +- generators/github/package_test.go | 4 +- generators/github/scheme_interface.go | 2 +- generators/github/url.go | 2 +- {models => generators/models}/interfaces.go | 4 +- .../core/policies/rego_policy_relationship.go | 6 +- .../meshmodel/core/v1alpha2/relationship.go | 93 +----- .../core/v1beta1/application_component.go | 66 ++++ models/meshmodel/core/v1beta1/category.go | 62 ---- models/meshmodel/core/v1beta1/component.go | 117 +------ models/meshmodel/core/v1beta1/host.go | 1 - .../{registry => core/v1beta1}/hostv1beta1.go | 53 ++- models/meshmodel/core/v1beta1/models.go | 50 +-- models/meshmodel/core/v1beta1/policy.go | 94 +++++- models/meshmodel/entity/error.go | 2 +- models/meshmodel/entity/status.go | 26 +- models/meshmodel/entity/types.go | 3 +- models/meshmodel/registry/category_filter.go | 68 ++++ models/meshmodel/registry/component_filter.go | 131 ++++++++ models/meshmodel/registry/error.go | 3 +- models/meshmodel/registry/host.go | 238 +++++++------- models/meshmodel/registry/model_filter.go | 171 ++++++++++ models/meshmodel/registry/policy_filter.go | 59 ++++ models/meshmodel/registry/registry.go | 307 ++++++------------ .../meshmodel/registry/relationship_filter.go | 108 ++++++ .../core/v1alpha1/application_component.go | 110 +++---- utils/component/generator.go | 16 +- utils/component/generator_test.go | 26 +- 34 files changed, 1100 insertions(+), 749 deletions(-) rename {models => generators/models}/interfaces.go (80%) create mode 100644 models/meshmodel/core/v1beta1/application_component.go delete mode 100644 models/meshmodel/core/v1beta1/host.go rename models/meshmodel/{registry => core/v1beta1}/hostv1beta1.go (62%) create mode 100644 models/meshmodel/registry/category_filter.go create mode 100644 models/meshmodel/registry/component_filter.go create mode 100644 models/meshmodel/registry/model_filter.go create mode 100644 models/meshmodel/registry/policy_filter.go create mode 100644 models/meshmodel/registry/relationship_filter.go diff --git a/generators/artifacthub/package.go b/generators/artifacthub/package.go index f37e94b2..e9dee8bf 100644 --- a/generators/artifacthub/package.go +++ b/generators/artifacthub/package.go @@ -7,7 +7,7 @@ import ( "strings" "time" - "github.com/layer5io/meshkit/models/meshmodel/core/v1alpha1" + "github.com/layer5io/meshkit/models/meshmodel/core/v1beta1" "github.com/layer5io/meshkit/utils" "github.com/layer5io/meshkit/utils/component" "github.com/layer5io/meshkit/utils/manifests" @@ -36,8 +36,8 @@ func (pkg AhPackage) GetVersion() string { return pkg.Version } -func (pkg AhPackage) GenerateComponents() ([]v1alpha1.ComponentDefinition, error) { - components := make([]v1alpha1.ComponentDefinition, 0) +func (pkg AhPackage) GenerateComponents() ([]v1beta1.ComponentDefinition, error) { + components := make([]v1beta1.ComponentDefinition, 0) // TODO: Move this to the configuration crds, err := manifests.GetCrdsFromHelm(pkg.ChartUrl) if err != nil { @@ -57,7 +57,7 @@ func (pkg AhPackage) GenerateComponents() ([]v1alpha1.ComponentDefinition, error comp.Model.Metadata["source_uri"] = pkg.ChartUrl comp.Model.Version = pkg.Version comp.Model.Name = pkg.Name - comp.Model.Category = v1alpha1.Category{ + comp.Model.Category = v1beta1.Category{ Name: "", } comp.Model.DisplayName = manifests.FormatToReadableString(comp.Model.Name) diff --git a/generators/artifacthub/package_manager.go b/generators/artifacthub/package_manager.go index f75a239d..3eead188 100644 --- a/generators/artifacthub/package_manager.go +++ b/generators/artifacthub/package_manager.go @@ -3,7 +3,7 @@ package artifacthub import ( "fmt" - "github.com/layer5io/meshkit/models" + "github.com/layer5io/meshkit/generators/models" ) type ArtifactHubPackageManager struct { diff --git a/generators/artifacthub/package_test.go b/generators/artifacthub/package_test.go index 11bfa692..c5994aca 100644 --- a/generators/artifacthub/package_test.go +++ b/generators/artifacthub/package_test.go @@ -48,7 +48,7 @@ func TestGetChartUrl(t *testing.T) { } byt, _ := json.MarshalIndent(comp, "", "") - f, err := os.Create(dirName + "/" + comp.Kind + ".json") + f, err := os.Create(dirName + "/" + comp.Component.Kind + ".json") if err != nil { t.Errorf("error create file : %v", err) continue diff --git a/generators/generator.go b/generators/generator.go index 2ea8a050..965621b9 100644 --- a/generators/generator.go +++ b/generators/generator.go @@ -5,7 +5,7 @@ import ( "github.com/layer5io/meshkit/generators/artifacthub" "github.com/layer5io/meshkit/generators/github" - "github.com/layer5io/meshkit/models" + "github.com/layer5io/meshkit/generators/models" "github.com/layer5io/meshkit/utils" ) diff --git a/generators/github/git_repo.go b/generators/github/git_repo.go index 69cfd284..ac162eed 100644 --- a/generators/github/git_repo.go +++ b/generators/github/git_repo.go @@ -7,8 +7,7 @@ import ( "os" "path/filepath" "strings" - - "github.com/layer5io/meshkit/models" + "github.com/layer5io/meshkit/generators/models" "github.com/layer5io/meshkit/utils" "github.com/layer5io/meshkit/utils/helm" "github.com/layer5io/meshkit/utils/walker" diff --git a/generators/github/package.go b/generators/github/package.go index ba570eec..53411699 100644 --- a/generators/github/package.go +++ b/generators/github/package.go @@ -4,7 +4,7 @@ import ( "bytes" "os" - "github.com/layer5io/meshkit/models/meshmodel/core/v1alpha1" + "github.com/layer5io/meshkit/models/meshmodel/core/v1beta1" "github.com/layer5io/meshkit/utils" "github.com/layer5io/meshkit/utils/component" "github.com/layer5io/meshkit/utils/manifests" @@ -23,8 +23,8 @@ func (gp GitHubPackage) GetVersion() string { return gp.version } -func (gp GitHubPackage) GenerateComponents() ([]v1alpha1.ComponentDefinition, error) { - components := make([]v1alpha1.ComponentDefinition, 0) +func (gp GitHubPackage) GenerateComponents() ([]v1beta1.ComponentDefinition, error) { + components := make([]v1beta1.ComponentDefinition, 0) data, err := os.ReadFile(gp.filePath) if err != nil { @@ -49,7 +49,7 @@ func (gp GitHubPackage) GenerateComponents() ([]v1alpha1.ComponentDefinition, er comp.Model.Metadata["source_uri"] = gp.SourceURL comp.Model.Version = gp.version comp.Model.Name = gp.Name - comp.Model.Category = v1alpha1.Category{ + comp.Model.Category = v1beta1.Category{ Name: "", } comp.Model.DisplayName = manifests.FormatToReadableString(comp.Model.Name) diff --git a/generators/github/package_manager.go b/generators/github/package_manager.go index 98d3d7b9..f26fc198 100644 --- a/generators/github/package_manager.go +++ b/generators/github/package_manager.go @@ -3,7 +3,7 @@ package github import ( "net/url" - "github.com/layer5io/meshkit/models" + "github.com/layer5io/meshkit/generators/models" "github.com/layer5io/meshkit/utils/walker" ) diff --git a/generators/github/package_test.go b/generators/github/package_test.go index 12447718..14fad58d 100644 --- a/generators/github/package_test.go +++ b/generators/github/package_test.go @@ -103,9 +103,9 @@ func TestGenerateCompFromGitHub(t *testing.T) { } byt, _ := json.MarshalIndent(comp, "", "") - f, err := os.Create(fmt.Sprintf("%s/%s%s", dirName, comp.Kind, ".json")) + f, err := os.Create(fmt.Sprintf("%s/%s%s", dirName, comp.Component.Kind, ".json")) if err != nil { - t.Errorf("error creating file for %s: %v", comp.Kind, err) + t.Errorf("error creating file for %s: %v", comp.Component.Kind, err) continue } _, _ = f.Write(byt) diff --git a/generators/github/scheme_interface.go b/generators/github/scheme_interface.go index 177e7e98..36316a8a 100644 --- a/generators/github/scheme_interface.go +++ b/generators/github/scheme_interface.go @@ -3,7 +3,7 @@ package github import ( "net/url" - "github.com/layer5io/meshkit/models" + "github.com/layer5io/meshkit/generators/models" ) type DownloaderScheme interface { diff --git a/generators/github/url.go b/generators/github/url.go index 3b63b2aa..89541251 100644 --- a/generators/github/url.go +++ b/generators/github/url.go @@ -9,7 +9,7 @@ import ( "path/filepath" "strings" - "github.com/layer5io/meshkit/models" + "github.com/layer5io/meshkit/generators/models" "github.com/layer5io/meshkit/utils" "github.com/layer5io/meshkit/utils/helm" ) diff --git a/models/interfaces.go b/generators/models/interfaces.go similarity index 80% rename from models/interfaces.go rename to generators/models/interfaces.go index 3c89fd54..7d68c209 100644 --- a/models/interfaces.go +++ b/generators/models/interfaces.go @@ -1,6 +1,6 @@ package models -import "github.com/layer5io/meshkit/models/meshmodel/core/v1alpha1" +import "github.com/layer5io/meshkit/models/meshmodel/core/v1beta1" // anything that can be validated is a Validator type Validator interface { @@ -11,7 +11,7 @@ type Validator interface { // system's capabilities in Meshery // A Package should have all the information that we need to generate the components type Package interface { - GenerateComponents() ([]v1alpha1.ComponentDefinition, error) + GenerateComponents() ([]v1beta1.ComponentDefinition, error) GetVersion() string } diff --git a/models/meshmodel/core/policies/rego_policy_relationship.go b/models/meshmodel/core/policies/rego_policy_relationship.go index 8ff09a62..a7f0f1f0 100644 --- a/models/meshmodel/core/policies/rego_policy_relationship.go +++ b/models/meshmodel/core/policies/rego_policy_relationship.go @@ -4,7 +4,6 @@ import ( "context" "fmt" - "github.com/layer5io/meshkit/models/meshmodel/core/v1alpha2" "github.com/layer5io/meshkit/models/meshmodel/registry" "github.com/layer5io/meshkit/utils" "github.com/open-policy-agent/opa/rego" @@ -26,7 +25,10 @@ func NewRegoInstance(policyDir string, regManager *registry.RegistryManager) (*R var store storage.Store ctx := context.Background() - registeredRelationships, _, _ := regManager.GetEntities(&v1alpha2.RelationshipFilter{}) + registeredRelationships, _, _, err := regManager.GetEntities(®istry.RelationshipFilter{}) + if err != nil { + return nil, err + } if len(registeredRelationships) > 0 { data := map[string]interface{}{ diff --git a/models/meshmodel/core/v1alpha2/relationship.go b/models/meshmodel/core/v1alpha2/relationship.go index 21e3fd6e..8eabea0a 100644 --- a/models/meshmodel/core/v1alpha2/relationship.go +++ b/models/meshmodel/core/v1alpha2/relationship.go @@ -9,9 +9,7 @@ import ( "github.com/layer5io/meshkit/database" "github.com/layer5io/meshkit/models/meshmodel/core/v1beta1" "github.com/layer5io/meshkit/models/meshmodel/entity" - "github.com/layer5io/meshkit/models/meshmodel/registry" "github.com/layer5io/meshkit/utils" - "gorm.io/gorm/clause" ) // Remove additional DB structs from all entites they are used just for marshaling & unmarshlling and tarcking modelID vs model(modelID when referenced in DB and full model when sending data, use foregin key references tag of gorm for that as used in MeshSync tables) when saving retrivng from database, insted use gorm' serailization tag as used in events. @@ -43,30 +41,6 @@ type RelationshipDefinitionDB struct { Selectors []byte `json:"selectors" yaml:"selectors"` } -type relationshipDefinitionWithModel struct { - RelationshipDefinitionDB - ModelDB v1beta1.ModelDB - // acoount for overridn fields - // v1beta1.ModelDB.Version `json:"modelVersion"` - CategoryDB v1beta1.CategoryDB -} - -// For now, only filtering by Kind and SubType are allowed. -// In the future, we will add support to query using `selectors` (using CUE) -// TODO: Add support for Model -type RelationshipFilter struct { - Kind string - Greedy bool //when set to true - instead of an exact match, kind will be prefix matched - SubType string - RelationshipType string - Version string - ModelName string - OrderOn string - Sort string //asc or desc. Default behavior is asc - Limit int //If 0 or unspecified then all records are returned and limit is not used - Offset int -} - func (r RelationshipDefinition) Type() entity.EntityType { return entity.RelationshipDefinition } @@ -74,69 +48,8 @@ func (r RelationshipDefinition) GetID() uuid.UUID { return r.ID } -func (relationshipFilter *RelationshipFilter) Get(db *database.Handler, f entity.Filter) ([]RelationshipDefinition, int64, int, error) { - // relationshipFilter, err := utils.Cast[RelationshipFilter](f) - // if err != nil { - // return nil, 0, 0, err - // } - - var relationshipDefinitionsWithModel []relationshipDefinitionWithModel - finder := db.Model(&RelationshipDefinitionDB{}). - Select("relationship_definition_dbs.*, model_dbs.*"). - Joins("JOIN model_dbs ON relationship_definition_dbs.model_id = model_dbs.id"). // - Joins("JOIN category_dbs ON model_dbs.category_id = category_dbs.id") // - if relationshipFilter.Kind != "" { - if relationshipFilter.Greedy { - finder = finder.Where("relationship_definition_dbs.kind LIKE ?", "%"+relationshipFilter.Kind+"%") - } else { - finder = finder.Where("relationship_definition_dbs.kind = ?", relationshipFilter.Kind) - } - } - - if relationshipFilter.RelationshipType != "" { - finder = finder.Where("relationship_definition_dbs.type = ?", relationshipFilter.RelationshipType) - } - - if relationshipFilter.SubType != "" { - finder = finder.Where("relationship_definition_dbs.sub_type = ?", relationshipFilter.SubType) - } - if relationshipFilter.ModelName != "" { - finder = finder.Where("model_dbs.name = ?", relationshipFilter.ModelName) - } - if relationshipFilter.Version != "" { - finder = finder.Where("model_dbs.version = ?", relationshipFilter.Version) - } - if relationshipFilter.OrderOn != "" { - if relationshipFilter.Sort == "desc" { - finder = finder.Order(clause.OrderByColumn{Column: clause.Column{Name: relationshipFilter.OrderOn}, Desc: true}) - } else { - finder = finder.Order(relationshipFilter.OrderOn) - } - } - - var count int64 - finder.Count(&count) - - finder = finder.Offset(relationshipFilter.Offset) - if relationshipFilter.Limit != 0 { - finder = finder.Limit(relationshipFilter.Limit) - } - err := finder. - Scan(&relationshipDefinitionsWithModel).Error - if err != nil { - fmt.Println(err.Error()) //for debugging - } - defs := make([]RelationshipDefinition, len(relationshipDefinitionsWithModel)) - - // remove this when reldef and reldefdb struct is consolidated. - for _, cm := range relationshipDefinitionsWithModel { - // Ensure correct reg is passed, rn it is dummy for sake of testing. - // In the first query above where we do seelection i think there changes will be requrired, an when that two def and defDB structs are consolidated, using association and preload i think we can do. - reg := registry.Hostv1beta1{} - defs = append(defs, cm.RelationshipDefinitionDB.GetRelationshipDefinition(cm.ModelDB.GetModel(cm.CategoryDB.GetCategory(db), reg))) - } - // Should have count unique relationships (by model version, model name, and relationship's kind, type, subtype, version) - return defs, count, int(count), nil +func (r *RelationshipDefinition) GetEntityDetail() string { + return fmt.Sprintf("type: %s, definition version: %s, kind: %s, model: %s, version: %s", r.Type(), r.Version, r.Kind, r.Model.Name, r.Model.Version) } func (r *RelationshipDefinition) Create(db *database.Handler) (uuid.UUID, error) { @@ -154,7 +67,7 @@ func (r *RelationshipDefinition) Create(db *database.Handler) (uuid.UUID, error) return r.ID, err } -func (m *RelationshipDefinition) UpdateStatus(db database.Handler, status entity.EntityStatus) error { +func (m *RelationshipDefinition) UpdateStatus(db *database.Handler, status entity.EntityStatus) error { return nil } diff --git a/models/meshmodel/core/v1beta1/application_component.go b/models/meshmodel/core/v1beta1/application_component.go new file mode 100644 index 00000000..f6778c8d --- /dev/null +++ b/models/meshmodel/core/v1beta1/application_component.go @@ -0,0 +1,66 @@ +package v1beta1 + +import ( + "fmt" + "strings" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// Component is the structure for the core OAM Application Component +type Component struct { + metav1.TypeMeta `json:",inline"` + metav1.ObjectMeta `json:"metadata,omitempty"` + + Spec ComponentSpec `json:"spec,omitempty"` +} + +// ComponentSpec is the structure for the core OAM Application Component Spec +type ComponentSpec struct { + Type string `json:"type"` + Version string `json:"version"` + APIVersion string `json:"apiVersion"` + Model string `json:"model"` + Settings map[string]interface{} `json:"settings"` + Parameters []ComponentParameter `json:"parameters"` +} + +// ComponentParameter is the structure for the core OAM Application Component +// Paramater +type ComponentParameter struct { + Name string `json:"name"` + FieldPaths []string `json:"fieldPaths"` + Required *bool `json:"required,omitempty"` + Description *string `json:"description,omitempty"` +} + +const MesheryAnnotationPrefix = "design.meshmodel.io" + +func GetAPIVersionFromComponent(comp Component) string { + return comp.Annotations[MesheryAnnotationPrefix+".k8s.APIVersion"] +} + +func GetKindFromComponent(comp Component) string { + kind := strings.TrimPrefix(comp.Annotations[MesheryAnnotationPrefix+".k8s.Kind"], "/") + return kind +} + +func GetAnnotationsForWorkload(w ComponentDefinition) map[string]string { + res := map[string]string{} + + for key, val := range w.Metadata { + if v, ok := val.(string); ok { + res[strings.ReplaceAll(fmt.Sprintf("%s.%s", MesheryAnnotationPrefix, key), " ", "")] = v + } + } + sourceURI, ok := w.Model.Metadata["source_uri"].(string) + if ok { + res[fmt.Sprintf("%s.model.source_uri", MesheryAnnotationPrefix)] = sourceURI + } + res[fmt.Sprintf("%s.model.name", MesheryAnnotationPrefix)] = w.Model.Name + res[fmt.Sprintf("%s.k8s.APIVersion", MesheryAnnotationPrefix)] = w.Component.Version + res[fmt.Sprintf("%s.k8s.Kind", MesheryAnnotationPrefix)] = w.Component.Kind + res[fmt.Sprintf("%s.model.version", MesheryAnnotationPrefix)] = w.Model.Version + res[fmt.Sprintf("%s.model.category", MesheryAnnotationPrefix)] = w.Model.Category.Name + return res +} diff --git a/models/meshmodel/core/v1beta1/category.go b/models/meshmodel/core/v1beta1/category.go index ef32f143..410a08c9 100644 --- a/models/meshmodel/core/v1beta1/category.go +++ b/models/meshmodel/core/v1beta1/category.go @@ -8,7 +8,6 @@ import ( "github.com/layer5io/meshkit/database" "github.com/layer5io/meshkit/models/meshmodel/entity" "gorm.io/gorm" - "gorm.io/gorm/clause" ) var categoryCreationLock sync.Mutex //Each model will perform a check and if the category already doesn't exist, it will create a category. This lock will make sure that there are no race conditions. @@ -26,70 +25,9 @@ type CategoryDB struct { Metadata []byte `json:"categoryMetadata" gorm:"categoryMetadata"` } -type CategoryFilter struct { - Name string - OrderOn string - Greedy bool - Sort string //asc or desc. Default behavior is asc - Limit int //If 0 or unspecified then all records are returned and limit is not used - Offset int -} - // "Uncategorized" is assigned when Category is empty in the component definitions. const DefaultCategory = "Uncategorized" -// Create the filter from map[string]interface{} -func (cf *CategoryFilter) Create(m map[string]interface{}) { - if m == nil { - return - } - cf.Name = m["name"].(string) -} - -func (cf *CategoryFilter) Get(db *database.Handler) ([]Category, int64, int, error) { - var catdb []CategoryDB - var cat []Category - finder := db.Model(&catdb) - - // total count before pagination - var count int64 - - if cf.Name != "" { - if cf.Greedy { - finder = finder.Where("name LIKE ?", "%"+cf.Name+"%") - } else { - finder = finder.Where("name = ?", cf.Name) - } - } - if cf.OrderOn != "" { - if cf.Sort == "desc" { - finder = finder.Order(clause.OrderByColumn{Column: clause.Column{Name: cf.OrderOn}, Desc: true}) - } else { - finder = finder.Order(cf.OrderOn) - } - } - - finder.Count(&count) - - if cf.Limit != 0 { - finder = finder.Limit(cf.Limit) - } - if cf.Offset != 0 { - finder = finder.Offset(cf.Offset) - } - - if count == 0 { - finder.Count(&count) - } - - _ = finder.Find(&catdb).Error - for _, c := range catdb { - cat = append(cat, c.GetCategory(db)) - } - // duplicate category ? - return cat, count, int(count), nil -} - func (cat *Category) Create(db *database.Handler) (uuid.UUID, error) { if cat.Name == "" { cat.Name = DefaultCategory diff --git a/models/meshmodel/core/v1beta1/component.go b/models/meshmodel/core/v1beta1/component.go index 4b244f65..f0559c84 100644 --- a/models/meshmodel/core/v1beta1/component.go +++ b/models/meshmodel/core/v1beta1/component.go @@ -35,11 +35,6 @@ type component struct { Schema string `json:"schema,omitempty" yaml:"schema"` } -type componentDefinitionWithModel struct { - ComponentDefinitionDB - ModelDB ModelDB // acoount for overridn fields - CategoryDB CategoryDB -} // swagger:response ComponentDefinition // use NewComponent function for instantiating @@ -66,22 +61,6 @@ type ComponentDefinitionDB struct { Component component `json:"component,omitempty" yaml:"component" gorm:"component"` } -type ComponentFilter struct { - Name string - APIVersion string - Greedy bool //when set to true - instead of an exact match, name will be matched as a substring - Trim bool //when set to true - the schema is not returned - DisplayName string - ModelName string - CategoryName string - Version string - Sort string //asc or desc. Default behavior is asc - OrderOn string - Limit int //If 0 or unspecified then all records are returned and limit is not used - Offset int - Annotations string //When this query parameter is "true", only components with the "isAnnotation" property set to true are returned. When this query parameter is "false", all components except those considered to be annotation components are returned. Any other value of the query parameter results in both annotations as well as non-annotation models being returned. -} - func (c ComponentDefinition) Type() entity.EntityType { return entity.ComponentDefinition } @@ -90,96 +69,20 @@ func (c ComponentDefinition) GetID() uuid.UUID { return c.ID } -func (componentFilter *ComponentFilter) Get(db *database.Handler) ([]ComponentDefinition, int64, int, error) { - // componentFilter, err := utils.Cast[ComponentFilter](f) - // if err != nil { - // return nil, 0, 0, err - // } - - countUniqueComponents := func(components []componentDefinitionWithModel) int { - set := make(map[string]struct{}) - for _, compWithModel := range components { - key := compWithModel.Component.Kind + "@" + compWithModel.Component.Version + "@" + compWithModel.ModelDB.Name + "@" + compWithModel.ModelDB.Version - if _, ok := set[key]; !ok { - set[key] = struct{}{} - } - } - return len(set) - } - var componentDefinitionsWithModel []componentDefinitionWithModel - finder := db.Model(&ComponentDefinitionDB{}). - Select("component_definition_dbs.*, model_dbs.*,category_dbs.*"). - Joins("JOIN model_dbs ON component_definition_dbs.model_id = model_dbs.id"). - Joins("JOIN category_dbs ON model_dbs.category_id = category_dbs.id") // - - if componentFilter.Greedy { - if componentFilter.Name != "" && componentFilter.DisplayName != "" { - finder = finder.Where("component_definition_dbs.component->>'kind' LIKE ? OR display_name LIKE ?", "%"+componentFilter.Name+"%", componentFilter.DisplayName+"%") - } else if componentFilter.Name != "" { - finder = finder.Where("component_definition_dbs.component->>'kind' LIKE ?", "%"+componentFilter.Name+"%") - } else if componentFilter.DisplayName != "" { - finder = finder.Where("component_definition_dbs.display_name LIKE ?", "%"+componentFilter.DisplayName+"%") - } - } else { - if componentFilter.Name != "" { - finder = finder.Where("component_definition_dbs.component->>'kind' = ?", componentFilter.Name) - } - if componentFilter.DisplayName != "" { - finder = finder.Where("component_definition_dbs.display_name = ?", componentFilter.DisplayName) - } - } - - if componentFilter.ModelName != "" && componentFilter.ModelName != "all" { - finder = finder.Where("model_dbs.name = ?", componentFilter.ModelName) - } - - if componentFilter.Annotations == "true" { - finder = finder.Where("component_definition_dbs.metadata->>'isAnnotation' = true") - } else if componentFilter.Annotations == "false" { - finder = finder.Where("component_definition_dbs.metadata->>'isAnnotation' = false") - } +func (c *ComponentDefinition) GetEntityDetail() string { + return fmt.Sprintf("type: %s, definition version: %s, name: %s, model: %s, version: %s", c.Type(), c.Version, c.DisplayName, c.Model.Name, c.Model.Version) +} - if componentFilter.APIVersion != "" { - finder = finder.Where("component_definition_dbs.component->>'version' = ?", componentFilter.APIVersion) - } - if componentFilter.CategoryName != "" { - finder = finder.Where("category_dbs.name = ?", componentFilter.CategoryName) - } - if componentFilter.Version != "" { - finder = finder.Where("model_dbs.version = ?", componentFilter.Version) - } +func (c *ComponentDefinition) Create(db *database.Handler) (uuid.UUID, error) { + c.ID = uuid.New() - var count int64 - finder.Count(&count) - finder = finder.Offset(componentFilter.Offset) - if componentFilter.Limit != 0 { - finder = finder.Limit(componentFilter.Limit) - } - err := finder. - Scan(&componentDefinitionsWithModel).Error - if err != nil { - fmt.Println(err.Error()) //for debugging - } + isAnnotation, _ := c.Metadata["isAnnotation"].(bool) - defs := make([]ComponentDefinition, len(componentDefinitionsWithModel)) - // remove this when compoentdef and componetdefdb struct is consolidated. - for _, cm := range componentDefinitionsWithModel { - if componentFilter.Trim { - cm.Component.Schema = "" - } - // Ensure correct reg is passed, rn it is dummy for sake of testing. - // In the first query above where we do seelection i think there changes will be requrired, an when that two def and defDB structs are consolidated, using association and preload i think we can do. - reg := registry.Hostv1beta1{} - defs = append(defs, cm.ComponentDefinitionDB.GetComponentDefinition(cm.ModelDB.GetModel(cm.CategoryDB.GetCategory(db), reg))) + if c.Component.Schema == "" && !isAnnotation { //For components which has an empty schema and is not an annotation, return error + // return ErrEmptySchema() + return uuid.Nil, nil } - unique := countUniqueComponents(componentDefinitionsWithModel) - - return defs, count, unique, nil -} - -func (c *ComponentDefinition) Create(db *database.Handler) (uuid.UUID, error) { - c.ID = uuid.New() mid, err := c.Model.Create(db) if err != nil { return uuid.UUID{}, err @@ -194,7 +97,7 @@ func (c *ComponentDefinition) Create(db *database.Handler) (uuid.UUID, error) { return c.ID, err } -func (m *ComponentDefinition) UpdateStatus(db database.Handler, status entity.EntityStatus) error { +func (m *ComponentDefinition) UpdateStatus(db *database.Handler, status entity.EntityStatus) error { return nil } diff --git a/models/meshmodel/core/v1beta1/host.go b/models/meshmodel/core/v1beta1/host.go deleted file mode 100644 index fd8416ce..00000000 --- a/models/meshmodel/core/v1beta1/host.go +++ /dev/null @@ -1 +0,0 @@ -package v1beta1 \ No newline at end of file diff --git a/models/meshmodel/registry/hostv1beta1.go b/models/meshmodel/core/v1beta1/hostv1beta1.go similarity index 62% rename from models/meshmodel/registry/hostv1beta1.go rename to models/meshmodel/core/v1beta1/hostv1beta1.go index dde23631..6d7ec968 100644 --- a/models/meshmodel/registry/hostv1beta1.go +++ b/models/meshmodel/core/v1beta1/hostv1beta1.go @@ -1,4 +1,4 @@ -package registry +package v1beta1 import ( "encoding/json" @@ -8,7 +8,6 @@ import ( "github.com/google/uuid" "github.com/layer5io/meshkit/database" - "github.com/layer5io/meshkit/models/oam/core/v1alpha1" "github.com/layer5io/meshkit/utils/kubernetes" "gorm.io/gorm" ) @@ -22,7 +21,41 @@ type Hostv1beta1 struct { Metadata string CreatedAt time.Time `json:"-"` UpdatedAt time.Time `json:"-"` - IHost IHost `gorm:"-"` + IHost _IHost `gorm:"-"` +} + +type MeshModelHostsWithEntitySummary struct { + ID uuid.UUID `json:"id"` + Hostname string `json:"hostname"` + Port int `json:"port"` + Summary EntitySummary `json:"summary"` +} +type EntitySummary struct { + Models int64 `json:"models"` + Components int64 `json:"components"` + Relationships int64 `json:"relationships"` + Policies int64 `json:"policies"` +} +type MesheryHostSummaryDB struct { + HostID uuid.UUID `json:"-" gorm:"id"` + Hostname string `json:"-" gorm:"hostname"` + Port int `json:"-" gorm:"port"` + Models int64 `json:"-" gorm:"models"` + Components int64 `json:"-" gorm:"components"` + Relationships int64 `json:"-" gorm:"relationships"` + Policies int64 `json:"-" gorm:"policies"` +} + +type HostFilter struct { + Name string + Greedy bool //when set to true - instead of an exact match, name will be prefix matched + Trim bool //when set to true - the schema is not returned + DisplayName string + Version string + Sort string //asc or desc. Default behavior is asc + OrderOn string + Limit int //If 0 or unspecified then all records are returned and limit is not used + Offset int } func(h *Hostv1beta1) Create(db *database.Handler) (uuid.UUID, error) { @@ -31,7 +64,7 @@ func(h *Hostv1beta1) Create(db *database.Handler) (uuid.UUID, error) { return uuid.UUID{}, err } hID := uuid.NewSHA1(uuid.UUID{}, byt) - var host Host + var host Hostv1beta1 hostv1beta1CreationLock.Lock() defer hostv1beta1CreationLock.Unlock() err = db.First(&host, "id = ?", hID).Error // check if the host already exists @@ -56,9 +89,9 @@ func(h *Hostv1beta1) Create(db *database.Handler) (uuid.UUID, error) { func (h *Hostv1beta1) AfterFind(tx *gorm.DB) error { switch h.Hostname { case "artifacthub": - h.IHost = ArtifactHub{} + h.IHost = _ArtifactHub{} case "kubernetes": - h.IHost = Kubernetes{} + h.IHost = _Kubernetes{} default: // do nothing if the host is not pre-unknown. Currently adapters fall into this case. return nil } @@ -68,14 +101,14 @@ func (h *Hostv1beta1) AfterFind(tx *gorm.DB) error { // Each host from where meshmodels can be generated needs to implement this interface // HandleDependents, contains host specific logic for provisioning required CRDs/operators for corresponding components. type _IHost interface { - HandleDependents(comp v1alpha1.Component, kc *kubernetes.Client, isDeploy bool) (string, error) + HandleDependents(comp Component, kc *kubernetes.Client, isDeploy bool) (string, error) String() string } type _ArtifactHub struct{} -func (ah _ArtifactHub) HandleDependents(comp v1alpha1.Component, kc *kubernetes.Client, isDeploy bool) (summary string, err error) { - source_uri := comp.Annotations[fmt.Sprintf("%s.model.source_uri", v1alpha1.MesheryAnnotationPrefix)] +func (ah _ArtifactHub) HandleDependents(comp Component, kc *kubernetes.Client, isDeploy bool) (summary string, err error) { + source_uri := comp.Annotations[fmt.Sprintf("%s.model.source_uri", MesheryAnnotationPrefix)] act := kubernetes.UNINSTALL if isDeploy { act = kubernetes.INSTALL @@ -112,7 +145,7 @@ func (ah _ArtifactHub) String() string { type _Kubernetes struct{} -func (k _Kubernetes) HandleDependents(comp v1alpha1.Component, kc *kubernetes.Client, isDeploy bool) (summary string, err error) { +func (k _Kubernetes) HandleDependents(comp Component, kc *kubernetes.Client, isDeploy bool) (summary string, err error) { return summary, err } diff --git a/models/meshmodel/core/v1beta1/models.go b/models/meshmodel/core/v1beta1/models.go index 6e730a29..c1a8dc23 100644 --- a/models/meshmodel/core/v1beta1/models.go +++ b/models/meshmodel/core/v1beta1/models.go @@ -9,31 +9,13 @@ import ( "github.com/google/uuid" "github.com/layer5io/meshkit/database" "github.com/layer5io/meshkit/models/meshmodel/entity" - "github.com/layer5io/meshkit/models/meshmodel/registry" "github.com/layer5io/meshkit/utils" "gorm.io/gorm" ) var modelCreationLock sync.Mutex //Each component/relationship will perform a check and if the model already doesn't exist, it will create a model. This lock will make sure that there are no race conditions. -type ModelFilter struct { - Name string - Registrant string //name of the registrant for a given model - DisplayName string //If Name is already passed, avoid passing Display name unless greedy=true, else the filter will translate to an AND returning only the models where name and display name match exactly. Ignore, if this behavior is expected. - Greedy bool //when set to true - instead of an exact match, name will be prefix matched. Also an OR will be performed of name and display_name - Version string - Category string - OrderOn string - Sort string //asc or desc. Default behavior is asc - Limit int //If 0 or unspecified then all records are returned and limit is not used - Offset int - Annotations string //When this query parameter is "true", only models with the "isAnnotation" property set to true are returned. When this query parameter is "false", all models except those considered to be annotation models are returned. Any other value of the query parameter results in both annoations as well as non-annotation models being returned. - - // When these are set to true, we also retrieve components/relationships associated with the model. - Components bool - Relationships bool - Status string -} + type model struct { Version string `json:"version,omitempty" yaml:"version"` @@ -46,8 +28,8 @@ type Model struct { Name string `json:"name"` DisplayName string `json:"displayName" gorm:"modelDisplayName"` Description string `json:"description" gorm:"description"` - Status entity.EntityStatus `json:"status" gorm:"status"` - Registrant registry.Hostv1beta1 `json:"registrant" gorm:"registrant"` // to be Connection + Status entity.EntityStatus `json:"status" gorm:"status"` + Registrant Hostv1beta1 `json:"registrant" gorm:"registrant"` // to be Connection Category Category `json:"category"` SubCategory string `json:"subCategory" gorm:"subCategory"` Metadata map[string]interface{} `json:"metadata" yaml:"modelMetadata"` @@ -59,15 +41,15 @@ type Model struct { type ModelDB struct { ID uuid.UUID `json:"id"` VersionMeta - Name string `json:"modelName" gorm:"modelName"` - DisplayName string `json:"modelDisplayName" gorm:"modelDisplayName"` - Description string `json:"description" gorm:"description"` + Name string `json:"modelName" gorm:"modelName"` + DisplayName string `json:"modelDisplayName" gorm:"modelDisplayName"` + Description string `json:"description" gorm:"description"` Status entity.EntityStatus `json:"status" gorm:"status"` - RegistrantID uuid.UUID `json:"hostID" gorm:"hostID"` - CategoryID uuid.UUID `json:"-" gorm:"categoryID"` - SubCategory string `json:"subCategory" gorm:"subCategory"` - Metadata []byte `json:"modelMetadata" gorm:"modelMetadata"` - Model model `json:"model,omitempty" gorm:"model"` + RegistrantID uuid.UUID `json:"hostID" gorm:"hostID"` + CategoryID uuid.UUID `json:"-" gorm:"categoryID"` + SubCategory string `json:"subCategory" gorm:"subCategory"` + Metadata []byte `json:"modelMetadata" gorm:"modelMetadata"` + Model model `json:"model,omitempty" gorm:"model"` } func (m Model) Type() entity.EntityType { @@ -77,6 +59,10 @@ func (m Model) GetID() uuid.UUID { return m.ID } +func (m *Model) GetEntityDetail() string { + return fmt.Sprintf("type: %s, model: %s, definition version: %s, version: %s", m.Type(), m.Name, m.Version, m.Model.Version) +} + func (m *Model) Create(db *database.Handler) (uuid.UUID, error) { hostID, err := m.Registrant.Create(db) @@ -118,10 +104,10 @@ func (m *Model) Create(db *database.Handler) (uuid.UUID, error) { return model.ID, nil } -func (m *Model) UpdateStatus(db database.Handler, status entity.EntityStatus) error { +func (m *Model) UpdateStatus(db *database.Handler, status entity.EntityStatus) error { err := db.Model(&ModelDB{}).Where("id = ?", m.ID).Update("status", status).Error if err != nil { - return entity.ErrUpdateEntityStatus(err, string(m.Type()), status.String()) + return entity.ErrUpdateEntityStatus(err, string(m.Type()), status) } return nil } @@ -143,7 +129,7 @@ func (c *Model) GetModelDB() (cmd ModelDB) { } // is reg should be passed as param? -func (cmd *ModelDB) GetModel(cat Category, reg registry.Hostv1beta1) (c Model) { +func (cmd *ModelDB) GetModel(cat Category, reg Hostv1beta1) (c Model) { c.ID = cmd.ID c.VersionMeta = cmd.VersionMeta c.Name = cmd.Name diff --git a/models/meshmodel/core/v1beta1/policy.go b/models/meshmodel/core/v1beta1/policy.go index fd8416ce..0922e132 100644 --- a/models/meshmodel/core/v1beta1/policy.go +++ b/models/meshmodel/core/v1beta1/policy.go @@ -1 +1,93 @@ -package v1beta1 \ No newline at end of file +package v1beta1 + +import ( + "encoding/json" + "fmt" + "path/filepath" + "time" + + "github.com/google/uuid" + "github.com/layer5io/meshkit/database" + "github.com/layer5io/meshkit/utils" + + "github.com/layer5io/meshkit/models/meshmodel/entity" +) + +type PolicyDefinition struct { + ID uuid.UUID `json:"-"` + TypeMeta + Model Model `json:"model"` + SubType string `json:"subType" yaml:"subType"` + Expression map[string]interface{} `json:"expression" yaml:"expression"` + CreatedAt time.Time `json:"-"` + UpdatedAt time.Time `json:"-"` +} + +type PolicyDefinitionDB struct { + ID uuid.UUID `json:"-"` + ModelID uuid.UUID `json:"-" gorm:"modelID"` + TypeMeta + SubType string `json:"subType" yaml:"subType"` + Expression []byte `json:"expression" yaml:"expression"` + CreatedAt time.Time `json:"-"` + UpdatedAt time.Time `json:"-"` +} + +func (p PolicyDefinition) GetID() uuid.UUID { + return p.ID +} + +func (p PolicyDefinition) Type() entity.EntityType { + return entity.PolicyDefinition +} + +func (p *PolicyDefinition) GetEntityDetail() string { + return fmt.Sprintf("type: %s, definition version: %s, name: %s, model: %s, version: %s", p.Type(), p.Version, p.Kind, p.Model.Name, p.Model.Version) +} + +func (p *PolicyDefinition) Create(db *database.Handler) (uuid.UUID, error) { + p.ID = uuid.New() + mid, err := p.Model.Create(db) + if err != nil { + return uuid.UUID{}, err + } + pdb := p.GetPolicyDefinitionDB() + pdb.ModelID = mid + err = db.Create(&pdb).Error + if err != nil { + return uuid.UUID{}, err + } + return pdb.ID, nil +} + +func (m *PolicyDefinition) UpdateStatus(db *database.Handler, status entity.EntityStatus) error { + return nil +} + +func (p *PolicyDefinition) GetPolicyDefinitionDB() (pdb PolicyDefinitionDB) { + pdb.ID = p.ID + pdb.TypeMeta = p.TypeMeta + pdb.SubType = p.SubType + pdb.ModelID = p.Model.ID + pdb.Expression, _ = json.Marshal(p.Expression) + return +} + +func (pdb *PolicyDefinitionDB) GetPolicyDefinition(m Model) (p PolicyDefinition) { + p.ID = pdb.ID + p.TypeMeta = pdb.TypeMeta + p.Model = m + p.SubType = pdb.SubType + if p.Expression == nil { + p.Expression = make(map[string]interface{}) + } + _ = json.Unmarshal(pdb.Expression, &p.Expression) + + return +} + +func (p PolicyDefinition) WritePolicyDefinition(policyDirPath string) error { + policyPath := filepath.Join(policyDirPath, p.Kind+".json") + err := utils.WriteJSONToFile[PolicyDefinition](policyPath, p) + return err +} diff --git a/models/meshmodel/entity/error.go b/models/meshmodel/entity/error.go index f5f61fc6..1517fd3e 100644 --- a/models/meshmodel/entity/error.go +++ b/models/meshmodel/entity/error.go @@ -10,6 +10,6 @@ const ( ErrUpdateEntityStatusCode = "" ) -func ErrUpdateEntityStatus(err error, entity, status string) error { +func ErrUpdateEntityStatus(err error, entity string, status EntityStatus) error { return errors.New(ErrUpdateEntityStatusCode, errors.Alert, []string{fmt.Sprintf("unable to update %s to %s", entity, status)}, []string{err.Error()}, []string{}, []string{}) } diff --git a/models/meshmodel/entity/status.go b/models/meshmodel/entity/status.go index e12dd7a8..1e84ae50 100644 --- a/models/meshmodel/entity/status.go +++ b/models/meshmodel/entity/status.go @@ -2,28 +2,14 @@ package entity import "github.com/layer5io/meshkit/database" -type EntityStatus int +type EntityStatus string const ( - Ignored EntityStatus = iota - Enabled - Duplicate + Ignored EntityStatus = "ignored" + Enabled EntityStatus = "enabled" + Duplicate EntityStatus = "duplicate" ) -func(e EntityStatus) String() string { - switch e { - case Ignored: - return "ignored" - case Duplicate: - return "duplicate" - case Enabled: - fallthrough - default: - return "enabled" - } - -} - type Status interface { - UpdateStatus(db *database.Handler, status EntityStatus) error -} \ No newline at end of file + UpdateStatus(db *database.Handler, status EntityStatus) error +} diff --git a/models/meshmodel/entity/types.go b/models/meshmodel/entity/types.go index e79f22de..fe4a288f 100644 --- a/models/meshmodel/entity/types.go +++ b/models/meshmodel/entity/types.go @@ -17,13 +17,14 @@ const ( // Each entity will have it's own Filter implementation via which it exposes the nobs and dials to fetch entities type Filter interface { Create(map[string]interface{}) - Get(db *database.Handler, f Filter) (entities []Entity, count int64, unique int, err error) + Get(db *database.Handler) (entities []Entity, count int64, unique int, err error) } type Entity interface { // Entity is referred as any type of schema managed by the registry // ComponentDefinitions and PolicyDefinitions are examples of entities Type() EntityType + GetEntityDetail() string GetID() uuid.UUID Status Create(db *database.Handler) (entityID uuid.UUID, err error) diff --git a/models/meshmodel/registry/category_filter.go b/models/meshmodel/registry/category_filter.go new file mode 100644 index 00000000..e2a6cc5b --- /dev/null +++ b/models/meshmodel/registry/category_filter.go @@ -0,0 +1,68 @@ +package registry + +import ( + "github.com/layer5io/meshkit/database" + "github.com/layer5io/meshkit/models/meshmodel/core/v1beta1" + "gorm.io/gorm/clause" +) + +type CategoryFilter struct { + Name string + OrderOn string + Greedy bool + Sort string //asc or desc. Default behavior is asc + Limit int //If 0 or unspecified then all records are returned and limit is not used + Offset int +} + +// Create the filter from map[string]interface{} +func (cf *CategoryFilter) Create(m map[string]interface{}) { + if m == nil { + return + } + cf.Name = m["name"].(string) +} + +func (cf *CategoryFilter) Get(db *database.Handler) ([]v1beta1.Category, int64, int, error) { + var catdb []v1beta1.CategoryDB + var cat []v1beta1.Category + finder := db.Model(&catdb) + + // total count before pagination + var count int64 + + if cf.Name != "" { + if cf.Greedy { + finder = finder.Where("name LIKE ?", "%"+cf.Name+"%") + } else { + finder = finder.Where("name = ?", cf.Name) + } + } + if cf.OrderOn != "" { + if cf.Sort == "desc" { + finder = finder.Order(clause.OrderByColumn{Column: clause.Column{Name: cf.OrderOn}, Desc: true}) + } else { + finder = finder.Order(cf.OrderOn) + } + } + + finder.Count(&count) + + if cf.Limit != 0 { + finder = finder.Limit(cf.Limit) + } + if cf.Offset != 0 { + finder = finder.Offset(cf.Offset) + } + + if count == 0 { + finder.Count(&count) + } + + _ = finder.Find(&catdb).Error + for _, c := range catdb { + cat = append(cat, c.GetCategory(db)) + } + // duplicate category ? + return cat, count, int(count), nil +} diff --git a/models/meshmodel/registry/component_filter.go b/models/meshmodel/registry/component_filter.go new file mode 100644 index 00000000..19ecccd7 --- /dev/null +++ b/models/meshmodel/registry/component_filter.go @@ -0,0 +1,131 @@ +package registry + +import ( + "fmt" + + "github.com/layer5io/meshkit/database" + "github.com/layer5io/meshkit/models/meshmodel/core/v1beta1" + "github.com/layer5io/meshkit/models/meshmodel/entity" +) + +type ComponentFilter struct { + Name string + APIVersion string + Greedy bool //when set to true - instead of an exact match, name will be matched as a substring + Trim bool //when set to true - the schema is not returned + DisplayName string + ModelName string + CategoryName string + Version string + Sort string //asc or desc. Default behavior is asc + OrderOn string + Limit int //If 0 or unspecified then all records are returned and limit is not used + Offset int + Annotations string //When this query parameter is "true", only components with the "isAnnotation" property set to true are returned. When this query parameter is "false", all components except those considered to be annotation components are returned. Any other value of the query parameter results in both annotations as well as non-annotation models being returned. +} + +type componentDefinitionWithModel struct { + v1beta1.ComponentDefinitionDB + ModelDB v1beta1.ModelDB // acoount for overridn fields + CategoryDB v1beta1.CategoryDB + HostsDB v1beta1.Hostv1beta1 +} + +// Create the filter from map[string]interface{} +func (cf *ComponentFilter) Create(m map[string]interface{}) { + if m == nil { + return + } + cf.Name = m["name"].(string) +} + +func (componentFilter *ComponentFilter) Get(db *database.Handler) ([]entity.Entity, int64, int, error) { + // componentFilter, err := utils.Cast[ComponentFilter](f) + // if err != nil { + // return nil, 0, 0, err + // } + + countUniqueComponents := func(components []componentDefinitionWithModel) int { + set := make(map[string]struct{}) + for _, compWithModel := range components { + key := compWithModel.Component.Kind + "@" + compWithModel.Component.Version + "@" + compWithModel.ModelDB.Name + "@" + compWithModel.ModelDB.Version + if _, ok := set[key]; !ok { + set[key] = struct{}{} + } + } + return len(set) + } + var componentDefinitionsWithModel []componentDefinitionWithModel + finder := db.Model(&v1beta1.ComponentDefinitionDB{}). + Select("component_definition_dbs.*, model_dbs.*,category_dbs.*"). + Joins("JOIN model_dbs ON component_definition_dbs.model_id = model_dbs.id"). + Joins("JOIN category_dbs ON model_dbs.category_id = category_dbs.id"). + Joins("JOIN hosts ON models.registrant_id = hosts.id") + // + + if componentFilter.Greedy { + if componentFilter.Name != "" && componentFilter.DisplayName != "" { + finder = finder.Where("component_definition_dbs.component->>'kind' LIKE ? OR display_name LIKE ?", "%"+componentFilter.Name+"%", componentFilter.DisplayName+"%") + } else if componentFilter.Name != "" { + finder = finder.Where("component_definition_dbs.component->>'kind' LIKE ?", "%"+componentFilter.Name+"%") + } else if componentFilter.DisplayName != "" { + finder = finder.Where("component_definition_dbs.display_name LIKE ?", "%"+componentFilter.DisplayName+"%") + } + } else { + if componentFilter.Name != "" { + finder = finder.Where("component_definition_dbs.component->>'kind' = ?", componentFilter.Name) + } + if componentFilter.DisplayName != "" { + finder = finder.Where("component_definition_dbs.display_name = ?", componentFilter.DisplayName) + } + } + + if componentFilter.ModelName != "" && componentFilter.ModelName != "all" { + finder = finder.Where("model_dbs.name = ?", componentFilter.ModelName) + } + + if componentFilter.Annotations == "true" { + finder = finder.Where("component_definition_dbs.metadata->>'isAnnotation' = true") + } else if componentFilter.Annotations == "false" { + finder = finder.Where("component_definition_dbs.metadata->>'isAnnotation' = false") + } + + if componentFilter.APIVersion != "" { + finder = finder.Where("component_definition_dbs.component->>'version' = ?", componentFilter.APIVersion) + } + if componentFilter.CategoryName != "" { + finder = finder.Where("category_dbs.name = ?", componentFilter.CategoryName) + } + if componentFilter.Version != "" { + finder = finder.Where("model_dbs.version = ?", componentFilter.Version) + } + + var count int64 + finder.Count(&count) + finder = finder.Offset(componentFilter.Offset) + if componentFilter.Limit != 0 { + finder = finder.Limit(componentFilter.Limit) + } + err := finder. + Scan(&componentDefinitionsWithModel).Error + if err != nil { + fmt.Println(err.Error()) //for debugging + } + + defs := make([]entity.Entity, len(componentDefinitionsWithModel)) + // remove this when compoentdef and componetdefdb struct is consolidated. + for _, cm := range componentDefinitionsWithModel { + if componentFilter.Trim { + cm.Component.Schema = "" + } + // Ensure correct reg is passed, rn it is dummy for sake of testing. + // In the first query above where we do seelection i think there changes will be requrired, an when that two def and defDB structs are consolidated, using association and preload i think we can do. + reg := cm.HostsDB + cd := cm.ComponentDefinitionDB.GetComponentDefinition(cm.ModelDB.GetModel(cm.CategoryDB.GetCategory(db), reg)) + defs = append(defs, &cd) + } + + unique := countUniqueComponents(componentDefinitionsWithModel) + + return defs, count, unique, nil +} diff --git a/models/meshmodel/registry/error.go b/models/meshmodel/registry/error.go index e5ae90d2..5167c0c0 100644 --- a/models/meshmodel/registry/error.go +++ b/models/meshmodel/registry/error.go @@ -5,7 +5,8 @@ import ( ) var ( - ErrUnknownHostCode = "meshkit-11146" + ErrUnknownHostCode = "meshkit-11146" + ErrRegisterEntityCode = "" ) func ErrUnknownHost(err error) error { diff --git a/models/meshmodel/registry/host.go b/models/meshmodel/registry/host.go index cbc1350d..aee5a13a 100644 --- a/models/meshmodel/registry/host.go +++ b/models/meshmodel/registry/host.go @@ -1,121 +1,121 @@ package registry -import ( - "encoding/json" - "fmt" - "sync" - "time" - - "github.com/google/uuid" - "github.com/layer5io/meshkit/database" - "github.com/layer5io/meshkit/models/oam/core/v1alpha1" - "github.com/layer5io/meshkit/utils/kubernetes" - "gorm.io/gorm" -) - -var hostCreationLock sync.Mutex //Each entity will perform a check and if the host already doesn't exist, it will create a host. This lock will make sure that there are no race conditions. - -type Host struct { - ID uuid.UUID `json:"-"` - Hostname string - Port int - Metadata string - CreatedAt time.Time `json:"-"` - UpdatedAt time.Time `json:"-"` - IHost IHost `gorm:"-"` -} - -func createHost(db *database.Handler, h Host) (uuid.UUID, error) { - byt, err := json.Marshal(h) - if err != nil { - return uuid.UUID{}, err - } - hID := uuid.NewSHA1(uuid.UUID{}, byt) - var host Host - hostCreationLock.Lock() - defer hostCreationLock.Unlock() - err = db.First(&host, "id = ?", hID).Error // check if the host already exists - if err != nil && err != gorm.ErrRecordNotFound { - return uuid.UUID{}, err - } - - // if not exists then create a new host and return the id - if err == gorm.ErrRecordNotFound { - h.ID = hID - err = db.Create(&h).Error - if err != nil { - return uuid.UUID{}, err - } - return h.ID, nil - } - - // else return the id of the existing host - return host.ID, nil -} - -func (h *Host) AfterFind(tx *gorm.DB) error { - switch h.Hostname { - case "artifacthub": - h.IHost = ArtifactHub{} - case "kubernetes": - h.IHost = Kubernetes{} - default: // do nothing if the host is not pre-unknown. Currently adapters fall into this case. - return nil - } - return nil -} - -// Each host from where meshmodels can be generated needs to implement this interface -// HandleDependents, contains host specific logic for provisioning required CRDs/operators for corresponding components. -type IHost interface { - HandleDependents(comp v1alpha1.Component, kc *kubernetes.Client, isDeploy bool) (string, error) - String() string -} - -type ArtifactHub struct{} - -func (ah ArtifactHub) HandleDependents(comp v1alpha1.Component, kc *kubernetes.Client, isDeploy bool) (summary string, err error) { - source_uri := comp.Annotations[fmt.Sprintf("%s.model.source_uri", v1alpha1.MesheryAnnotationPrefix)] - act := kubernetes.UNINSTALL - if isDeploy { - act = kubernetes.INSTALL - } - - if source_uri != "" { - err = kc.ApplyHelmChart(kubernetes.ApplyHelmChartConfig{ - URL: source_uri, - Namespace: comp.Namespace, - CreateNamespace: true, - Action: act, - SkipUpgradeIfInstalled: true, - }) - if err != nil { - if !isDeploy { - summary = fmt.Sprintf("error undeploying dependent helm chart for %s, please proceed with manual uninstall or try again", comp.Name) - } else { - summary = fmt.Sprintf("error deploying dependent helm chart for %s, please procced with manual install or try again", comp.Name) - } - } else { - if !isDeploy { - summary = fmt.Sprintf("Undeployed dependent helm chart for %s", comp.Name) - } else { - summary = fmt.Sprintf("Deployed dependent helm chart for %s", comp.Name) - } - } - } - return summary, err -} - -func (ah ArtifactHub) String() string { - return "artifacthub" -} - -type Kubernetes struct{} - -func (k Kubernetes) HandleDependents(comp v1alpha1.Component, kc *kubernetes.Client, isDeploy bool) (summary string, err error) { - return summary, err -} - -func (k Kubernetes) String() string { - return "kubernetes" -} +// import ( +// "encoding/json" +// "fmt" +// "sync" +// "time" + +// "github.com/google/uuid" +// "github.com/layer5io/meshkit/database" +// "github.com/layer5io/meshkit/models/oam/core/v1alpha1" +// "github.com/layer5io/meshkit/utils/kubernetes" +// "gorm.io/gorm" +// ) + +// var hostCreationLock sync.Mutex //Each entity will perform a check and if the host already doesn't exist, it will create a host. This lock will make sure that there are no race conditions. + +// type Host struct { +// ID uuid.UUID `json:"-"` +// Hostname string +// Port int +// Metadata string +// CreatedAt time.Time `json:"-"` +// UpdatedAt time.Time `json:"-"` +// IHost IHost `gorm:"-"` +// } + +// func createHost(db *database.Handler, h Host) (uuid.UUID, error) { +// byt, err := json.Marshal(h) +// if err != nil { +// return uuid.UUID{}, err +// } +// hID := uuid.NewSHA1(uuid.UUID{}, byt) +// var host Host +// hostCreationLock.Lock() +// defer hostCreationLock.Unlock() +// err = db.First(&host, "id = ?", hID).Error // check if the host already exists +// if err != nil && err != gorm.ErrRecordNotFound { +// return uuid.UUID{}, err +// } + +// // if not exists then create a new host and return the id +// if err == gorm.ErrRecordNotFound { +// h.ID = hID +// err = db.Create(&h).Error +// if err != nil { +// return uuid.UUID{}, err +// } +// return h.ID, nil +// } + +// // else return the id of the existing host +// return host.ID, nil +// } + +// func (h *Host) AfterFind(tx *gorm.DB) error { +// switch h.Hostname { +// case "artifacthub": +// h.IHost = ArtifactHub{} +// case "kubernetes": +// h.IHost = Kubernetes{} +// default: // do nothing if the host is not pre-unknown. Currently adapters fall into this case. +// return nil +// } +// return nil +// } + +// // Each host from where meshmodels can be generated needs to implement this interface +// // HandleDependents, contains host specific logic for provisioning required CRDs/operators for corresponding components. +// type IHost interface { +// HandleDependents(comp v1alpha1.Component, kc *kubernetes.Client, isDeploy bool) (string, error) +// String() string +// } + +// type ArtifactHub struct{} + +// func (ah ArtifactHub) HandleDependents(comp v1alpha1.Component, kc *kubernetes.Client, isDeploy bool) (summary string, err error) { +// source_uri := comp.Annotations[fmt.Sprintf("%s.model.source_uri", v1alpha1.MesheryAnnotationPrefix)] +// act := kubernetes.UNINSTALL +// if isDeploy { +// act = kubernetes.INSTALL +// } + +// if source_uri != "" { +// err = kc.ApplyHelmChart(kubernetes.ApplyHelmChartConfig{ +// URL: source_uri, +// Namespace: comp.Namespace, +// CreateNamespace: true, +// Action: act, +// SkipUpgradeIfInstalled: true, +// }) +// if err != nil { +// if !isDeploy { +// summary = fmt.Sprintf("error undeploying dependent helm chart for %s, please proceed with manual uninstall or try again", comp.Name) +// } else { +// summary = fmt.Sprintf("error deploying dependent helm chart for %s, please procced with manual install or try again", comp.Name) +// } +// } else { +// if !isDeploy { +// summary = fmt.Sprintf("Undeployed dependent helm chart for %s", comp.Name) +// } else { +// summary = fmt.Sprintf("Deployed dependent helm chart for %s", comp.Name) +// } +// } +// } +// return summary, err +// } + +// func (ah ArtifactHub) String() string { +// return "artifacthub" +// } + +// type Kubernetes struct{} + +// func (k Kubernetes) HandleDependents(comp v1alpha1.Component, kc *kubernetes.Client, isDeploy bool) (summary string, err error) { +// return summary, err +// } + +// func (k Kubernetes) String() string { +// return "kubernetes" +// } diff --git a/models/meshmodel/registry/model_filter.go b/models/meshmodel/registry/model_filter.go new file mode 100644 index 00000000..28af4a1a --- /dev/null +++ b/models/meshmodel/registry/model_filter.go @@ -0,0 +1,171 @@ +package registry + +import ( + "fmt" + + "github.com/layer5io/meshkit/database" + "github.com/layer5io/meshkit/models/meshmodel/core/v1alpha2" + "github.com/layer5io/meshkit/models/meshmodel/core/v1beta1" + "github.com/layer5io/meshkit/models/meshmodel/entity" + "gorm.io/gorm/clause" +) + +type ModelFilter struct { + Name string + Registrant string //name of the registrant for a given model + DisplayName string //If Name is already passed, avoid passing Display name unless greedy=true, else the filter will translate to an AND returning only the models where name and display name match exactly. Ignore, if this behavior is expected. + Greedy bool //when set to true - instead of an exact match, name will be prefix matched. Also an OR will be performed of name and display_name + Version string + Category string + OrderOn string + Sort string //asc or desc. Default behavior is asc + Limit int //If 0 or unspecified then all records are returned and limit is not used + Offset int + Annotations string //When this query parameter is "true", only models with the "isAnnotation" property set to true are returned. When this query parameter is "false", all models except those considered to be annotation models are returned. Any other value of the query parameter results in both annoations as well as non-annotation models being returned. + + // When these are set to true, we also retrieve components/relationships associated with the model. + Components bool + Relationships bool + Status string +} + +// Create the filter from map[string]interface{} +func (mf *ModelFilter) Create(m map[string]interface{}) { + if m == nil { + return + } + mf.Name = m["name"].(string) +} + +func (mf *ModelFilter) Get(db *database.Handler) ([]entity.Entity, int64, int, error) { + // I have removed Regsitry struct here check if filter by Registrant works. + type modelWithCategories struct { + v1beta1.ModelDB + v1beta1.CategoryDB + v1beta1.Hostv1beta1 + } + + countUniqueModels := func(models []modelWithCategories) int { + set := make(map[string]struct{}) + for _, model := range models { + key := model.ModelDB.Name + "@" + model.ModelDB.Version + if _, ok := set[key]; !ok { + set[key] = struct{}{} + } + } + return len(set) + } + + var modelWithCategoriess []modelWithCategories + finder := db.Model(&v1beta1.ModelDB{}). + Select("model_dbs.*, category_dbs.*", "registries.*", "hosts.*"). + Joins("JOIN category_dbs ON model_dbs.category_id = category_dbs.id"). + Joins("JOIN registries ON registries.entity = model_dbs.id"). + Joins("JOIN hosts ON hosts.id = registries.registrant_id") + + // total count before pagination + var count int64 + + // include components and relationships in response body + var includeComponents, includeRelationships bool + + if mf.Greedy { + if mf.Name != "" && mf.DisplayName != "" { + finder = finder.Where("model_dbs.name LIKE ? OR model_dbs.display_name LIKE ?", "%"+mf.Name+"%", "%"+mf.DisplayName+"%") + } else if mf.Name != "" { + finder = finder.Where("model_dbs.name LIKE ?", "%"+mf.Name+"%") + } else if mf.DisplayName != "" { + finder = finder.Where("model_dbs.display_name LIKE ?", "%"+mf.DisplayName+"%") + } + } else { + if mf.Name != "" { + finder = finder.Where("model_dbs.name = ?", mf.Name) + } + if mf.DisplayName != "" { + finder = finder.Where("model_dbs.display_name = ?", mf.DisplayName) + } + } + if mf.Annotations == "true" { + finder = finder.Where("model_dbs.metadata->>'isAnnotation' = true") + } else if mf.Annotations == "false" { + finder = finder.Where("model_dbs.metadata->>'isAnnotation' = false") + } + if mf.Version != "" { + finder = finder.Where("model_dbs.version = ?", mf.Version) + } + if mf.Category != "" { + finder = finder.Where("category_dbs.name = ?", mf.Category) + } + if mf.Registrant != "" { + finder = finder.Where("hosts.hostname = ?", mf.Registrant) + } + if mf.Annotations == "true" { + finder = finder.Where("model_dbs.metadata->>'isAnnotation' = true") + } else if mf.Annotations == "false" { + finder = finder.Where("model_dbs.metadata->>'isAnnotation' = false") + } + if mf.OrderOn != "" { + if mf.Sort == "desc" { + finder = finder.Order(clause.OrderByColumn{Column: clause.Column{Name: mf.OrderOn}, Desc: true}) + } else { + finder = finder.Order(mf.OrderOn) + } + } else { + finder = finder.Order("display_name") + } + + finder.Count(&count) + + if mf.Limit != 0 { + finder = finder.Limit(mf.Limit) + } + if mf.Offset != 0 { + finder = finder.Offset(mf.Offset) + } + if mf.Status != "" { + finder = finder.Where("model_dbs.status = ?", mf.Status) + } + includeComponents = mf.Components + includeRelationships = mf.Relationships + + err := finder. + Scan(&modelWithCategoriess).Error + if err != nil { + fmt.Println(modelWithCategoriess) + fmt.Println(err.Error()) //for debugging + } + + defs := []entity.Entity{} + + for _, modelDB := range modelWithCategoriess { + reg := v1beta1.Hostv1beta1{} + model := modelDB.ModelDB.GetModel(modelDB.CategoryDB.GetCategory(db), reg) + // host := rm.GetRegistrant(model) + // model.HostID = host.ID + // model.HostName = host.Hostname + // model.DisplayHostName = host.Hostname + + if includeComponents { + var components []v1beta1.ComponentDefinitionDB + finder := db.Model(&v1beta1.ComponentDefinitionDB{}). + Select("component_definition_dbs.id, component_definition_dbs.kind,component_definition_dbs.display_name, component_definition_dbs.api_version, component_definition_dbs.metadata"). + Where("component_definition_dbs.model_id = ?", model.ID) + if err := finder.Scan(&components).Error; err != nil { + fmt.Println(err) + } + // model.Components = components + } + if includeRelationships { + var relationships []v1alpha2.RelationshipDefinitionDB + finder := db.Model(&v1alpha2.RelationshipDefinitionDB{}). + Select("relationship_definition_dbs.*"). + Where("relationship_definition_dbs.model_id = ?", model.ID) + if err := finder.Scan(&relationships).Error; err != nil { + fmt.Println(err) + } + // model.Relationships = relationships + } + defs = append(defs, &model) + } + return defs, count, countUniqueModels(modelWithCategoriess), nil +} diff --git a/models/meshmodel/registry/policy_filter.go b/models/meshmodel/registry/policy_filter.go new file mode 100644 index 00000000..12edbd76 --- /dev/null +++ b/models/meshmodel/registry/policy_filter.go @@ -0,0 +1,59 @@ +package registry + +import ( + "github.com/layer5io/meshkit/database" + "github.com/layer5io/meshkit/models/meshmodel/core/v1beta1" + "github.com/layer5io/meshkit/models/meshmodel/entity" +) + +type PolicyFilter struct { + Kind string + Greedy bool + SubType string + ModelName string + OrderOn string + Sort string + Limit int + Offset int +} + +func (pf *PolicyFilter) Create(m map[string]interface{}) { + if m == nil { + return + } +} + +type policyDefinitionWithModel struct { + v1beta1.PolicyDefinitionDB + Model v1beta1.Model +} + +func (pf *PolicyFilter) Get(db *database.Handler) ([]entity.Entity, int64, int, error) { + pl := []entity.Entity{} + var componentDefinitionsWithModel []policyDefinitionWithModel + finder := db.Model(&v1beta1.PolicyDefinitionDB{}). + Select("policy_definition_dbs.*, models_dbs.*"). + Joins("JOIN model_dbs ON model_dbs.id = policy_definition_dbs.model_id") + if pf.Kind != "" { + finder = finder.Where("policy_definition_dbs.kind = ?", pf.Kind) + } + if pf.SubType != "" { + finder = finder.Where("policy_definition_dbs.sub_type = ?", pf.SubType) + } + if pf.ModelName != "" { + finder = finder.Where("model_dbs.name = ?", pf.ModelName) + } + + var count int64 + finder.Count(&count) + + err := finder.Scan(&componentDefinitionsWithModel).Error + if err != nil { + return pl, 0, 0, err + } + for _, cm := range componentDefinitionsWithModel { + policyDef := cm.PolicyDefinitionDB.GetPolicyDefinition(cm.Model) + pl = append(pl, &policyDef) + } + return pl, count, int(count), nil +} diff --git a/models/meshmodel/registry/registry.go b/models/meshmodel/registry/registry.go index ebdf49c6..22528289 100644 --- a/models/meshmodel/registry/registry.go +++ b/models/meshmodel/registry/registry.go @@ -8,8 +8,9 @@ import ( "github.com/google/uuid" "github.com/layer5io/meshkit/database" - types "github.com/layer5io/meshkit/models/meshmodel/entity" - "github.com/layer5io/meshkit/models/meshmodel/core/v1alpha1" + "github.com/layer5io/meshkit/models/meshmodel/core/v1alpha2" + "github.com/layer5io/meshkit/models/meshmodel/core/v1beta1" + "github.com/layer5io/meshkit/models/meshmodel/entity" "golang.org/x/text/cases" "golang.org/x/text/language" "gorm.io/gorm" @@ -24,26 +25,19 @@ import ( // 2. Entity type // 3. Entity type MeshModelRegistrantData struct { - Host Host `json:"host"` - EntityType types.EntityType `json:"entityType"` - Entity []byte `json:"entity"` //This will be type converted to appropriate entity on server based on passed entity type + Host v1beta1.Hostv1beta1 `json:"host"` + EntityType entity.EntityType `json:"entityType"` + Entity []byte `json:"entity"` //This will be type converted to appropriate entity on server based on passed entity type } type Registry struct { ID uuid.UUID RegistrantID uuid.UUID Entity uuid.UUID - Type types.EntityType + Type entity.EntityType CreatedAt time.Time UpdatedAt time.Time } -// Entity is referred as any type of schema managed by the registry -// ComponentDefinitions and PolicyDefinitions are examples of entities -type Entity interface { - Type() types.EntityType - GetID() uuid.UUID -} - // RegistryManager instance will expose methods for registry operations & sits between the database level operations and user facing API handlers. type RegistryManager struct { db *database.Handler //This database handler will be used to perform queries inside the database @@ -54,7 +48,7 @@ func registerModel(db *database.Handler, regID, modelID uuid.UUID) error { entity := Registry{ RegistrantID: regID, Entity: modelID, - Type: types.Model, + Type: entity.Model, } byt, err := json.Marshal(entity) @@ -91,12 +85,12 @@ func NewRegistryManager(db *database.Handler) (*RegistryManager, error) { } err := rm.db.AutoMigrate( &Registry{}, - &Host{}, - &v1alpha1.ComponentDefinitionDB{}, - &v1alpha1.RelationshipDefinitionDB{}, - &v1alpha1.PolicyDefinitionDB{}, - &v1alpha1.ModelDB{}, - &v1alpha1.CategoryDB{}, + &v1beta1.Hostv1beta1{}, + &v1beta1.ComponentDefinitionDB{}, + &v1alpha2.RelationshipDefinitionDB{}, + &v1beta1.PolicyDefinitionDB{}, + &v1beta1.ModelDB{}, + &v1beta1.CategoryDB{}, ) if err != nil { return nil, err @@ -106,114 +100,51 @@ func NewRegistryManager(db *database.Handler) (*RegistryManager, error) { func (rm *RegistryManager) Cleanup() { _ = rm.db.Migrator().DropTable( &Registry{}, - &Host{}, - &v1alpha1.ComponentDefinitionDB{}, - &v1alpha1.ModelDB{}, - &v1alpha1.CategoryDB{}, - &v1alpha1.RelationshipDefinitionDB{}, + &v1beta1.Hostv1beta1{}, + &v1beta1.ComponentDefinitionDB{}, + &v1beta1.ModelDB{}, + &v1beta1.CategoryDB{}, + &v1alpha2.RelationshipDefinitionDB{}, ) } -func (rm *RegistryManager) RegisterEntity(h Host, en Entity) error { - switch entity := en.(type) { - case v1alpha1.ComponentDefinition: - isAnnotation, _ := entity.Metadata["isAnnotation"].(bool) - if entity.Schema == "" && !isAnnotation { //For components which an empty schema and is not an annotation, exit quietly - return nil - } - - registrantID, err := createHost(rm.db, h) - if err != nil { - return err - } - - componentID, modelID, err := v1alpha1.CreateComponent(rm.db, entity) - if err != nil { - return err - } - - err = registerModel(rm.db, registrantID, modelID) - if err != nil { - return err - } - - entry := Registry{ - ID: uuid.New(), - RegistrantID: registrantID, - Entity: componentID, - Type: en.Type(), - CreatedAt: time.Now(), - UpdatedAt: time.Now(), - } - return rm.db.Create(&entry).Error - case v1alpha1.RelationshipDefinition: - - registrantID, err := createHost(rm.db, h) - if err != nil { - return err - } - - relationshipID, modelID, err := v1alpha1.CreateRelationship(rm.db, entity) - if err != nil { - return err - } - - err = registerModel(rm.db, registrantID, modelID) - if err != nil { - return err - } - - entry := Registry{ - ID: uuid.New(), - RegistrantID: registrantID, - Entity: relationshipID, - Type: en.Type(), - CreatedAt: time.Now(), - UpdatedAt: time.Now(), - } - return rm.db.Create(&entry).Error - //Add logic for Policies and other entities below - case v1alpha1.PolicyDefinition: - registrantID, err := createHost(rm.db, h) - if err != nil { - return err - } - - policyID, modelID, err := v1alpha1.CreatePolicy(rm.db, entity) - if err != nil { - return err - } - - err = registerModel(rm.db, registrantID, modelID) - if err != nil { - return err - } +func (rm *RegistryManager) RegisterEntity(h v1beta1.Hostv1beta1, en entity.Entity) error { + registrantID, err := h.Create(rm.db) + if err != nil { + return err + } - entry := Registry{ - ID: uuid.New(), - RegistrantID: registrantID, - Entity: policyID, - Type: en.Type(), - CreatedAt: time.Now(), - UpdatedAt: time.Now(), - } - return rm.db.Create(&entry).Error + entityID, err := en.Create(rm.db) + if err != nil { + return err + } - default: - return nil + entry := Registry{ + ID: uuid.New(), + RegistrantID: registrantID, + Entity: entityID, + Type: en.Type(), + CreatedAt: time.Now(), + UpdatedAt: time.Now(), } + err = rm.db.Create(&entry).Error + if err != nil { + return err + } + return nil } // UpdateEntityIgnoreStatus updates the ignore status of an entity based on the provided parameters. // By default during models generation ignore is set to false -func (rm *RegistryManager) UpdateEntityStatus(ID string, status string, entity string) error { +func (rm *RegistryManager) UpdateEntityStatus(ID string, status string, entityType string) error { // Convert string UUID to google UUID entityID, err := uuid.Parse(ID) if err != nil { return err } - switch entity { + switch entityType { case "models": - err := v1alpha1.UpdateModelsStatus(rm.db, entityID, status) + model := v1beta1.Model{ID: entityID} + model.UpdateStatus(rm.db, entity.EntityStatus(status)) if err != nil { return err } @@ -223,8 +154,8 @@ func (rm *RegistryManager) UpdateEntityStatus(ID string, status string, entity s } } -func (rm *RegistryManager) GetRegistrants(f *v1alpha1.HostFilter) ([]v1alpha1.MeshModelHostsWithEntitySummary, int64, error) { - var result []v1alpha1.MesheryHostSummaryDB +func (rm *RegistryManager) GetRegistrants(f *v1beta1.HostFilter) ([]v1beta1.MeshModelHostsWithEntitySummary, int64, error) { + var result []v1beta1.MesheryHostSummaryDB var totalcount int64 db := rm.db @@ -263,14 +194,14 @@ func (rm *RegistryManager) GetRegistrants(f *v1alpha1.HostFilter) ([]v1alpha1.Me return nil, 0, err } - var response []v1alpha1.MeshModelHostsWithEntitySummary + var response []v1beta1.MeshModelHostsWithEntitySummary for _, r := range result { - res := v1alpha1.MeshModelHostsWithEntitySummary{ + res := v1beta1.MeshModelHostsWithEntitySummary{ ID: r.HostID, Hostname: HostnameToPascalCase(r.Hostname), Port: r.Port, - Summary: v1alpha1.EntitySummary{ + Summary: v1beta1.EntitySummary{ Models: r.Models, Components: r.Components, Relationships: r.Relationships, @@ -282,40 +213,43 @@ func (rm *RegistryManager) GetRegistrants(f *v1alpha1.HostFilter) ([]v1alpha1.Me return response, totalcount, nil } -func (rm *RegistryManager) GetEntities(f types.Filter) ([]Entity, *int64, *int) { - switch filter := f.(type) { - case *v1alpha1.ComponentFilter: - en := make([]Entity, 0) - comps, count, unique := v1alpha1.GetMeshModelComponents(rm.db, *filter) - for _, comp := range comps { - en = append(en, comp) - } - return en, &count, &unique - case *v1alpha1.RelationshipFilter: - en := make([]Entity, 0) - relationships, count := v1alpha1.GetMeshModelRelationship(rm.db, *filter) - for _, rel := range relationships { - en = append(en, rel) - } - return en, &count, nil - case *v1alpha1.PolicyFilter: - en := make([]Entity, 0) - policies := v1alpha1.GetMeshModelPolicy(rm.db, *filter) - for _, pol := range policies { - en = append(en, pol) - } - return en, nil, nil - default: - return nil, nil, nil - } +func (rm *RegistryManager) GetEntities(f entity.Filter) ([]entity.Entity, int64, int, error) { + return f.Get(rm.db) + + // switch filter := f.(type) { + // case *v1beta1.ComponentFilter: + // en := make([]Entity, 0) + // comps, count, unique := v1beta1.GetMeshModelComponents(rm.db, *filter) + // for _, comp := range comps { + // en = append(en, comp) + // } + // return en, &count, &unique + // case *v1beta1.RelationshipFilter: + // en := make([]Entity, 0) + // relationships, count := v1beta1.GetMeshModelRelationship(rm.db, *filter) + // for _, rel := range relationships { + // en = append(en, rel) + // } + // return en, &count, nil + // case *v1beta1.PolicyFilter: + // en := make([]Entity, 0) + // policies := v1beta1.GetMeshModelPolicy(rm.db, *filter) + // for _, pol := range policies { + // en = append(en, pol) + // } + // return en, nil, nil + // default: + // return nil, nil, nil + // } } -func (rm *RegistryManager) GetModels(db *database.Handler, f types.Filter) ([]v1alpha1.Model, int64, int) { - var m []v1alpha1.Model + +func (rm *RegistryManager) GetModels(db *database.Handler, f entity.Filter) ([]v1beta1.Model, int64, int) { + var m []v1beta1.Model type modelWithCategories struct { - v1alpha1.ModelDB - v1alpha1.CategoryDB + v1beta1.ModelDB + v1beta1.CategoryDB Registry - Host + v1beta1.Hostv1beta1 } countUniqueModels := func(models []modelWithCategories) int { @@ -330,7 +264,7 @@ func (rm *RegistryManager) GetModels(db *database.Handler, f types.Filter) ([]v1 } var modelWithCategoriess []modelWithCategories - finder := db.Model(&v1alpha1.ModelDB{}). + finder := db.Model(&v1beta1.ModelDB{}). Select("model_dbs.*, category_dbs.*", "registries.*", "hosts.*"). Joins("JOIN category_dbs ON model_dbs.category_id = category_dbs.id"). Joins("JOIN registries ON registries.entity = model_dbs.id"). @@ -342,7 +276,7 @@ func (rm *RegistryManager) GetModels(db *database.Handler, f types.Filter) ([]v1 // include components and relationships in response body var includeComponents, includeRelationships bool - if mf, ok := f.(*v1alpha1.ModelFilter); ok { + if mf, ok := f.(*ModelFilter); ok { if mf.Greedy { if mf.Name != "" && mf.DisplayName != "" { finder = finder.Where("model_dbs.name LIKE ? OR model_dbs.display_name LIKE ?", "%"+mf.Name+"%", "%"+mf.DisplayName+"%") @@ -410,87 +344,48 @@ func (rm *RegistryManager) GetModels(db *database.Handler, f types.Filter) ([]v1 } for _, modelDB := range modelWithCategoriess { - model := modelDB.ModelDB.GetModel(modelDB.GetCategory(db)) - host := rm.GetRegistrant(model) - model.HostID = host.ID - model.HostName = host.Hostname - model.DisplayHostName = host.Hostname + reg := v1beta1.Hostv1beta1{} + model := modelDB.ModelDB.GetModel(modelDB.CategoryDB.GetCategory(db), reg) + // host := rm.GetRegistrant(model) + // model.HostID = host.ID + // model.HostName = host.Hostname + // model.DisplayHostName = host.Hostname if includeComponents { - var components []v1alpha1.ComponentDefinitionDB - finder := db.Model(&v1alpha1.ComponentDefinitionDB{}). + var components []v1beta1.ComponentDefinitionDB + finder := db.Model(&v1beta1.ComponentDefinitionDB{}). Select("component_definition_dbs.id, component_definition_dbs.kind,component_definition_dbs.display_name, component_definition_dbs.api_version, component_definition_dbs.metadata"). Where("component_definition_dbs.model_id = ?", model.ID) if err := finder.Scan(&components).Error; err != nil { fmt.Println(err) } - model.Components = components + // model.Components = components } if includeRelationships { - var relationships []v1alpha1.RelationshipDefinitionDB - finder := db.Model(&v1alpha1.RelationshipDefinitionDB{}). + var relationships []v1alpha2.RelationshipDefinitionDB + finder := db.Model(&v1alpha2.RelationshipDefinitionDB{}). Select("relationship_definition_dbs.*"). Where("relationship_definition_dbs.model_id = ?", model.ID) if err := finder.Scan(&relationships).Error; err != nil { fmt.Println(err) } - model.Relationships = relationships + // model.Relationships = relationships } m = append(m, model) } return m, count, countUniqueModels(modelWithCategoriess) } -func (rm *RegistryManager) GetCategories(db *database.Handler, f types.Filter) ([]v1alpha1.Category, int64) { - var catdb []v1alpha1.CategoryDB - var cat []v1alpha1.Category - finder := rm.db.Model(&catdb) - - // total count before pagination - var count int64 - - if mf, ok := f.(*v1alpha1.CategoryFilter); ok { - if mf.Name != "" { - if mf.Greedy { - finder = finder.Where("name LIKE ?", "%"+mf.Name+"%") - } else { - finder = finder.Where("name = ?", mf.Name) - } - } - if mf.OrderOn != "" { - if mf.Sort == "desc" { - finder = finder.Order(clause.OrderByColumn{Column: clause.Column{Name: mf.OrderOn}, Desc: true}) - } else { - finder = finder.Order(mf.OrderOn) - } - } - - finder.Count(&count) - - if mf.Limit != 0 { - finder = finder.Limit(mf.Limit) - } - if mf.Offset != 0 { - finder = finder.Offset(mf.Offset) - } - } - - if count == 0 { - finder.Count(&count) - } - - _ = finder.Find(&catdb).Error - for _, c := range catdb { - cat = append(cat, c.GetCategory(db)) - } +func (rm *RegistryManager) GetCategories(db *database.Handler, f entity.Filter) ([]entity.Entity, int64) { + cat, count, _, _ := f.Get(rm.db) return cat, count } -func (rm *RegistryManager) GetRegistrant(e Entity) Host { +func (rm *RegistryManager) GetRegistrant(e entity.Entity) v1beta1.Hostv1beta1 { eID := e.GetID() var reg Registry _ = rm.db.Where("entity = ?", eID).Find(®).Error - var h Host + var h v1beta1.Hostv1beta1 _ = rm.db.Where("id = ?", reg.RegistrantID).Find(&h).Error return h } diff --git a/models/meshmodel/registry/relationship_filter.go b/models/meshmodel/registry/relationship_filter.go new file mode 100644 index 00000000..f352b2e1 --- /dev/null +++ b/models/meshmodel/registry/relationship_filter.go @@ -0,0 +1,108 @@ +package registry + +import ( + "fmt" + + "github.com/layer5io/meshkit/database" + "github.com/layer5io/meshkit/models/meshmodel/core/v1alpha2" + "github.com/layer5io/meshkit/models/meshmodel/core/v1beta1" + "github.com/layer5io/meshkit/models/meshmodel/entity" + "gorm.io/gorm/clause" +) + +type relationshipDefinitionWithModel struct { + v1alpha2.RelationshipDefinitionDB + ModelDB v1beta1.ModelDB + // acoount for overridn fields + // v1beta1.ModelDB.Version `json:"modelVersion"` + CategoryDB v1beta1.CategoryDB +} + +// For now, only filtering by Kind and SubType are allowed. +// In the future, we will add support to query using `selectors` (using CUE) +// TODO: Add support for Model +type RelationshipFilter struct { + Kind string + Greedy bool //when set to true - instead of an exact match, kind will be prefix matched + SubType string + RelationshipType string + Version string + ModelName string + OrderOn string + Sort string //asc or desc. Default behavior is asc + Limit int //If 0 or unspecified then all records are returned and limit is not used + Offset int +} + +// Create the filter from map[string]interface{} +func (rf *RelationshipFilter) Create(m map[string]interface{}) { + if m == nil { + return + } + rf.Kind = m["kind"].(string) +} +func (relationshipFilter *RelationshipFilter) Get(db *database.Handler) ([]entity.Entity, int64, int, error) { + // relationshipFilter, err := utils.Cast[RelationshipFilter](f) + // if err != nil { + // return nil, 0, 0, err + // } + + var relationshipDefinitionsWithModel []relationshipDefinitionWithModel + finder := db.Model(&v1alpha2.RelationshipDefinitionDB{}). + Select("relationship_definition_dbs.*, model_dbs.*"). + Joins("JOIN model_dbs ON relationship_definition_dbs.model_id = model_dbs.id"). // + Joins("JOIN category_dbs ON model_dbs.category_id = category_dbs.id") // + if relationshipFilter.Kind != "" { + if relationshipFilter.Greedy { + finder = finder.Where("relationship_definition_dbs.kind LIKE ?", "%"+relationshipFilter.Kind+"%") + } else { + finder = finder.Where("relationship_definition_dbs.kind = ?", relationshipFilter.Kind) + } + } + + if relationshipFilter.RelationshipType != "" { + finder = finder.Where("relationship_definition_dbs.type = ?", relationshipFilter.RelationshipType) + } + + if relationshipFilter.SubType != "" { + finder = finder.Where("relationship_definition_dbs.sub_type = ?", relationshipFilter.SubType) + } + if relationshipFilter.ModelName != "" { + finder = finder.Where("model_dbs.name = ?", relationshipFilter.ModelName) + } + if relationshipFilter.Version != "" { + finder = finder.Where("model_dbs.version = ?", relationshipFilter.Version) + } + if relationshipFilter.OrderOn != "" { + if relationshipFilter.Sort == "desc" { + finder = finder.Order(clause.OrderByColumn{Column: clause.Column{Name: relationshipFilter.OrderOn}, Desc: true}) + } else { + finder = finder.Order(relationshipFilter.OrderOn) + } + } + + var count int64 + finder.Count(&count) + + finder = finder.Offset(relationshipFilter.Offset) + if relationshipFilter.Limit != 0 { + finder = finder.Limit(relationshipFilter.Limit) + } + err := finder. + Scan(&relationshipDefinitionsWithModel).Error + if err != nil { + fmt.Println(err.Error()) //for debugging + } + defs := make([]entity.Entity, len(relationshipDefinitionsWithModel)) + + // remove this when reldef and reldefdb struct is consolidated. + for _, cm := range relationshipDefinitionsWithModel { + // Ensure correct reg is passed, rn it is dummy for sake of testing. + // In the first query above where we do seelection i think there changes will be requrired, an when that two def and defDB structs are consolidated, using association and preload i think we can do. + reg := v1beta1.Hostv1beta1{} + rd := cm.RelationshipDefinitionDB.GetRelationshipDefinition(cm.ModelDB.GetModel(cm.CategoryDB.GetCategory(db), reg)) + defs = append(defs, &rd) + } + // Should have count unique relationships (by model version, model name, and relationship's kind, type, subtype, version) + return defs, count, int(count), nil +} diff --git a/models/oam/core/v1alpha1/application_component.go b/models/oam/core/v1alpha1/application_component.go index 505e9d37..a983f9c7 100644 --- a/models/oam/core/v1alpha1/application_component.go +++ b/models/oam/core/v1alpha1/application_component.go @@ -1,67 +1,67 @@ package v1alpha1 -import ( - "fmt" - "strings" +// import ( +// "fmt" +// "strings" - "github.com/layer5io/meshkit/models/meshmodel/core/v1alpha1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" -) +// "github.com/layer5io/meshkit/models/meshmodel/core/v1alpha1" +// metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +// ) -// Component is the structure for the core OAM Application Component -type Component struct { - metav1.TypeMeta `json:",inline"` - metav1.ObjectMeta `json:"metadata,omitempty"` +// // Component is the structure for the core OAM Application Component +// type Component struct { +// metav1.TypeMeta `json:",inline"` +// metav1.ObjectMeta `json:"metadata,omitempty"` - Spec ComponentSpec `json:"spec,omitempty"` -} +// Spec ComponentSpec `json:"spec,omitempty"` +// } -// ComponentSpec is the structure for the core OAM Application Component Spec -type ComponentSpec struct { - Type string `json:"type"` - Version string `json:"version"` - APIVersion string `json:"apiVersion"` - Model string `json:"model"` - Settings map[string]interface{} `json:"settings"` - Parameters []ComponentParameter `json:"parameters"` -} +// // ComponentSpec is the structure for the core OAM Application Component Spec +// type ComponentSpec struct { +// Type string `json:"type"` +// Version string `json:"version"` +// APIVersion string `json:"apiVersion"` +// Model string `json:"model"` +// Settings map[string]interface{} `json:"settings"` +// Parameters []ComponentParameter `json:"parameters"` +// } -// ComponentParameter is the structure for the core OAM Application Component -// Paramater -type ComponentParameter struct { - Name string `json:"name"` - FieldPaths []string `json:"fieldPaths"` - Required *bool `json:"required,omitempty"` - Description *string `json:"description,omitempty"` -} +// // ComponentParameter is the structure for the core OAM Application Component +// // Paramater +// type ComponentParameter struct { +// Name string `json:"name"` +// FieldPaths []string `json:"fieldPaths"` +// Required *bool `json:"required,omitempty"` +// Description *string `json:"description,omitempty"` +// } -const MesheryAnnotationPrefix = "design.meshmodel.io" +// const MesheryAnnotationPrefix = "design.meshmodel.io" -func GetAPIVersionFromComponent(comp Component) string { - return comp.Annotations[MesheryAnnotationPrefix+".k8s.APIVersion"] -} +// func GetAPIVersionFromComponent(comp Component) string { +// return comp.Annotations[MesheryAnnotationPrefix+".k8s.APIVersion"] +// } -func GetKindFromComponent(comp Component) string { - kind := strings.TrimPrefix(comp.Annotations[MesheryAnnotationPrefix+".k8s.Kind"], "/") - return kind -} +// func GetKindFromComponent(comp Component) string { +// kind := strings.TrimPrefix(comp.Annotations[MesheryAnnotationPrefix+".k8s.Kind"], "/") +// return kind +// } -func GetAnnotationsForWorkload(w v1alpha1.ComponentDefinition) map[string]string { - res := map[string]string{} +// func GetAnnotationsForWorkload(w v1alpha1.ComponentDefinition) map[string]string { +// res := map[string]string{} - for key, val := range w.Metadata { - if v, ok := val.(string); ok { - res[strings.ReplaceAll(fmt.Sprintf("%s.%s", MesheryAnnotationPrefix, key), " ", "")] = v - } - } - sourceURI, ok := w.Model.Metadata["source_uri"].(string) - if ok { - res[fmt.Sprintf("%s.model.source_uri", MesheryAnnotationPrefix)] = sourceURI - } - res[fmt.Sprintf("%s.model.name", MesheryAnnotationPrefix)] = w.Model.Name - res[fmt.Sprintf("%s.k8s.APIVersion", MesheryAnnotationPrefix)] = w.APIVersion - res[fmt.Sprintf("%s.k8s.Kind", MesheryAnnotationPrefix)] = w.Kind - res[fmt.Sprintf("%s.model.version", MesheryAnnotationPrefix)] = w.Model.Version - res[fmt.Sprintf("%s.model.category", MesheryAnnotationPrefix)] = w.Model.Category.Name - return res -} +// for key, val := range w.Metadata { +// if v, ok := val.(string); ok { +// res[strings.ReplaceAll(fmt.Sprintf("%s.%s", MesheryAnnotationPrefix, key), " ", "")] = v +// } +// } +// sourceURI, ok := w.Model.Metadata["source_uri"].(string) +// if ok { +// res[fmt.Sprintf("%s.model.source_uri", MesheryAnnotationPrefix)] = sourceURI +// } +// res[fmt.Sprintf("%s.model.name", MesheryAnnotationPrefix)] = w.Model.Name +// res[fmt.Sprintf("%s.k8s.APIVersion", MesheryAnnotationPrefix)] = w.APIVersion +// res[fmt.Sprintf("%s.k8s.Kind", MesheryAnnotationPrefix)] = w.Kind +// res[fmt.Sprintf("%s.model.version", MesheryAnnotationPrefix)] = w.Model.Version +// res[fmt.Sprintf("%s.model.category", MesheryAnnotationPrefix)] = w.Model.Category.Name +// return res +// } diff --git a/utils/component/generator.go b/utils/component/generator.go index 1a14dbd3..ccf1df32 100644 --- a/utils/component/generator.go +++ b/utils/component/generator.go @@ -6,7 +6,7 @@ import ( "fmt" "cuelang.org/go/cue" - "github.com/layer5io/meshkit/models/meshmodel/core/v1alpha1" + "github.com/layer5io/meshkit/models/meshmodel/core/v1beta1" "github.com/layer5io/meshkit/utils" "github.com/layer5io/meshkit/utils/manifests" ) @@ -46,8 +46,8 @@ var DefaultPathConfig2 = CuePathConfig{ var Configs = []CuePathConfig{DefaultPathConfig, DefaultPathConfig2} -func Generate(crd string) (v1alpha1.ComponentDefinition, error) { - component := v1alpha1.ComponentDefinition{} +func Generate(crd string) (v1beta1.ComponentDefinition, error) { + component := v1beta1.ComponentDefinition{} component.Metadata = make(map[string]interface{}) crdCue, err := utils.YamlToCue(crd) if err != nil { @@ -60,7 +60,7 @@ func Generate(crd string) (v1alpha1.ComponentDefinition, error) { break } } - component.Schema = schema + component.Component.Schema = schema name, err := extractCueValueFromPath(crdCue, DefaultPathConfig.NamePath) if err != nil { return component, err @@ -80,14 +80,14 @@ func Generate(crd string) (v1alpha1.ComponentDefinition, error) { } else if scope == "Namespaced" { component.Metadata["isNamespaced"] = true } - component.Kind = name + component.Component.Kind = name if group != "" { - component.APIVersion = fmt.Sprintf("%s/%s", group, version) + component.Component.Version = fmt.Sprintf("%s/%s", group, version) } else { - component.APIVersion = version + component.Component.Version = version } - component.Format = v1alpha1.JSON + component.Format = v1beta1.JSON component.DisplayName = manifests.FormatToReadableString(name) return component, nil } diff --git a/utils/component/generator_test.go b/utils/component/generator_test.go index a8de77a9..ed26c8d6 100644 --- a/utils/component/generator_test.go +++ b/utils/component/generator_test.go @@ -3,7 +3,7 @@ package component import ( "testing" - "github.com/layer5io/meshkit/models/meshmodel/core/v1alpha1" + "github.com/layer5io/meshkit/models/meshmodel/core/v1beta1" "github.com/layer5io/meshkit/utils/manifests" ) @@ -40,7 +40,7 @@ spec: jsonPath: .metadata.creationTimestamp name: Age type: date - name: v1alpha1 + name: v1beta1 schema: openAPIV3Schema: properties: @@ -124,21 +124,21 @@ spec: status: {} ` -func getNewComponent(spec string, name string, version string) v1alpha1.ComponentDefinition { - comp := v1alpha1.ComponentDefinition{} - comp.Schema = spec +func getNewComponent(spec string, name string, version string) v1beta1.ComponentDefinition { + comp := v1beta1.ComponentDefinition{} + comp.Component.Schema = spec comp.DisplayName = manifests.FormatToReadableString(name) - comp.APIVersion = version - comp.Kind = name + comp.Component.Version = version + comp.Component.Kind = name return comp } func TestGenerate(t *testing.T) { var tests = []struct { crd string - want v1alpha1.ComponentDefinition + want v1beta1.ComponentDefinition }{ - {istioCrd, getNewComponent("", "WasmPlugin", "extensions.istio.io/v1alpha1")}, + {istioCrd, getNewComponent("", "WasmPlugin", "extensions.istio.io/v1beta1")}, } for _, tt := range tests { t.Run("generateComponent", func(t *testing.T) { @@ -146,11 +146,11 @@ func TestGenerate(t *testing.T) { if got.DisplayName != tt.want.DisplayName { t.Errorf("got %v, want %v", got.DisplayName, tt.want.DisplayName) } - if !(got.Kind == tt.want.Kind) { - t.Errorf("got %v, want %v", got.Kind, tt.want.Kind) + if !(got.Component.Kind == tt.want.Component.Kind) { + t.Errorf("got %v, want %v", got.Component.Kind, tt.want.Component.Kind) } - if !(got.APIVersion == tt.want.APIVersion) { - t.Errorf("got %v, want %v", got.APIVersion, tt.want.APIVersion) + if !(got.Component.Version == tt.want.Component.Version) { + t.Errorf("got %v, want %v", got.Component.Version, tt.want.Component.Version) } }) } From 7c35024f1cd7e0011b2ce281223d7167754c8eac Mon Sep 17 00:00:00 2001 From: MUzairS15 Date: Tue, 26 Mar 2024 16:30:56 +0530 Subject: [PATCH 16/32] move filters to their versioned packages Signed-off-by: MUzairS15 --- .../core/policies/rego_policy_relationship.go | 3 +- models/meshmodel/entity/types.go | 5 +- models/meshmodel/registry/registry.go | 166 +----------------- .../registry/{ => v1beta1}/category_filter.go | 2 +- .../{ => v1beta1}/component_filter.go | 2 +- .../registry/{ => v1beta1}/model_filter.go | 107 +++++------ .../registry/{ => v1beta1}/policy_filter.go | 2 +- .../{ => v1beta1}/relationship_filter.go | 2 +- 8 files changed, 63 insertions(+), 226 deletions(-) rename models/meshmodel/registry/{ => v1beta1}/category_filter.go (98%) rename models/meshmodel/registry/{ => v1beta1}/component_filter.go (99%) rename models/meshmodel/registry/{ => v1beta1}/model_filter.go (69%) rename models/meshmodel/registry/{ => v1beta1}/policy_filter.go (98%) rename models/meshmodel/registry/{ => v1beta1}/relationship_filter.go (99%) diff --git a/models/meshmodel/core/policies/rego_policy_relationship.go b/models/meshmodel/core/policies/rego_policy_relationship.go index a7f0f1f0..dcd91459 100644 --- a/models/meshmodel/core/policies/rego_policy_relationship.go +++ b/models/meshmodel/core/policies/rego_policy_relationship.go @@ -5,6 +5,7 @@ import ( "fmt" "github.com/layer5io/meshkit/models/meshmodel/registry" + "github.com/layer5io/meshkit/models/meshmodel/registry/v1beta1" "github.com/layer5io/meshkit/utils" "github.com/open-policy-agent/opa/rego" "github.com/open-policy-agent/opa/storage" @@ -25,7 +26,7 @@ func NewRegoInstance(policyDir string, regManager *registry.RegistryManager) (*R var store storage.Store ctx := context.Background() - registeredRelationships, _, _, err := regManager.GetEntities(®istry.RelationshipFilter{}) + registeredRelationships, _, _, err := regManager.GetEntities(&v1beta1.RelationshipFilter{}) if err != nil { return nil, err } diff --git a/models/meshmodel/entity/types.go b/models/meshmodel/entity/types.go index fe4a288f..f5c19866 100644 --- a/models/meshmodel/entity/types.go +++ b/models/meshmodel/entity/types.go @@ -21,11 +21,10 @@ type Filter interface { } type Entity interface { -// Entity is referred as any type of schema managed by the registry -// ComponentDefinitions and PolicyDefinitions are examples of entities + // Entity is referred as any type of schema managed by the registry + // ComponentDefinitions and PolicyDefinitions are examples of entities Type() EntityType GetEntityDetail() string GetID() uuid.UUID - Status Create(db *database.Handler) (entityID uuid.UUID, err error) } diff --git a/models/meshmodel/registry/registry.go b/models/meshmodel/registry/registry.go index 22528289..ec347ab1 100644 --- a/models/meshmodel/registry/registry.go +++ b/models/meshmodel/registry/registry.go @@ -153,7 +153,7 @@ func (rm *RegistryManager) UpdateEntityStatus(ID string, status string, entityTy return nil } } - +// to be removed func (rm *RegistryManager) GetRegistrants(f *v1beta1.HostFilter) ([]v1beta1.MeshModelHostsWithEntitySummary, int64, error) { var result []v1beta1.MesheryHostSummaryDB var totalcount int64 @@ -215,172 +215,8 @@ func (rm *RegistryManager) GetRegistrants(f *v1beta1.HostFilter) ([]v1beta1.Mesh func (rm *RegistryManager) GetEntities(f entity.Filter) ([]entity.Entity, int64, int, error) { return f.Get(rm.db) - - // switch filter := f.(type) { - // case *v1beta1.ComponentFilter: - // en := make([]Entity, 0) - // comps, count, unique := v1beta1.GetMeshModelComponents(rm.db, *filter) - // for _, comp := range comps { - // en = append(en, comp) - // } - // return en, &count, &unique - // case *v1beta1.RelationshipFilter: - // en := make([]Entity, 0) - // relationships, count := v1beta1.GetMeshModelRelationship(rm.db, *filter) - // for _, rel := range relationships { - // en = append(en, rel) - // } - // return en, &count, nil - // case *v1beta1.PolicyFilter: - // en := make([]Entity, 0) - // policies := v1beta1.GetMeshModelPolicy(rm.db, *filter) - // for _, pol := range policies { - // en = append(en, pol) - // } - // return en, nil, nil - // default: - // return nil, nil, nil - // } -} - -func (rm *RegistryManager) GetModels(db *database.Handler, f entity.Filter) ([]v1beta1.Model, int64, int) { - var m []v1beta1.Model - type modelWithCategories struct { - v1beta1.ModelDB - v1beta1.CategoryDB - Registry - v1beta1.Hostv1beta1 - } - - countUniqueModels := func(models []modelWithCategories) int { - set := make(map[string]struct{}) - for _, model := range models { - key := model.ModelDB.Name + "@" + model.ModelDB.Version - if _, ok := set[key]; !ok { - set[key] = struct{}{} - } - } - return len(set) - } - - var modelWithCategoriess []modelWithCategories - finder := db.Model(&v1beta1.ModelDB{}). - Select("model_dbs.*, category_dbs.*", "registries.*", "hosts.*"). - Joins("JOIN category_dbs ON model_dbs.category_id = category_dbs.id"). - Joins("JOIN registries ON registries.entity = model_dbs.id"). - Joins("JOIN hosts ON hosts.id = registries.registrant_id") - - // total count before pagination - var count int64 - - // include components and relationships in response body - var includeComponents, includeRelationships bool - - if mf, ok := f.(*ModelFilter); ok { - if mf.Greedy { - if mf.Name != "" && mf.DisplayName != "" { - finder = finder.Where("model_dbs.name LIKE ? OR model_dbs.display_name LIKE ?", "%"+mf.Name+"%", "%"+mf.DisplayName+"%") - } else if mf.Name != "" { - finder = finder.Where("model_dbs.name LIKE ?", "%"+mf.Name+"%") - } else if mf.DisplayName != "" { - finder = finder.Where("model_dbs.display_name LIKE ?", "%"+mf.DisplayName+"%") - } - } else { - if mf.Name != "" { - finder = finder.Where("model_dbs.name = ?", mf.Name) - } - if mf.DisplayName != "" { - finder = finder.Where("model_dbs.display_name = ?", mf.DisplayName) - } - } - if mf.Annotations == "true" { - finder = finder.Where("model_dbs.metadata->>'isAnnotation' = true") - } else if mf.Annotations == "false" { - finder = finder.Where("model_dbs.metadata->>'isAnnotation' = false") - } - if mf.Version != "" { - finder = finder.Where("model_dbs.version = ?", mf.Version) - } - if mf.Category != "" { - finder = finder.Where("category_dbs.name = ?", mf.Category) - } - if mf.Registrant != "" { - finder = finder.Where("hosts.hostname = ?", mf.Registrant) - } - if mf.Annotations == "true" { - finder = finder.Where("model_dbs.metadata->>'isAnnotation' = true") - } else if mf.Annotations == "false" { - finder = finder.Where("model_dbs.metadata->>'isAnnotation' = false") - } - if mf.OrderOn != "" { - if mf.Sort == "desc" { - finder = finder.Order(clause.OrderByColumn{Column: clause.Column{Name: mf.OrderOn}, Desc: true}) - } else { - finder = finder.Order(mf.OrderOn) - } - } else { - finder = finder.Order("display_name") - } - - finder.Count(&count) - - if mf.Limit != 0 { - finder = finder.Limit(mf.Limit) - } - if mf.Offset != 0 { - finder = finder.Offset(mf.Offset) - } - if mf.Status != "" { - finder = finder.Where("model_dbs.status = ?", mf.Status) - } - includeComponents = mf.Components - includeRelationships = mf.Relationships - } - err := finder. - Scan(&modelWithCategoriess).Error - if err != nil { - fmt.Println(modelWithCategoriess) - fmt.Println(err.Error()) //for debugging - } - - for _, modelDB := range modelWithCategoriess { - reg := v1beta1.Hostv1beta1{} - model := modelDB.ModelDB.GetModel(modelDB.CategoryDB.GetCategory(db), reg) - // host := rm.GetRegistrant(model) - // model.HostID = host.ID - // model.HostName = host.Hostname - // model.DisplayHostName = host.Hostname - - if includeComponents { - var components []v1beta1.ComponentDefinitionDB - finder := db.Model(&v1beta1.ComponentDefinitionDB{}). - Select("component_definition_dbs.id, component_definition_dbs.kind,component_definition_dbs.display_name, component_definition_dbs.api_version, component_definition_dbs.metadata"). - Where("component_definition_dbs.model_id = ?", model.ID) - if err := finder.Scan(&components).Error; err != nil { - fmt.Println(err) - } - // model.Components = components - } - if includeRelationships { - var relationships []v1alpha2.RelationshipDefinitionDB - finder := db.Model(&v1alpha2.RelationshipDefinitionDB{}). - Select("relationship_definition_dbs.*"). - Where("relationship_definition_dbs.model_id = ?", model.ID) - if err := finder.Scan(&relationships).Error; err != nil { - fmt.Println(err) - } - // model.Relationships = relationships - } - - m = append(m, model) - } - return m, count, countUniqueModels(modelWithCategoriess) } -func (rm *RegistryManager) GetCategories(db *database.Handler, f entity.Filter) ([]entity.Entity, int64) { - cat, count, _, _ := f.Get(rm.db) - return cat, count -} func (rm *RegistryManager) GetRegistrant(e entity.Entity) v1beta1.Hostv1beta1 { eID := e.GetID() var reg Registry diff --git a/models/meshmodel/registry/category_filter.go b/models/meshmodel/registry/v1beta1/category_filter.go similarity index 98% rename from models/meshmodel/registry/category_filter.go rename to models/meshmodel/registry/v1beta1/category_filter.go index e2a6cc5b..2126decf 100644 --- a/models/meshmodel/registry/category_filter.go +++ b/models/meshmodel/registry/v1beta1/category_filter.go @@ -1,4 +1,4 @@ -package registry +package v1beta1 import ( "github.com/layer5io/meshkit/database" diff --git a/models/meshmodel/registry/component_filter.go b/models/meshmodel/registry/v1beta1/component_filter.go similarity index 99% rename from models/meshmodel/registry/component_filter.go rename to models/meshmodel/registry/v1beta1/component_filter.go index 19ecccd7..4c1786b7 100644 --- a/models/meshmodel/registry/component_filter.go +++ b/models/meshmodel/registry/v1beta1/component_filter.go @@ -1,4 +1,4 @@ -package registry +package v1beta1 import ( "fmt" diff --git a/models/meshmodel/registry/model_filter.go b/models/meshmodel/registry/v1beta1/model_filter.go similarity index 69% rename from models/meshmodel/registry/model_filter.go rename to models/meshmodel/registry/v1beta1/model_filter.go index 28af4a1a..e77d8c94 100644 --- a/models/meshmodel/registry/model_filter.go +++ b/models/meshmodel/registry/v1beta1/model_filter.go @@ -1,4 +1,4 @@ -package registry +package v1beta1 import ( "fmt" @@ -42,6 +42,7 @@ func (mf *ModelFilter) Get(db *database.Handler) ([]entity.Entity, int64, int, e type modelWithCategories struct { v1beta1.ModelDB v1beta1.CategoryDB + // registry.Registry v1beta1.Hostv1beta1 } @@ -69,65 +70,65 @@ func (mf *ModelFilter) Get(db *database.Handler) ([]entity.Entity, int64, int, e // include components and relationships in response body var includeComponents, includeRelationships bool - if mf.Greedy { - if mf.Name != "" && mf.DisplayName != "" { - finder = finder.Where("model_dbs.name LIKE ? OR model_dbs.display_name LIKE ?", "%"+mf.Name+"%", "%"+mf.DisplayName+"%") - } else if mf.Name != "" { - finder = finder.Where("model_dbs.name LIKE ?", "%"+mf.Name+"%") - } else if mf.DisplayName != "" { - finder = finder.Where("model_dbs.display_name LIKE ?", "%"+mf.DisplayName+"%") - } - } else { - if mf.Name != "" { - finder = finder.Where("model_dbs.name = ?", mf.Name) - } - if mf.DisplayName != "" { - finder = finder.Where("model_dbs.display_name = ?", mf.DisplayName) - } - } - if mf.Annotations == "true" { - finder = finder.Where("model_dbs.metadata->>'isAnnotation' = true") - } else if mf.Annotations == "false" { - finder = finder.Where("model_dbs.metadata->>'isAnnotation' = false") - } - if mf.Version != "" { - finder = finder.Where("model_dbs.version = ?", mf.Version) + if mf.Greedy { + if mf.Name != "" && mf.DisplayName != "" { + finder = finder.Where("model_dbs.name LIKE ? OR model_dbs.display_name LIKE ?", "%"+mf.Name+"%", "%"+mf.DisplayName+"%") + } else if mf.Name != "" { + finder = finder.Where("model_dbs.name LIKE ?", "%"+mf.Name+"%") + } else if mf.DisplayName != "" { + finder = finder.Where("model_dbs.display_name LIKE ?", "%"+mf.DisplayName+"%") } - if mf.Category != "" { - finder = finder.Where("category_dbs.name = ?", mf.Category) + } else { + if mf.Name != "" { + finder = finder.Where("model_dbs.name = ?", mf.Name) } - if mf.Registrant != "" { - finder = finder.Where("hosts.hostname = ?", mf.Registrant) + if mf.DisplayName != "" { + finder = finder.Where("model_dbs.display_name = ?", mf.DisplayName) } - if mf.Annotations == "true" { - finder = finder.Where("model_dbs.metadata->>'isAnnotation' = true") - } else if mf.Annotations == "false" { - finder = finder.Where("model_dbs.metadata->>'isAnnotation' = false") - } - if mf.OrderOn != "" { - if mf.Sort == "desc" { - finder = finder.Order(clause.OrderByColumn{Column: clause.Column{Name: mf.OrderOn}, Desc: true}) - } else { - finder = finder.Order(mf.OrderOn) - } + } + if mf.Annotations == "true" { + finder = finder.Where("model_dbs.metadata->>'isAnnotation' = true") + } else if mf.Annotations == "false" { + finder = finder.Where("model_dbs.metadata->>'isAnnotation' = false") + } + if mf.Version != "" { + finder = finder.Where("model_dbs.version = ?", mf.Version) + } + if mf.Category != "" { + finder = finder.Where("category_dbs.name = ?", mf.Category) + } + if mf.Registrant != "" { + finder = finder.Where("hosts.hostname = ?", mf.Registrant) + } + if mf.Annotations == "true" { + finder = finder.Where("model_dbs.metadata->>'isAnnotation' = true") + } else if mf.Annotations == "false" { + finder = finder.Where("model_dbs.metadata->>'isAnnotation' = false") + } + if mf.OrderOn != "" { + if mf.Sort == "desc" { + finder = finder.Order(clause.OrderByColumn{Column: clause.Column{Name: mf.OrderOn}, Desc: true}) } else { - finder = finder.Order("display_name") + finder = finder.Order(mf.OrderOn) } + } else { + finder = finder.Order("display_name") + } - finder.Count(&count) + finder.Count(&count) + + if mf.Limit != 0 { + finder = finder.Limit(mf.Limit) + } + if mf.Offset != 0 { + finder = finder.Offset(mf.Offset) + } + if mf.Status != "" { + finder = finder.Where("model_dbs.status = ?", mf.Status) + } + includeComponents = mf.Components + includeRelationships = mf.Relationships - if mf.Limit != 0 { - finder = finder.Limit(mf.Limit) - } - if mf.Offset != 0 { - finder = finder.Offset(mf.Offset) - } - if mf.Status != "" { - finder = finder.Where("model_dbs.status = ?", mf.Status) - } - includeComponents = mf.Components - includeRelationships = mf.Relationships - err := finder. Scan(&modelWithCategoriess).Error if err != nil { diff --git a/models/meshmodel/registry/policy_filter.go b/models/meshmodel/registry/v1beta1/policy_filter.go similarity index 98% rename from models/meshmodel/registry/policy_filter.go rename to models/meshmodel/registry/v1beta1/policy_filter.go index 12edbd76..4affab35 100644 --- a/models/meshmodel/registry/policy_filter.go +++ b/models/meshmodel/registry/v1beta1/policy_filter.go @@ -1,4 +1,4 @@ -package registry +package v1beta1 import ( "github.com/layer5io/meshkit/database" diff --git a/models/meshmodel/registry/relationship_filter.go b/models/meshmodel/registry/v1beta1/relationship_filter.go similarity index 99% rename from models/meshmodel/registry/relationship_filter.go rename to models/meshmodel/registry/v1beta1/relationship_filter.go index f352b2e1..cbbba058 100644 --- a/models/meshmodel/registry/relationship_filter.go +++ b/models/meshmodel/registry/v1beta1/relationship_filter.go @@ -1,4 +1,4 @@ -package registry +package v1beta1 import ( "fmt" From b8c056aaa31fdf0d1e07c1ac60930bae2133b990 Mon Sep 17 00:00:00 2001 From: MUzairS15 Date: Tue, 26 Mar 2024 17:00:40 +0530 Subject: [PATCH 17/32] remove unneeded code Signed-off-by: MUzairS15 --- models/meshmodel/core/v1beta1/models.go | 4 -- models/meshmodel/registry/registry.go | 9 --- .../registry/v1beta1/component_filter.go | 5 -- .../registry/v1beta1/model_filter.go | 56 ++++++++++--------- 4 files changed, 30 insertions(+), 44 deletions(-) diff --git a/models/meshmodel/core/v1beta1/models.go b/models/meshmodel/core/v1beta1/models.go index c1a8dc23..b92b8d2a 100644 --- a/models/meshmodel/core/v1beta1/models.go +++ b/models/meshmodel/core/v1beta1/models.go @@ -15,8 +15,6 @@ import ( var modelCreationLock sync.Mutex //Each component/relationship will perform a check and if the model already doesn't exist, it will create a model. This lock will make sure that there are no race conditions. - - type model struct { Version string `json:"version,omitempty" yaml:"version"` } @@ -34,8 +32,6 @@ type Model struct { SubCategory string `json:"subCategory" gorm:"subCategory"` Metadata map[string]interface{} `json:"metadata" yaml:"modelMetadata"` Model model `json:"model,omitempty" gorm:"model"` - // Components []ComponentDefinitionDB `json:"components"` - // Relationships []RelationshipDefinitionDB `json:"relationships"` } type ModelDB struct { diff --git a/models/meshmodel/registry/registry.go b/models/meshmodel/registry/registry.go index ec347ab1..a570b16c 100644 --- a/models/meshmodel/registry/registry.go +++ b/models/meshmodel/registry/registry.go @@ -217,15 +217,6 @@ func (rm *RegistryManager) GetEntities(f entity.Filter) ([]entity.Entity, int64, return f.Get(rm.db) } -func (rm *RegistryManager) GetRegistrant(e entity.Entity) v1beta1.Hostv1beta1 { - eID := e.GetID() - var reg Registry - _ = rm.db.Where("entity = ?", eID).Find(®).Error - var h v1beta1.Hostv1beta1 - _ = rm.db.Where("id = ?", reg.RegistrantID).Find(&h).Error - return h -} - func HostnameToPascalCase(input string) string { parts := strings.Split(input, ".") caser := cases.Title(language.English) diff --git a/models/meshmodel/registry/v1beta1/component_filter.go b/models/meshmodel/registry/v1beta1/component_filter.go index 4c1786b7..8027f131 100644 --- a/models/meshmodel/registry/v1beta1/component_filter.go +++ b/models/meshmodel/registry/v1beta1/component_filter.go @@ -40,11 +40,6 @@ func (cf *ComponentFilter) Create(m map[string]interface{}) { } func (componentFilter *ComponentFilter) Get(db *database.Handler) ([]entity.Entity, int64, int, error) { - // componentFilter, err := utils.Cast[ComponentFilter](f) - // if err != nil { - // return nil, 0, 0, err - // } - countUniqueComponents := func(components []componentDefinitionWithModel) int { set := make(map[string]struct{}) for _, compWithModel := range components { diff --git a/models/meshmodel/registry/v1beta1/model_filter.go b/models/meshmodel/registry/v1beta1/model_filter.go index e77d8c94..f7326bf6 100644 --- a/models/meshmodel/registry/v1beta1/model_filter.go +++ b/models/meshmodel/registry/v1beta1/model_filter.go @@ -4,7 +4,6 @@ import ( "fmt" "github.com/layer5io/meshkit/database" - "github.com/layer5io/meshkit/models/meshmodel/core/v1alpha2" "github.com/layer5io/meshkit/models/meshmodel/core/v1beta1" "github.com/layer5io/meshkit/models/meshmodel/entity" "gorm.io/gorm/clause" @@ -68,7 +67,7 @@ func (mf *ModelFilter) Get(db *database.Handler) ([]entity.Entity, int64, int, e var count int64 // include components and relationships in response body - var includeComponents, includeRelationships bool + // var includeComponents, includeRelationships bool if mf.Greedy { if mf.Name != "" && mf.DisplayName != "" { @@ -126,8 +125,10 @@ func (mf *ModelFilter) Get(db *database.Handler) ([]entity.Entity, int64, int, e if mf.Status != "" { finder = finder.Where("model_dbs.status = ?", mf.Status) } - includeComponents = mf.Components - includeRelationships = mf.Relationships + + // is this required? not used by UI, confirm with Yash once + // includeComponents = mf.Components + // includeRelationships = mf.Relationships err := finder. Scan(&modelWithCategoriess).Error @@ -139,33 +140,36 @@ func (mf *ModelFilter) Get(db *database.Handler) ([]entity.Entity, int64, int, e defs := []entity.Entity{} for _, modelDB := range modelWithCategoriess { - reg := v1beta1.Hostv1beta1{} - model := modelDB.ModelDB.GetModel(modelDB.CategoryDB.GetCategory(db), reg) // host := rm.GetRegistrant(model) // model.HostID = host.ID // model.HostName = host.Hostname // model.DisplayHostName = host.Hostname + // all these above accounted by having registrant as an atribute in the model schema. + reg := modelDB.Hostv1beta1 + model := modelDB.ModelDB.GetModel(modelDB.CategoryDB.GetCategory(db), reg) - if includeComponents { - var components []v1beta1.ComponentDefinitionDB - finder := db.Model(&v1beta1.ComponentDefinitionDB{}). - Select("component_definition_dbs.id, component_definition_dbs.kind,component_definition_dbs.display_name, component_definition_dbs.api_version, component_definition_dbs.metadata"). - Where("component_definition_dbs.model_id = ?", model.ID) - if err := finder.Scan(&components).Error; err != nil { - fmt.Println(err) - } - // model.Components = components - } - if includeRelationships { - var relationships []v1alpha2.RelationshipDefinitionDB - finder := db.Model(&v1alpha2.RelationshipDefinitionDB{}). - Select("relationship_definition_dbs.*"). - Where("relationship_definition_dbs.model_id = ?", model.ID) - if err := finder.Scan(&relationships).Error; err != nil { - fmt.Println(err) - } - // model.Relationships = relationships - } + // is this required? not used by UI, confirm with Yash once + // if includeComponents { + // var components []v1beta1.ComponentDefinitionDB + // finder := db.Model(&v1beta1.ComponentDefinitionDB{}). + // Select("component_definition_dbs.id, component_definition_dbs.kind,component_definition_dbs.display_name, component_definition_dbs.api_version, component_definition_dbs.metadata"). + // Where("component_definition_dbs.model_id = ?", model.ID) + // if err := finder.Scan(&components).Error; err != nil { + // fmt.Println(err) + // } + // // model.Components = components + // } + // is this required? not used by UI, confirm with Yash once + // if includeRelationships { + // var relationships []v1alpha2.RelationshipDefinitionDB + // finder := db.Model(&v1alpha2.RelationshipDefinitionDB{}). + // Select("relationship_definition_dbs.*"). + // Where("relationship_definition_dbs.model_id = ?", model.ID) + // if err := finder.Scan(&relationships).Error; err != nil { + // fmt.Println(err) + // } + // // model.Relationships = relationships + // } defs = append(defs, &model) } return defs, count, countUniqueModels(modelWithCategoriess), nil From ddd4ce3df64dc3241d4d92760654922caa18b9f0 Mon Sep 17 00:00:00 2001 From: MUzairS15 Date: Tue, 26 Mar 2024 17:01:01 +0530 Subject: [PATCH 18/32] run gofmt Signed-off-by: MUzairS15 --- generators/github/git_repo.go | 8 ++++---- models/meshmodel/core/policies/data_models.go | 2 +- models/meshmodel/core/v1beta1/component.go | 1 - models/meshmodel/core/v1beta1/hostv1beta1.go | 4 ++-- models/meshmodel/registry/registry.go | 1 + utils/cue.go | 2 +- utils/utils.go | 2 +- 7 files changed, 10 insertions(+), 10 deletions(-) diff --git a/generators/github/git_repo.go b/generators/github/git_repo.go index ac162eed..66ba8133 100644 --- a/generators/github/git_repo.go +++ b/generators/github/git_repo.go @@ -3,14 +3,14 @@ package github import ( "bufio" "fmt" - "net/url" - "os" - "path/filepath" - "strings" "github.com/layer5io/meshkit/generators/models" "github.com/layer5io/meshkit/utils" "github.com/layer5io/meshkit/utils/helm" "github.com/layer5io/meshkit/utils/walker" + "net/url" + "os" + "path/filepath" + "strings" ) type GitRepo struct { diff --git a/models/meshmodel/core/policies/data_models.go b/models/meshmodel/core/policies/data_models.go index a563cb04..e37e4a50 100644 --- a/models/meshmodel/core/policies/data_models.go +++ b/models/meshmodel/core/policies/data_models.go @@ -18,4 +18,4 @@ type NetworkPolicyRegoResponse struct { ServiceDeploymentRelationships []RelationObject `json:"service_deployment_relationships,omitempty"` } -// Add response struct based on schema for all relationships evaluations. binding, inventory, network... \ No newline at end of file +// Add response struct based on schema for all relationships evaluations. binding, inventory, network... diff --git a/models/meshmodel/core/v1beta1/component.go b/models/meshmodel/core/v1beta1/component.go index f0559c84..734fba9c 100644 --- a/models/meshmodel/core/v1beta1/component.go +++ b/models/meshmodel/core/v1beta1/component.go @@ -35,7 +35,6 @@ type component struct { Schema string `json:"schema,omitempty" yaml:"schema"` } - // swagger:response ComponentDefinition // use NewComponent function for instantiating type ComponentDefinition struct { diff --git a/models/meshmodel/core/v1beta1/hostv1beta1.go b/models/meshmodel/core/v1beta1/hostv1beta1.go index 6d7ec968..9879f529 100644 --- a/models/meshmodel/core/v1beta1/hostv1beta1.go +++ b/models/meshmodel/core/v1beta1/hostv1beta1.go @@ -21,7 +21,7 @@ type Hostv1beta1 struct { Metadata string CreatedAt time.Time `json:"-"` UpdatedAt time.Time `json:"-"` - IHost _IHost `gorm:"-"` + IHost _IHost `gorm:"-"` } type MeshModelHostsWithEntitySummary struct { @@ -58,7 +58,7 @@ type HostFilter struct { Offset int } -func(h *Hostv1beta1) Create(db *database.Handler) (uuid.UUID, error) { +func (h *Hostv1beta1) Create(db *database.Handler) (uuid.UUID, error) { byt, err := json.Marshal(h) if err != nil { return uuid.UUID{}, err diff --git a/models/meshmodel/registry/registry.go b/models/meshmodel/registry/registry.go index a570b16c..1b06c5bd 100644 --- a/models/meshmodel/registry/registry.go +++ b/models/meshmodel/registry/registry.go @@ -153,6 +153,7 @@ func (rm *RegistryManager) UpdateEntityStatus(ID string, status string, entityTy return nil } } + // to be removed func (rm *RegistryManager) GetRegistrants(f *v1beta1.HostFilter) ([]v1beta1.MeshModelHostsWithEntitySummary, int64, error) { var result []v1beta1.MesheryHostSummaryDB diff --git a/utils/cue.go b/utils/cue.go index 2e90f863..64f0db4d 100644 --- a/utils/cue.go +++ b/utils/cue.go @@ -112,4 +112,4 @@ func Lookup(rootVal cue.Value, path string) (cue.Value, error) { } return res.Value(), nil -} \ No newline at end of file +} diff --git a/utils/utils.go b/utils/utils.go index 855f9874..cfd50df3 100644 --- a/utils/utils.go +++ b/utils/utils.go @@ -409,4 +409,4 @@ func IsSchemaEmpty(schema string) (valid bool) { } valid = true return -} \ No newline at end of file +} From b3d02d1f255c29c5bc30b23c9bc36362bc427d62 Mon Sep 17 00:00:00 2001 From: MUzairS15 Date: Tue, 26 Mar 2024 17:35:22 +0530 Subject: [PATCH 19/32] add hostID as param in interface function Signed-off-by: MUzairS15 --- .../meshmodel/core/v1alpha2/relationship.go | 4 +- models/meshmodel/core/v1beta1/component.go | 4 +- models/meshmodel/core/v1beta1/models.go | 26 ++++++++++--- models/meshmodel/entity/types.go | 2 +- models/meshmodel/registry/registry.go | 37 +------------------ 5 files changed, 27 insertions(+), 46 deletions(-) diff --git a/models/meshmodel/core/v1alpha2/relationship.go b/models/meshmodel/core/v1alpha2/relationship.go index 8eabea0a..b7efd1fc 100644 --- a/models/meshmodel/core/v1alpha2/relationship.go +++ b/models/meshmodel/core/v1alpha2/relationship.go @@ -52,9 +52,9 @@ func (r *RelationshipDefinition) GetEntityDetail() string { return fmt.Sprintf("type: %s, definition version: %s, kind: %s, model: %s, version: %s", r.Type(), r.Version, r.Kind, r.Model.Name, r.Model.Version) } -func (r *RelationshipDefinition) Create(db *database.Handler) (uuid.UUID, error) { +func (r *RelationshipDefinition) Create(db *database.Handler, hostID uuid.UUID) (uuid.UUID, error) { r.ID = uuid.New() - mid, err := r.Model.Create(db) + mid, err := r.Model.Create(db, hostID) if err != nil { return uuid.UUID{}, err } diff --git a/models/meshmodel/core/v1beta1/component.go b/models/meshmodel/core/v1beta1/component.go index 734fba9c..7bf17636 100644 --- a/models/meshmodel/core/v1beta1/component.go +++ b/models/meshmodel/core/v1beta1/component.go @@ -72,7 +72,7 @@ func (c *ComponentDefinition) GetEntityDetail() string { return fmt.Sprintf("type: %s, definition version: %s, name: %s, model: %s, version: %s", c.Type(), c.Version, c.DisplayName, c.Model.Name, c.Model.Version) } -func (c *ComponentDefinition) Create(db *database.Handler) (uuid.UUID, error) { +func (c *ComponentDefinition) Create(db *database.Handler, hostID uuid.UUID) (uuid.UUID, error) { c.ID = uuid.New() isAnnotation, _ := c.Metadata["isAnnotation"].(bool) @@ -82,7 +82,7 @@ func (c *ComponentDefinition) Create(db *database.Handler) (uuid.UUID, error) { return uuid.Nil, nil } - mid, err := c.Model.Create(db) + mid, err := c.Model.Create(db, hostID) if err != nil { return uuid.UUID{}, err } diff --git a/models/meshmodel/core/v1beta1/models.go b/models/meshmodel/core/v1beta1/models.go index b92b8d2a..72a32fbb 100644 --- a/models/meshmodel/core/v1beta1/models.go +++ b/models/meshmodel/core/v1beta1/models.go @@ -59,12 +59,7 @@ func (m *Model) GetEntityDetail() string { return fmt.Sprintf("type: %s, model: %s, definition version: %s, version: %s", m.Type(), m.Name, m.Version, m.Model.Version) } -func (m *Model) Create(db *database.Handler) (uuid.UUID, error) { - - hostID, err := m.Registrant.Create(db) - if err != nil { - return uuid.UUID{}, err - } +func (m *Model) Create(db *database.Handler, hostID uuid.UUID) (uuid.UUID, error) { byt, err := json.Marshal(m) if err != nil { @@ -95,6 +90,8 @@ func (m *Model) Create(db *database.Handler) (uuid.UUID, error) { if err != nil { return uuid.UUID{}, err } + // register model inside registries table + return mdb.ID, nil } return model.ID, nil @@ -155,3 +152,20 @@ func (c Model) WriteModelDefinition(modelDefPath string) error { } return nil } + +// Registers models into registries table. +func registerModel(db *database.Handler, regID, modelID uuid.UUID) error { + err := db.Select("registries.* from registries").Where("registrant_id = ?", regID).Where("type = ?", "model").Where("entity = ?", modelID).Error + + if err != nil && err != gorm.ErrRecordNotFound { + return err + } + + if err == gorm.ErrRecordNotFound { + err = db.Exec("INSERT INTO registries (registrant_id, entity, type) VALUES (?,?,?)", regID, modelID, "model").Error + if err != nil { + return err + } + } + return nil +} diff --git a/models/meshmodel/entity/types.go b/models/meshmodel/entity/types.go index f5c19866..e409b616 100644 --- a/models/meshmodel/entity/types.go +++ b/models/meshmodel/entity/types.go @@ -26,5 +26,5 @@ type Entity interface { Type() EntityType GetEntityDetail() string GetID() uuid.UUID - Create(db *database.Handler) (entityID uuid.UUID, err error) + Create(db *database.Handler, hostID uuid.UUID) (entityID uuid.UUID, err error) } diff --git a/models/meshmodel/registry/registry.go b/models/meshmodel/registry/registry.go index 1b06c5bd..84f54e8b 100644 --- a/models/meshmodel/registry/registry.go +++ b/models/meshmodel/registry/registry.go @@ -1,7 +1,6 @@ package registry import ( - "encoding/json" "fmt" "strings" "time" @@ -13,7 +12,6 @@ import ( "github.com/layer5io/meshkit/models/meshmodel/entity" "golang.org/x/text/cases" "golang.org/x/text/language" - "gorm.io/gorm" "gorm.io/gorm/clause" ) @@ -43,37 +41,6 @@ type RegistryManager struct { db *database.Handler //This database handler will be used to perform queries inside the database } -// Registers models into registries table. -func registerModel(db *database.Handler, regID, modelID uuid.UUID) error { - entity := Registry{ - RegistrantID: regID, - Entity: modelID, - Type: entity.Model, - } - - byt, err := json.Marshal(entity) - if err != nil { - return err - } - - entityID := uuid.NewSHA1(uuid.UUID{}, byt) - var reg Registry - err = db.First(®, "id = ?", entityID).Error - if err != nil && err != gorm.ErrRecordNotFound { - return err - } - - if err == gorm.ErrRecordNotFound { - entity.ID = entityID - err = db.Create(&entity).Error - if err != nil { - return err - } - } - - return nil -} - // NewRegistryManager initializes the registry manager by creating appropriate tables. // Any new entities that are added to the registry should be migrated here into the database func NewRegistryManager(db *database.Handler) (*RegistryManager, error) { @@ -113,7 +80,7 @@ func (rm *RegistryManager) RegisterEntity(h v1beta1.Hostv1beta1, en entity.Entit return err } - entityID, err := en.Create(rm.db) + entityID, err := en.Create(rm.db, registrantID) if err != nil { return err } @@ -144,7 +111,7 @@ func (rm *RegistryManager) UpdateEntityStatus(ID string, status string, entityTy switch entityType { case "models": model := v1beta1.Model{ID: entityID} - model.UpdateStatus(rm.db, entity.EntityStatus(status)) + err := model.UpdateStatus(rm.db, entity.EntityStatus(status)) if err != nil { return err } From 89313ef742cbe4e89a5ff7feec670c12c7b257af Mon Sep 17 00:00:00 2001 From: MUzairS15 Date: Tue, 26 Mar 2024 17:43:21 +0530 Subject: [PATCH 20/32] cleanup: Signed-off-by: MUzairS15 --- .../core/v1beta1/application_component.go | 7 +- .../core/v1beta1/{hostv1beta1.go => host.go} | 34 ++--- models/meshmodel/core/v1beta1/models.go | 13 +- models/meshmodel/core/v1beta1/policy.go | 4 +- models/meshmodel/registry/host.go | 121 ------------------ models/meshmodel/registry/registry.go | 12 +- .../registry/v1beta1/component_filter.go | 2 +- .../registry/v1beta1/model_filter.go | 4 +- .../registry/v1beta1/relationship_filter.go | 2 +- .../core/v1alpha1/application_component.go | 67 ---------- 10 files changed, 41 insertions(+), 225 deletions(-) rename models/meshmodel/core/v1beta1/{hostv1beta1.go => host.go} (79%) delete mode 100644 models/meshmodel/registry/host.go delete mode 100644 models/oam/core/v1alpha1/application_component.go diff --git a/models/meshmodel/core/v1beta1/application_component.go b/models/meshmodel/core/v1beta1/application_component.go index f6778c8d..c094f29b 100644 --- a/models/meshmodel/core/v1beta1/application_component.go +++ b/models/meshmodel/core/v1beta1/application_component.go @@ -6,8 +6,9 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) +// To be removed when Design schema is refactored. -// Component is the structure for the core OAM Application Component +// Component is the structure for the core Application Component type Component struct { metav1.TypeMeta `json:",inline"` metav1.ObjectMeta `json:"metadata,omitempty"` @@ -15,7 +16,7 @@ type Component struct { Spec ComponentSpec `json:"spec,omitempty"` } -// ComponentSpec is the structure for the core OAM Application Component Spec +// ComponentSpec is the structure for the core Application Component Spec type ComponentSpec struct { Type string `json:"type"` Version string `json:"version"` @@ -25,7 +26,7 @@ type ComponentSpec struct { Parameters []ComponentParameter `json:"parameters"` } -// ComponentParameter is the structure for the core OAM Application Component +// ComponentParameter is the structure for the core Application Component // Paramater type ComponentParameter struct { Name string `json:"name"` diff --git a/models/meshmodel/core/v1beta1/hostv1beta1.go b/models/meshmodel/core/v1beta1/host.go similarity index 79% rename from models/meshmodel/core/v1beta1/hostv1beta1.go rename to models/meshmodel/core/v1beta1/host.go index 9879f529..1208aa38 100644 --- a/models/meshmodel/core/v1beta1/hostv1beta1.go +++ b/models/meshmodel/core/v1beta1/host.go @@ -12,16 +12,16 @@ import ( "gorm.io/gorm" ) -var hostv1beta1CreationLock sync.Mutex //Each entity will perform a check and if the host already doesn't exist, it will create a host. This lock will make sure that there are no race conditions. +var hostCreationLock sync.Mutex //Each entity will perform a check and if the host already doesn't exist, it will create a host. This lock will make sure that there are no race conditions. -type Hostv1beta1 struct { +type Host struct { ID uuid.UUID `json:"-"` Hostname string Port int Metadata string CreatedAt time.Time `json:"-"` UpdatedAt time.Time `json:"-"` - IHost _IHost `gorm:"-"` + IHost IHost `gorm:"-"` } type MeshModelHostsWithEntitySummary struct { @@ -58,15 +58,15 @@ type HostFilter struct { Offset int } -func (h *Hostv1beta1) Create(db *database.Handler) (uuid.UUID, error) { +func (h *Host) Create(db *database.Handler) (uuid.UUID, error) { byt, err := json.Marshal(h) if err != nil { return uuid.UUID{}, err } hID := uuid.NewSHA1(uuid.UUID{}, byt) - var host Hostv1beta1 - hostv1beta1CreationLock.Lock() - defer hostv1beta1CreationLock.Unlock() + var host Host + hostCreationLock.Lock() + defer hostCreationLock.Unlock() err = db.First(&host, "id = ?", hID).Error // check if the host already exists if err != nil && err != gorm.ErrRecordNotFound { return uuid.UUID{}, err @@ -86,12 +86,12 @@ func (h *Hostv1beta1) Create(db *database.Handler) (uuid.UUID, error) { return host.ID, nil } -func (h *Hostv1beta1) AfterFind(tx *gorm.DB) error { +func (h *Host) AfterFind(tx *gorm.DB) error { switch h.Hostname { case "artifacthub": - h.IHost = _ArtifactHub{} + h.IHost = ArtifactHub{} case "kubernetes": - h.IHost = _Kubernetes{} + h.IHost = Kubernetes{} default: // do nothing if the host is not pre-unknown. Currently adapters fall into this case. return nil } @@ -100,14 +100,14 @@ func (h *Hostv1beta1) AfterFind(tx *gorm.DB) error { // Each host from where meshmodels can be generated needs to implement this interface // HandleDependents, contains host specific logic for provisioning required CRDs/operators for corresponding components. -type _IHost interface { +type IHost interface { HandleDependents(comp Component, kc *kubernetes.Client, isDeploy bool) (string, error) String() string } -type _ArtifactHub struct{} +type ArtifactHub struct{} -func (ah _ArtifactHub) HandleDependents(comp Component, kc *kubernetes.Client, isDeploy bool) (summary string, err error) { +func (ah ArtifactHub) HandleDependents(comp Component, kc *kubernetes.Client, isDeploy bool) (summary string, err error) { source_uri := comp.Annotations[fmt.Sprintf("%s.model.source_uri", MesheryAnnotationPrefix)] act := kubernetes.UNINSTALL if isDeploy { @@ -139,16 +139,16 @@ func (ah _ArtifactHub) HandleDependents(comp Component, kc *kubernetes.Client, i return summary, err } -func (ah _ArtifactHub) String() string { +func (ah ArtifactHub) String() string { return "artifacthub" } -type _Kubernetes struct{} +type Kubernetes struct{} -func (k _Kubernetes) HandleDependents(comp Component, kc *kubernetes.Client, isDeploy bool) (summary string, err error) { +func (k Kubernetes) HandleDependents(comp Component, kc *kubernetes.Client, isDeploy bool) (summary string, err error) { return summary, err } -func (k _Kubernetes) String() string { +func (k Kubernetes) String() string { return "kubernetes" } diff --git a/models/meshmodel/core/v1beta1/models.go b/models/meshmodel/core/v1beta1/models.go index 72a32fbb..a2cd386f 100644 --- a/models/meshmodel/core/v1beta1/models.go +++ b/models/meshmodel/core/v1beta1/models.go @@ -27,7 +27,7 @@ type Model struct { DisplayName string `json:"displayName" gorm:"modelDisplayName"` Description string `json:"description" gorm:"description"` Status entity.EntityStatus `json:"status" gorm:"status"` - Registrant Hostv1beta1 `json:"registrant" gorm:"registrant"` // to be Connection + Registrant Host `json:"registrant" gorm:"registrant"` // to be Connection Category Category `json:"category"` SubCategory string `json:"subCategory" gorm:"subCategory"` Metadata map[string]interface{} `json:"metadata" yaml:"modelMetadata"` @@ -41,7 +41,7 @@ type ModelDB struct { DisplayName string `json:"modelDisplayName" gorm:"modelDisplayName"` Description string `json:"description" gorm:"description"` Status entity.EntityStatus `json:"status" gorm:"status"` - RegistrantID uuid.UUID `json:"hostID" gorm:"hostID"` + RegistrantID uuid.UUID `json:"hostID" gorm:"host_id"` CategoryID uuid.UUID `json:"-" gorm:"categoryID"` SubCategory string `json:"subCategory" gorm:"subCategory"` Metadata []byte `json:"modelMetadata" gorm:"modelMetadata"` @@ -72,7 +72,7 @@ func (m *Model) Create(db *database.Handler, hostID uuid.UUID) (uuid.UUID, error } modelCreationLock.Lock() defer modelCreationLock.Unlock() - err = db.First(&model, "id = ? and hostID = ?", modelID, hostID).Error + err = db.First(&model, "id = ? and host_id = ?", modelID, hostID).Error if err != nil && err != gorm.ErrRecordNotFound { return uuid.UUID{}, err } @@ -91,7 +91,10 @@ func (m *Model) Create(db *database.Handler, hostID uuid.UUID) (uuid.UUID, error return uuid.UUID{}, err } // register model inside registries table - + err = registerModel(db, hostID, modelID) + if err != nil { + return uuid.UUID{}, err + } return mdb.ID, nil } return model.ID, nil @@ -122,7 +125,7 @@ func (c *Model) GetModelDB() (cmd ModelDB) { } // is reg should be passed as param? -func (cmd *ModelDB) GetModel(cat Category, reg Hostv1beta1) (c Model) { +func (cmd *ModelDB) GetModel(cat Category, reg Host) (c Model) { c.ID = cmd.ID c.VersionMeta = cmd.VersionMeta c.Name = cmd.Name diff --git a/models/meshmodel/core/v1beta1/policy.go b/models/meshmodel/core/v1beta1/policy.go index 0922e132..547c79a2 100644 --- a/models/meshmodel/core/v1beta1/policy.go +++ b/models/meshmodel/core/v1beta1/policy.go @@ -45,9 +45,9 @@ func (p *PolicyDefinition) GetEntityDetail() string { return fmt.Sprintf("type: %s, definition version: %s, name: %s, model: %s, version: %s", p.Type(), p.Version, p.Kind, p.Model.Name, p.Model.Version) } -func (p *PolicyDefinition) Create(db *database.Handler) (uuid.UUID, error) { +func (p *PolicyDefinition) Create(db *database.Handler, hostID uuid.UUID) (uuid.UUID, error) { p.ID = uuid.New() - mid, err := p.Model.Create(db) + mid, err := p.Model.Create(db, hostID) if err != nil { return uuid.UUID{}, err } diff --git a/models/meshmodel/registry/host.go b/models/meshmodel/registry/host.go deleted file mode 100644 index aee5a13a..00000000 --- a/models/meshmodel/registry/host.go +++ /dev/null @@ -1,121 +0,0 @@ -package registry - -// import ( -// "encoding/json" -// "fmt" -// "sync" -// "time" - -// "github.com/google/uuid" -// "github.com/layer5io/meshkit/database" -// "github.com/layer5io/meshkit/models/oam/core/v1alpha1" -// "github.com/layer5io/meshkit/utils/kubernetes" -// "gorm.io/gorm" -// ) - -// var hostCreationLock sync.Mutex //Each entity will perform a check and if the host already doesn't exist, it will create a host. This lock will make sure that there are no race conditions. - -// type Host struct { -// ID uuid.UUID `json:"-"` -// Hostname string -// Port int -// Metadata string -// CreatedAt time.Time `json:"-"` -// UpdatedAt time.Time `json:"-"` -// IHost IHost `gorm:"-"` -// } - -// func createHost(db *database.Handler, h Host) (uuid.UUID, error) { -// byt, err := json.Marshal(h) -// if err != nil { -// return uuid.UUID{}, err -// } -// hID := uuid.NewSHA1(uuid.UUID{}, byt) -// var host Host -// hostCreationLock.Lock() -// defer hostCreationLock.Unlock() -// err = db.First(&host, "id = ?", hID).Error // check if the host already exists -// if err != nil && err != gorm.ErrRecordNotFound { -// return uuid.UUID{}, err -// } - -// // if not exists then create a new host and return the id -// if err == gorm.ErrRecordNotFound { -// h.ID = hID -// err = db.Create(&h).Error -// if err != nil { -// return uuid.UUID{}, err -// } -// return h.ID, nil -// } - -// // else return the id of the existing host -// return host.ID, nil -// } - -// func (h *Host) AfterFind(tx *gorm.DB) error { -// switch h.Hostname { -// case "artifacthub": -// h.IHost = ArtifactHub{} -// case "kubernetes": -// h.IHost = Kubernetes{} -// default: // do nothing if the host is not pre-unknown. Currently adapters fall into this case. -// return nil -// } -// return nil -// } - -// // Each host from where meshmodels can be generated needs to implement this interface -// // HandleDependents, contains host specific logic for provisioning required CRDs/operators for corresponding components. -// type IHost interface { -// HandleDependents(comp v1alpha1.Component, kc *kubernetes.Client, isDeploy bool) (string, error) -// String() string -// } - -// type ArtifactHub struct{} - -// func (ah ArtifactHub) HandleDependents(comp v1alpha1.Component, kc *kubernetes.Client, isDeploy bool) (summary string, err error) { -// source_uri := comp.Annotations[fmt.Sprintf("%s.model.source_uri", v1alpha1.MesheryAnnotationPrefix)] -// act := kubernetes.UNINSTALL -// if isDeploy { -// act = kubernetes.INSTALL -// } - -// if source_uri != "" { -// err = kc.ApplyHelmChart(kubernetes.ApplyHelmChartConfig{ -// URL: source_uri, -// Namespace: comp.Namespace, -// CreateNamespace: true, -// Action: act, -// SkipUpgradeIfInstalled: true, -// }) -// if err != nil { -// if !isDeploy { -// summary = fmt.Sprintf("error undeploying dependent helm chart for %s, please proceed with manual uninstall or try again", comp.Name) -// } else { -// summary = fmt.Sprintf("error deploying dependent helm chart for %s, please procced with manual install or try again", comp.Name) -// } -// } else { -// if !isDeploy { -// summary = fmt.Sprintf("Undeployed dependent helm chart for %s", comp.Name) -// } else { -// summary = fmt.Sprintf("Deployed dependent helm chart for %s", comp.Name) -// } -// } -// } -// return summary, err -// } - -// func (ah ArtifactHub) String() string { -// return "artifacthub" -// } - -// type Kubernetes struct{} - -// func (k Kubernetes) HandleDependents(comp v1alpha1.Component, kc *kubernetes.Client, isDeploy bool) (summary string, err error) { -// return summary, err -// } - -// func (k Kubernetes) String() string { -// return "kubernetes" -// } diff --git a/models/meshmodel/registry/registry.go b/models/meshmodel/registry/registry.go index 84f54e8b..f55ea199 100644 --- a/models/meshmodel/registry/registry.go +++ b/models/meshmodel/registry/registry.go @@ -23,9 +23,9 @@ import ( // 2. Entity type // 3. Entity type MeshModelRegistrantData struct { - Host v1beta1.Hostv1beta1 `json:"host"` - EntityType entity.EntityType `json:"entityType"` - Entity []byte `json:"entity"` //This will be type converted to appropriate entity on server based on passed entity type + Host v1beta1.Host `json:"host"` + EntityType entity.EntityType `json:"entityType"` + Entity []byte `json:"entity"` //This will be type converted to appropriate entity on server based on passed entity type } type Registry struct { ID uuid.UUID @@ -52,7 +52,7 @@ func NewRegistryManager(db *database.Handler) (*RegistryManager, error) { } err := rm.db.AutoMigrate( &Registry{}, - &v1beta1.Hostv1beta1{}, + &v1beta1.Host{}, &v1beta1.ComponentDefinitionDB{}, &v1alpha2.RelationshipDefinitionDB{}, &v1beta1.PolicyDefinitionDB{}, @@ -67,14 +67,14 @@ func NewRegistryManager(db *database.Handler) (*RegistryManager, error) { func (rm *RegistryManager) Cleanup() { _ = rm.db.Migrator().DropTable( &Registry{}, - &v1beta1.Hostv1beta1{}, + &v1beta1.Host{}, &v1beta1.ComponentDefinitionDB{}, &v1beta1.ModelDB{}, &v1beta1.CategoryDB{}, &v1alpha2.RelationshipDefinitionDB{}, ) } -func (rm *RegistryManager) RegisterEntity(h v1beta1.Hostv1beta1, en entity.Entity) error { +func (rm *RegistryManager) RegisterEntity(h v1beta1.Host, en entity.Entity) error { registrantID, err := h.Create(rm.db) if err != nil { return err diff --git a/models/meshmodel/registry/v1beta1/component_filter.go b/models/meshmodel/registry/v1beta1/component_filter.go index 8027f131..0c097921 100644 --- a/models/meshmodel/registry/v1beta1/component_filter.go +++ b/models/meshmodel/registry/v1beta1/component_filter.go @@ -28,7 +28,7 @@ type componentDefinitionWithModel struct { v1beta1.ComponentDefinitionDB ModelDB v1beta1.ModelDB // acoount for overridn fields CategoryDB v1beta1.CategoryDB - HostsDB v1beta1.Hostv1beta1 + HostsDB v1beta1.Host } // Create the filter from map[string]interface{} diff --git a/models/meshmodel/registry/v1beta1/model_filter.go b/models/meshmodel/registry/v1beta1/model_filter.go index f7326bf6..71f76c03 100644 --- a/models/meshmodel/registry/v1beta1/model_filter.go +++ b/models/meshmodel/registry/v1beta1/model_filter.go @@ -42,7 +42,7 @@ func (mf *ModelFilter) Get(db *database.Handler) ([]entity.Entity, int64, int, e v1beta1.ModelDB v1beta1.CategoryDB // registry.Registry - v1beta1.Hostv1beta1 + v1beta1.Host } countUniqueModels := func(models []modelWithCategories) int { @@ -145,7 +145,7 @@ func (mf *ModelFilter) Get(db *database.Handler) ([]entity.Entity, int64, int, e // model.HostName = host.Hostname // model.DisplayHostName = host.Hostname // all these above accounted by having registrant as an atribute in the model schema. - reg := modelDB.Hostv1beta1 + reg := modelDB.Host model := modelDB.ModelDB.GetModel(modelDB.CategoryDB.GetCategory(db), reg) // is this required? not used by UI, confirm with Yash once diff --git a/models/meshmodel/registry/v1beta1/relationship_filter.go b/models/meshmodel/registry/v1beta1/relationship_filter.go index cbbba058..05d96f42 100644 --- a/models/meshmodel/registry/v1beta1/relationship_filter.go +++ b/models/meshmodel/registry/v1beta1/relationship_filter.go @@ -99,7 +99,7 @@ func (relationshipFilter *RelationshipFilter) Get(db *database.Handler) ([]entit for _, cm := range relationshipDefinitionsWithModel { // Ensure correct reg is passed, rn it is dummy for sake of testing. // In the first query above where we do seelection i think there changes will be requrired, an when that two def and defDB structs are consolidated, using association and preload i think we can do. - reg := v1beta1.Hostv1beta1{} + reg := v1beta1.Host{} rd := cm.RelationshipDefinitionDB.GetRelationshipDefinition(cm.ModelDB.GetModel(cm.CategoryDB.GetCategory(db), reg)) defs = append(defs, &rd) } diff --git a/models/oam/core/v1alpha1/application_component.go b/models/oam/core/v1alpha1/application_component.go deleted file mode 100644 index a983f9c7..00000000 --- a/models/oam/core/v1alpha1/application_component.go +++ /dev/null @@ -1,67 +0,0 @@ -package v1alpha1 - -// import ( -// "fmt" -// "strings" - -// "github.com/layer5io/meshkit/models/meshmodel/core/v1alpha1" -// metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" -// ) - -// // Component is the structure for the core OAM Application Component -// type Component struct { -// metav1.TypeMeta `json:",inline"` -// metav1.ObjectMeta `json:"metadata,omitempty"` - -// Spec ComponentSpec `json:"spec,omitempty"` -// } - -// // ComponentSpec is the structure for the core OAM Application Component Spec -// type ComponentSpec struct { -// Type string `json:"type"` -// Version string `json:"version"` -// APIVersion string `json:"apiVersion"` -// Model string `json:"model"` -// Settings map[string]interface{} `json:"settings"` -// Parameters []ComponentParameter `json:"parameters"` -// } - -// // ComponentParameter is the structure for the core OAM Application Component -// // Paramater -// type ComponentParameter struct { -// Name string `json:"name"` -// FieldPaths []string `json:"fieldPaths"` -// Required *bool `json:"required,omitempty"` -// Description *string `json:"description,omitempty"` -// } - -// const MesheryAnnotationPrefix = "design.meshmodel.io" - -// func GetAPIVersionFromComponent(comp Component) string { -// return comp.Annotations[MesheryAnnotationPrefix+".k8s.APIVersion"] -// } - -// func GetKindFromComponent(comp Component) string { -// kind := strings.TrimPrefix(comp.Annotations[MesheryAnnotationPrefix+".k8s.Kind"], "/") -// return kind -// } - -// func GetAnnotationsForWorkload(w v1alpha1.ComponentDefinition) map[string]string { -// res := map[string]string{} - -// for key, val := range w.Metadata { -// if v, ok := val.(string); ok { -// res[strings.ReplaceAll(fmt.Sprintf("%s.%s", MesheryAnnotationPrefix, key), " ", "")] = v -// } -// } -// sourceURI, ok := w.Model.Metadata["source_uri"].(string) -// if ok { -// res[fmt.Sprintf("%s.model.source_uri", MesheryAnnotationPrefix)] = sourceURI -// } -// res[fmt.Sprintf("%s.model.name", MesheryAnnotationPrefix)] = w.Model.Name -// res[fmt.Sprintf("%s.k8s.APIVersion", MesheryAnnotationPrefix)] = w.APIVersion -// res[fmt.Sprintf("%s.k8s.Kind", MesheryAnnotationPrefix)] = w.Kind -// res[fmt.Sprintf("%s.model.version", MesheryAnnotationPrefix)] = w.Model.Version -// res[fmt.Sprintf("%s.model.category", MesheryAnnotationPrefix)] = w.Model.Category.Name -// return res -// } From 1e6a189b8c2ac56bebb92fc0477340f6f9208ca2 Mon Sep 17 00:00:00 2001 From: MUzairS15 Date: Fri, 29 Mar 2024 15:47:27 +0530 Subject: [PATCH 21/32] fix tags Signed-off-by: MUzairS15 --- .../core/policies/rego_policy_relationship.go | 4 ++-- models/meshmodel/core/v1alpha2/relationship.go | 3 +-- models/meshmodel/core/v1beta1/category.go | 15 ++++++++++++++- models/meshmodel/core/v1beta1/component.go | 10 +++++----- models/meshmodel/core/v1beta1/host.go | 8 ++++---- models/meshmodel/core/v1beta1/models.go | 15 ++++++++------- models/meshmodel/entity/types.go | 1 + models/meshmodel/registry/registry.go | 9 +++++++++ .../{v1beta1 => v1alpha2}/relationship_filter.go | 6 +++--- .../meshmodel/registry/v1beta1/category_filter.go | 8 +++++--- utils/component/generator.go | 2 ++ 11 files changed, 54 insertions(+), 27 deletions(-) rename models/meshmodel/registry/{v1beta1 => v1alpha2}/relationship_filter.go (98%) diff --git a/models/meshmodel/core/policies/rego_policy_relationship.go b/models/meshmodel/core/policies/rego_policy_relationship.go index dcd91459..a77b6862 100644 --- a/models/meshmodel/core/policies/rego_policy_relationship.go +++ b/models/meshmodel/core/policies/rego_policy_relationship.go @@ -5,7 +5,7 @@ import ( "fmt" "github.com/layer5io/meshkit/models/meshmodel/registry" - "github.com/layer5io/meshkit/models/meshmodel/registry/v1beta1" + "github.com/layer5io/meshkit/models/meshmodel/registry/v1alpha2" "github.com/layer5io/meshkit/utils" "github.com/open-policy-agent/opa/rego" "github.com/open-policy-agent/opa/storage" @@ -26,7 +26,7 @@ func NewRegoInstance(policyDir string, regManager *registry.RegistryManager) (*R var store storage.Store ctx := context.Background() - registeredRelationships, _, _, err := regManager.GetEntities(&v1beta1.RelationshipFilter{}) + registeredRelationships, _, _, err := regManager.GetEntities(&v1alpha2.RelationshipFilter{}) if err != nil { return nil, err } diff --git a/models/meshmodel/core/v1alpha2/relationship.go b/models/meshmodel/core/v1alpha2/relationship.go index b7efd1fc..2e6068d4 100644 --- a/models/meshmodel/core/v1alpha2/relationship.go +++ b/models/meshmodel/core/v1alpha2/relationship.go @@ -72,8 +72,7 @@ func (m *RelationshipDefinition) UpdateStatus(db *database.Handler, status entit } func (r *RelationshipDefinition) GetRelationshipDefinitionDB() (rdb RelationshipDefinitionDB) { - // rdb.ID = r.ID id will be assigned by the database itself don't use this, as it will be always uuid.nil, because id is not known when comp gets generated. - // While database creates an entry with valid primary key but to avoid confusion, it is disabled and accidental assignment of custom id. + rdb.ID = r.ID rdb.VersionMeta = r.VersionMeta rdb.Kind = r.Kind rdb.RelationshipType = r.RelationshipType diff --git a/models/meshmodel/core/v1beta1/category.go b/models/meshmodel/core/v1beta1/category.go index 410a08c9..e8e9fe14 100644 --- a/models/meshmodel/core/v1beta1/category.go +++ b/models/meshmodel/core/v1beta1/category.go @@ -2,6 +2,7 @@ package v1beta1 import ( "encoding/json" + "fmt" "sync" "github.com/google/uuid" @@ -28,7 +29,19 @@ type CategoryDB struct { // "Uncategorized" is assigned when Category is empty in the component definitions. const DefaultCategory = "Uncategorized" -func (cat *Category) Create(db *database.Handler) (uuid.UUID, error) { +func (cat Category) Type() entity.EntityType { + return entity.Category +} +func (cat Category) GetID() uuid.UUID { + return cat.ID +} + + +func (cat *Category) GetEntityDetail() string { + return fmt.Sprintf("name: %s", cat.Name) +} + +func (cat *Category) Create(db *database.Handler, _ uuid.UUID) (uuid.UUID, error) { if cat.Name == "" { cat.Name = DefaultCategory } diff --git a/models/meshmodel/core/v1beta1/component.go b/models/meshmodel/core/v1beta1/component.go index 7bf17636..530204a8 100644 --- a/models/meshmodel/core/v1beta1/component.go +++ b/models/meshmodel/core/v1beta1/component.go @@ -30,7 +30,8 @@ const ( CUE ComponentFormat = "CUE" ) -type component struct { +// Contains information as extracted from the core underlying component eg: Pod's apiVersion, kind and schema +type ComponentEntity struct { TypeMeta Schema string `json:"schema,omitempty" yaml:"schema"` } @@ -46,7 +47,7 @@ type ComponentDefinition struct { Model Model `json:"model"` Metadata map[string]interface{} `json:"metadata" yaml:"metadata"` // component corresponds to the specifications of underlying entity eg: Pod/Deployment.... - Component component `json:"component,omitempty" yaml:"component"` + Component ComponentEntity `json:"component,omitempty" yaml:"component"` } type ComponentDefinitionDB struct { @@ -57,7 +58,7 @@ type ComponentDefinitionDB struct { Format ComponentFormat `json:"format" yaml:"format"` ModelID uuid.UUID `json:"-" gorm:"index:idx_component_definition_dbs_model_id,column:modelID"` Metadata []byte `json:"metadata" yaml:"metadata"` - Component component `json:"component,omitempty" yaml:"component" gorm:"component"` + Component ComponentEntity `json:"component,omitempty" yaml:"component" gorm:"type:bytes;serializer:json"` } func (c ComponentDefinition) Type() entity.EntityType { @@ -101,8 +102,7 @@ func (m *ComponentDefinition) UpdateStatus(db *database.Handler, status entity.E } func (c *ComponentDefinition) GetComponentDefinitionDB() (cmd ComponentDefinitionDB) { - // cmd.ID = c.ID id will be assigned by the database itself don't use this, as it will be always uuid.nil, because id is not known when comp gets generated. - // While database creates an entry with valid primary key but to avoid confusion, it is disabled and accidental assignment of custom id. + cmd.ID = c.ID cmd.VersionMeta = c.VersionMeta cmd.DisplayName = c.DisplayName cmd.Description = c.Description diff --git a/models/meshmodel/core/v1beta1/host.go b/models/meshmodel/core/v1beta1/host.go index 1208aa38..8c03ea3c 100644 --- a/models/meshmodel/core/v1beta1/host.go +++ b/models/meshmodel/core/v1beta1/host.go @@ -16,12 +16,12 @@ var hostCreationLock sync.Mutex //Each entity will perform a check and if the ho type Host struct { ID uuid.UUID `json:"-"` - Hostname string - Port int - Metadata string + Hostname string `json:"hostname"` + Port int `json:"port"` + Metadata string `json:""metadata` CreatedAt time.Time `json:"-"` UpdatedAt time.Time `json:"-"` - IHost IHost `gorm:"-"` + IHost IHost `gorm:"-"` } type MeshModelHostsWithEntitySummary struct { diff --git a/models/meshmodel/core/v1beta1/models.go b/models/meshmodel/core/v1beta1/models.go index a2cd386f..9c31f4ed 100644 --- a/models/meshmodel/core/v1beta1/models.go +++ b/models/meshmodel/core/v1beta1/models.go @@ -13,9 +13,11 @@ import ( "gorm.io/gorm" ) +const SchemaVersion = "core.meshery.io/v1beta1" + var modelCreationLock sync.Mutex //Each component/relationship will perform a check and if the model already doesn't exist, it will create a model. This lock will make sure that there are no race conditions. -type model struct { +type ModelEntity struct { Version string `json:"version,omitempty" yaml:"version"` } @@ -31,7 +33,7 @@ type Model struct { Category Category `json:"category"` SubCategory string `json:"subCategory" gorm:"subCategory"` Metadata map[string]interface{} `json:"metadata" yaml:"modelMetadata"` - Model model `json:"model,omitempty" gorm:"model"` + Model ModelEntity `json:"model,omitempty" gorm:"model;type:bytes;serializer:json"` } type ModelDB struct { @@ -41,11 +43,11 @@ type ModelDB struct { DisplayName string `json:"modelDisplayName" gorm:"modelDisplayName"` Description string `json:"description" gorm:"description"` Status entity.EntityStatus `json:"status" gorm:"status"` - RegistrantID uuid.UUID `json:"hostID" gorm:"host_id"` + RegistrantID uuid.UUID `json:"hostID" gorm:"column:host_id"` // make as a foreign refer to host's table CategoryID uuid.UUID `json:"-" gorm:"categoryID"` SubCategory string `json:"subCategory" gorm:"subCategory"` Metadata []byte `json:"modelMetadata" gorm:"modelMetadata"` - Model model `json:"model,omitempty" gorm:"model"` + Model ModelEntity `json:"model,omitempty" gorm:"model;type:bytes;serializer:json"` } func (m Model) Type() entity.EntityType { @@ -77,7 +79,7 @@ func (m *Model) Create(db *database.Handler, hostID uuid.UUID) (uuid.UUID, error return uuid.UUID{}, err } if err == gorm.ErrRecordNotFound { //The model is already not present and needs to be inserted - id, err := m.Category.Create(db) + id, err := m.Category.Create(db, hostID) if err != nil { return uuid.UUID{}, err } @@ -109,8 +111,7 @@ func (m *Model) UpdateStatus(db *database.Handler, status entity.EntityStatus) e } func (c *Model) GetModelDB() (cmd ModelDB) { - // cmd.ID = c.ID id will be assigned by the database itself don't use this, as it will be always uuid.nil, because id is not known when comp gets generated. - // While database creates an entry with valid primary key but to avoid confusion, it is disabled and accidental assignment of custom id. + cmd.ID = c.ID cmd.VersionMeta = c.VersionMeta cmd.Name = c.Name cmd.DisplayName = c.DisplayName diff --git a/models/meshmodel/entity/types.go b/models/meshmodel/entity/types.go index e409b616..d41f9638 100644 --- a/models/meshmodel/entity/types.go +++ b/models/meshmodel/entity/types.go @@ -12,6 +12,7 @@ const ( PolicyDefinition EntityType = "policy" RelationshipDefinition EntityType = "relationship" Model EntityType = "model" + Category EntityType = "category" ) // Each entity will have it's own Filter implementation via which it exposes the nobs and dials to fetch entities diff --git a/models/meshmodel/registry/registry.go b/models/meshmodel/registry/registry.go index f55ea199..206d6a15 100644 --- a/models/meshmodel/registry/registry.go +++ b/models/meshmodel/registry/registry.go @@ -121,6 +121,15 @@ func (rm *RegistryManager) UpdateEntityStatus(ID string, status string, entityTy } } +func (rm *RegistryManager) GetRegistrant(e entity.Entity) v1beta1.Host { + eID := e.GetID() + var reg Registry + _ = rm.db.Where("entity = ?", eID).Find(®).Error + var h v1beta1.Host + _ = rm.db.Where("id = ?", reg.RegistrantID).Find(&h).Error + return h +} + // to be removed func (rm *RegistryManager) GetRegistrants(f *v1beta1.HostFilter) ([]v1beta1.MeshModelHostsWithEntitySummary, int64, error) { var result []v1beta1.MesheryHostSummaryDB diff --git a/models/meshmodel/registry/v1beta1/relationship_filter.go b/models/meshmodel/registry/v1alpha2/relationship_filter.go similarity index 98% rename from models/meshmodel/registry/v1beta1/relationship_filter.go rename to models/meshmodel/registry/v1alpha2/relationship_filter.go index 05d96f42..33946ee7 100644 --- a/models/meshmodel/registry/v1beta1/relationship_filter.go +++ b/models/meshmodel/registry/v1alpha2/relationship_filter.go @@ -1,4 +1,4 @@ -package v1beta1 +package v1alpha2 import ( "fmt" @@ -12,10 +12,10 @@ import ( type relationshipDefinitionWithModel struct { v1alpha2.RelationshipDefinitionDB - ModelDB v1beta1.ModelDB + v1beta1.ModelDB // acoount for overridn fields // v1beta1.ModelDB.Version `json:"modelVersion"` - CategoryDB v1beta1.CategoryDB + v1beta1.CategoryDB } // For now, only filtering by Kind and SubType are allowed. diff --git a/models/meshmodel/registry/v1beta1/category_filter.go b/models/meshmodel/registry/v1beta1/category_filter.go index 2126decf..545ad4cf 100644 --- a/models/meshmodel/registry/v1beta1/category_filter.go +++ b/models/meshmodel/registry/v1beta1/category_filter.go @@ -3,6 +3,7 @@ package v1beta1 import ( "github.com/layer5io/meshkit/database" "github.com/layer5io/meshkit/models/meshmodel/core/v1beta1" + "github.com/layer5io/meshkit/models/meshmodel/entity" "gorm.io/gorm/clause" ) @@ -23,9 +24,9 @@ func (cf *CategoryFilter) Create(m map[string]interface{}) { cf.Name = m["name"].(string) } -func (cf *CategoryFilter) Get(db *database.Handler) ([]v1beta1.Category, int64, int, error) { +func (cf *CategoryFilter) Get(db *database.Handler) ([]entity.Entity, int64, int, error) { var catdb []v1beta1.CategoryDB - var cat []v1beta1.Category + var cat []entity.Entity finder := db.Model(&catdb) // total count before pagination @@ -61,7 +62,8 @@ func (cf *CategoryFilter) Get(db *database.Handler) ([]v1beta1.Category, int64, _ = finder.Find(&catdb).Error for _, c := range catdb { - cat = append(cat, c.GetCategory(db)) + category := c.GetCategory(db) + cat = append(cat, &category) } // duplicate category ? return cat, count, int(count), nil diff --git a/utils/component/generator.go b/utils/component/generator.go index ccf1df32..9c15a7b2 100644 --- a/utils/component/generator.go +++ b/utils/component/generator.go @@ -48,6 +48,8 @@ var Configs = []CuePathConfig{DefaultPathConfig, DefaultPathConfig2} func Generate(crd string) (v1beta1.ComponentDefinition, error) { component := v1beta1.ComponentDefinition{} + component.SchemaVersion = v1beta1.SchemaVersion + component.Metadata = make(map[string]interface{}) crdCue, err := utils.YamlToCue(crd) if err != nil { From 95dbd29fad6bed2a3ff4a1110a0a4fa965579dd5 Mon Sep 17 00:00:00 2001 From: MUzairS15 Date: Sun, 31 Mar 2024 16:58:20 +0530 Subject: [PATCH 22/32] register models in registres table, fix tags and sql Signed-off-by: MUzairS15 --- models/meshmodel/core/v1beta1/host.go | 6 +++--- models/meshmodel/core/v1beta1/models.go | 7 ++++--- .../registry/v1alpha2/relationship_filter.go | 6 +++--- .../registry/v1beta1/component_filter.go | 16 ++++++++-------- .../meshmodel/registry/v1beta1/model_filter.go | 15 ++++++++------- .../meshmodel/registry/v1beta1/policy_filter.go | 6 +++--- 6 files changed, 29 insertions(+), 27 deletions(-) diff --git a/models/meshmodel/core/v1beta1/host.go b/models/meshmodel/core/v1beta1/host.go index 8c03ea3c..f7baf54e 100644 --- a/models/meshmodel/core/v1beta1/host.go +++ b/models/meshmodel/core/v1beta1/host.go @@ -17,11 +17,11 @@ var hostCreationLock sync.Mutex //Each entity will perform a check and if the ho type Host struct { ID uuid.UUID `json:"-"` Hostname string `json:"hostname"` - Port int `json:"port"` - Metadata string `json:""metadata` + Port int `json:"port,omitempty"` + Metadata string `json:"metadata,omitempty"` CreatedAt time.Time `json:"-"` UpdatedAt time.Time `json:"-"` - IHost IHost `gorm:"-"` + IHost IHost `json:"-" gorm:"-"` } type MeshModelHostsWithEntitySummary struct { diff --git a/models/meshmodel/core/v1beta1/models.go b/models/meshmodel/core/v1beta1/models.go index 9c31f4ed..822232be 100644 --- a/models/meshmodel/core/v1beta1/models.go +++ b/models/meshmodel/core/v1beta1/models.go @@ -40,7 +40,7 @@ type ModelDB struct { ID uuid.UUID `json:"id"` VersionMeta Name string `json:"modelName" gorm:"modelName"` - DisplayName string `json:"modelDisplayName" gorm:"modelDisplayName"` + DisplayName string `json:"modelDisplayName"` Description string `json:"description" gorm:"description"` Status entity.EntityStatus `json:"status" gorm:"status"` RegistrantID uuid.UUID `json:"hostID" gorm:"column:host_id"` // make as a foreign refer to host's table @@ -159,13 +159,14 @@ func (c Model) WriteModelDefinition(modelDefPath string) error { // Registers models into registries table. func registerModel(db *database.Handler, regID, modelID uuid.UUID) error { - err := db.Select("registries.* from registries").Where("registrant_id = ?", regID).Where("type = ?", "model").Where("entity = ?", modelID).Error + var count int64 + err := db.Table("registries").Where("registrant_id = ?", regID).Where("type = ?", "model").Where("entity = ?", modelID).Count(&count).Error if err != nil && err != gorm.ErrRecordNotFound { return err } - if err == gorm.ErrRecordNotFound { + if count == 0 { err = db.Exec("INSERT INTO registries (registrant_id, entity, type) VALUES (?,?,?)", regID, modelID, "model").Error if err != nil { return err diff --git a/models/meshmodel/registry/v1alpha2/relationship_filter.go b/models/meshmodel/registry/v1alpha2/relationship_filter.go index 33946ee7..a752c54f 100644 --- a/models/meshmodel/registry/v1alpha2/relationship_filter.go +++ b/models/meshmodel/registry/v1alpha2/relationship_filter.go @@ -11,11 +11,11 @@ import ( ) type relationshipDefinitionWithModel struct { - v1alpha2.RelationshipDefinitionDB - v1beta1.ModelDB + RelationshipDefinitionDB v1alpha2.RelationshipDefinitionDB `gorm:"embedded"` + ModelDB v1beta1.ModelDB `gorm:"embedded"` // acoount for overridn fields // v1beta1.ModelDB.Version `json:"modelVersion"` - v1beta1.CategoryDB + CategoryDB v1beta1.CategoryDB `gorm:"embedded"` } // For now, only filtering by Kind and SubType are allowed. diff --git a/models/meshmodel/registry/v1beta1/component_filter.go b/models/meshmodel/registry/v1beta1/component_filter.go index 0c097921..d030594b 100644 --- a/models/meshmodel/registry/v1beta1/component_filter.go +++ b/models/meshmodel/registry/v1beta1/component_filter.go @@ -25,10 +25,10 @@ type ComponentFilter struct { } type componentDefinitionWithModel struct { - v1beta1.ComponentDefinitionDB - ModelDB v1beta1.ModelDB // acoount for overridn fields - CategoryDB v1beta1.CategoryDB - HostsDB v1beta1.Host + ComponentDefinitionDB v1beta1.ComponentDefinitionDB `gorm:"embedded"` + ModelDB v1beta1.ModelDB `gorm:"embedded"` + CategoryDB v1beta1.CategoryDB `gorm:"embedded"` + HostsDB v1beta1.Host `gorm:"embedded"` } // Create the filter from map[string]interface{} @@ -43,7 +43,7 @@ func (componentFilter *ComponentFilter) Get(db *database.Handler) ([]entity.Enti countUniqueComponents := func(components []componentDefinitionWithModel) int { set := make(map[string]struct{}) for _, compWithModel := range components { - key := compWithModel.Component.Kind + "@" + compWithModel.Component.Version + "@" + compWithModel.ModelDB.Name + "@" + compWithModel.ModelDB.Version + key := compWithModel.ComponentDefinitionDB.Component.Kind + "@" + compWithModel.ComponentDefinitionDB.Version + "@" + compWithModel.ModelDB.Name + "@" + compWithModel.ModelDB.Version if _, ok := set[key]; !ok { set[key] = struct{}{} } @@ -52,10 +52,10 @@ func (componentFilter *ComponentFilter) Get(db *database.Handler) ([]entity.Enti } var componentDefinitionsWithModel []componentDefinitionWithModel finder := db.Model(&v1beta1.ComponentDefinitionDB{}). - Select("component_definition_dbs.*, model_dbs.*,category_dbs.*"). + Select("component_definition_dbs.*, model_dbs.*,category_dbs.*, hosts.*"). Joins("JOIN model_dbs ON component_definition_dbs.model_id = model_dbs.id"). Joins("JOIN category_dbs ON model_dbs.category_id = category_dbs.id"). - Joins("JOIN hosts ON models.registrant_id = hosts.id") + Joins("JOIN hosts ON hosts.id = model_dbs.host_id") // if componentFilter.Greedy { @@ -111,7 +111,7 @@ func (componentFilter *ComponentFilter) Get(db *database.Handler) ([]entity.Enti // remove this when compoentdef and componetdefdb struct is consolidated. for _, cm := range componentDefinitionsWithModel { if componentFilter.Trim { - cm.Component.Schema = "" + cm.ComponentDefinitionDB.Component.Schema = "" } // Ensure correct reg is passed, rn it is dummy for sake of testing. // In the first query above where we do seelection i think there changes will be requrired, an when that two def and defDB structs are consolidated, using association and preload i think we can do. diff --git a/models/meshmodel/registry/v1beta1/model_filter.go b/models/meshmodel/registry/v1beta1/model_filter.go index 71f76c03..cefe369d 100644 --- a/models/meshmodel/registry/v1beta1/model_filter.go +++ b/models/meshmodel/registry/v1beta1/model_filter.go @@ -36,14 +36,15 @@ func (mf *ModelFilter) Create(m map[string]interface{}) { mf.Name = m["name"].(string) } +// I have removed Regsitry struct here check if filter by Registrant works. +type modelWithCategories struct { + ModelDB v1beta1.ModelDB `gorm:"embedded"` + CategoryDB v1beta1.CategoryDB `gorm:"embedded"` + Host v1beta1.Host `gorm:"embedded"` + // registry.Registry +} + func (mf *ModelFilter) Get(db *database.Handler) ([]entity.Entity, int64, int, error) { - // I have removed Regsitry struct here check if filter by Registrant works. - type modelWithCategories struct { - v1beta1.ModelDB - v1beta1.CategoryDB - // registry.Registry - v1beta1.Host - } countUniqueModels := func(models []modelWithCategories) int { set := make(map[string]struct{}) diff --git a/models/meshmodel/registry/v1beta1/policy_filter.go b/models/meshmodel/registry/v1beta1/policy_filter.go index 4affab35..6bba2f1c 100644 --- a/models/meshmodel/registry/v1beta1/policy_filter.go +++ b/models/meshmodel/registry/v1beta1/policy_filter.go @@ -24,8 +24,8 @@ func (pf *PolicyFilter) Create(m map[string]interface{}) { } type policyDefinitionWithModel struct { - v1beta1.PolicyDefinitionDB - Model v1beta1.Model + PolicyDB v1beta1.PolicyDefinitionDB `gorm:"embedded"` + Model v1beta1.Model `gorm:"embedded"` } func (pf *PolicyFilter) Get(db *database.Handler) ([]entity.Entity, int64, int, error) { @@ -52,7 +52,7 @@ func (pf *PolicyFilter) Get(db *database.Handler) ([]entity.Entity, int64, int, return pl, 0, 0, err } for _, cm := range componentDefinitionsWithModel { - policyDef := cm.PolicyDefinitionDB.GetPolicyDefinition(cm.Model) + policyDef := cm.PolicyDB.GetPolicyDefinition(cm.Model) pl = append(pl, &policyDef) } return pl, count, int(count), nil From 2302b3fbc95cfa8f65f08fad0e328f705bd82679 Mon Sep 17 00:00:00 2001 From: MUzairS15 Date: Wed, 3 Apr 2024 14:53:25 +0530 Subject: [PATCH 23/32] update model unique hash generation Signed-off-by: MUzairS15 --- .../core/policies/rego_policy_relationship.go | 7 ++++++- models/meshmodel/core/v1beta1/models.go | 11 +++++++++-- 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/models/meshmodel/core/policies/rego_policy_relationship.go b/models/meshmodel/core/policies/rego_policy_relationship.go index a77b6862..73434618 100644 --- a/models/meshmodel/core/policies/rego_policy_relationship.go +++ b/models/meshmodel/core/policies/rego_policy_relationship.go @@ -3,6 +3,8 @@ package policies import ( "context" "fmt" + "io/fs" + "path/filepath" "github.com/layer5io/meshkit/models/meshmodel/registry" "github.com/layer5io/meshkit/models/meshmodel/registry/v1alpha2" @@ -54,7 +56,10 @@ func (r *Rego) RegoPolicyHandler(regoQueryString string, designFile []byte) (int } regoEngine, err := rego.New( rego.Query(regoQueryString), - rego.Load([]string{r.policyDir}, nil), + rego.Load([]string{r.policyDir}, func(abspath string, info fs.FileInfo, depth int) bool { + extension := filepath.Ext(abspath) + return extension == "rego" + }), rego.Store(r.store), rego.Transaction(r.transaction), ).PrepareForEval(r.ctx) diff --git a/models/meshmodel/core/v1beta1/models.go b/models/meshmodel/core/v1beta1/models.go index 822232be..362156f1 100644 --- a/models/meshmodel/core/v1beta1/models.go +++ b/models/meshmodel/core/v1beta1/models.go @@ -62,8 +62,15 @@ func (m *Model) GetEntityDetail() string { } func (m *Model) Create(db *database.Handler, hostID uuid.UUID) (uuid.UUID, error) { - - byt, err := json.Marshal(m) + modelIdentifier := Model{ + Registrant: m.Registrant, + VersionMeta: m.VersionMeta, + Name: m.Name, + Model: ModelEntity{ + Version: m.Model.Version, + }, + } + byt, err := json.Marshal(modelIdentifier) if err != nil { return uuid.UUID{}, err } From 7a8ff4e1fc4da96032305961b4d7587a6d5345f7 Mon Sep 17 00:00:00 2001 From: MUzairS15 Date: Sun, 7 Apr 2024 15:49:34 +0530 Subject: [PATCH 24/32] finalize changes Signed-off-by: MUzairS15 --- .../meshmodel/core/v1alpha2/relationship.go | 67 +++----------- models/meshmodel/core/v1beta1/category.go | 32 ++----- models/meshmodel/core/v1beta1/component.go | 59 +++---------- models/meshmodel/core/v1beta1/models.go | 87 +++++-------------- models/meshmodel/core/v1beta1/policy.go | 45 ++-------- models/meshmodel/registry/registry.go | 21 +++-- .../registry/v1alpha2/relationship_filter.go | 30 ++----- .../registry/v1beta1/category_filter.go | 15 ++-- .../registry/v1beta1/component_filter.go | 42 ++++----- .../registry/v1beta1/model_filter.go | 53 +++++------ .../registry/v1beta1/policy_filter.go | 18 ++-- 11 files changed, 140 insertions(+), 329 deletions(-) diff --git a/models/meshmodel/core/v1alpha2/relationship.go b/models/meshmodel/core/v1alpha2/relationship.go index 2e6068d4..b46d8025 100644 --- a/models/meshmodel/core/v1alpha2/relationship.go +++ b/models/meshmodel/core/v1alpha2/relationship.go @@ -1,7 +1,6 @@ package v1alpha2 import ( - "encoding/json" "fmt" "path/filepath" @@ -10,35 +9,26 @@ import ( "github.com/layer5io/meshkit/models/meshmodel/core/v1beta1" "github.com/layer5io/meshkit/models/meshmodel/entity" "github.com/layer5io/meshkit/utils" + "gorm.io/gorm/clause" ) -// Remove additional DB structs from all entites they are used just for marshaling & unmarshlling and tarcking modelID vs model(modelID when referenced in DB and full model when sending data, use foregin key references tag of gorm for that as used in MeshSync tables) when saving retrivng from database, insted use gorm' serailization tag as used in events. type RelationshipDefinition struct { ID uuid.UUID `json:"id"` v1beta1.VersionMeta Kind string `json:"kind,omitempty" yaml:"kind"` // The property has been named RelationshipType instead of Type to avoid collision from Type() function, which enables support for dynamic type. // Though, the column name and the json representation is "type". - RelationshipType string `json:"type" yaml:"type" gorm:"type"` - SubType string `json:"subType" yaml:"subType" gorm:"subType"` - EvaluationQuery string `json:"evaluationQuery" yaml:"evaluationQuery" gorm:"evaluationQuery"` - Metadata map[string]interface{} `json:"metadata" yaml:"metadata"` - Model v1beta1.Model `json:"model"` - Selectors []map[string]interface{} `json:"selectors" yaml:"selectors"` + RelationshipType string `json:"type" yaml:"type" gorm:"type"` + SubType string `json:"subType" yaml:"subType"` + EvaluationQuery string `json:"evaluationQuery" yaml:"evaluationQuery" gorm:"evaluationQuery"` + Metadata []byte `json:"metadata" yaml:"metadata"` + ModelID uuid.UUID `json:"-" gorm:"index:idx_relationship_definition_dbs_model_id,column:model_id"` + Model v1beta1.Model `json:"model" gorm:"foreignKey:ModelID;references:ID"` + Selectors []byte `json:"selectors" yaml:"selectors"` } -type RelationshipDefinitionDB struct { - ID uuid.UUID `json:"id"` - v1beta1.VersionMeta - Kind string `json:"kind,omitempty" yaml:"kind"` - // The property has been named RelationshipType instead of Type to avoid collision from Type() function, which enables support for dynamic type. - // Though, the column name and the json representation is "type". - RelationshipType string `json:"type" yaml:"type" gorm:"type"` - SubType string `json:"subType" yaml:"subType"` - EvaluationQuery string `json:"evaluationQuery" yaml:"evaluationQuery" gorm:"evaluationQuery"` - Metadata []byte `json:"metadata" yaml:"metadata"` - ModelID uuid.UUID `json:"-" gorm:"index:idx_relationship_definition_dbs_model_id,column:modelID"` - Selectors []byte `json:"selectors" yaml:"selectors"` +func (r RelationshipDefinition) TableName() string { + return "relationship_definition_dbs" } func (r RelationshipDefinition) Type() entity.EntityType { @@ -58,9 +48,8 @@ func (r *RelationshipDefinition) Create(db *database.Handler, hostID uuid.UUID) if err != nil { return uuid.UUID{}, err } - rdb := r.GetRelationshipDefinitionDB() - rdb.ModelID = mid - err = db.Create(&rdb).Error + r.ModelID = mid + err = db.Omit(clause.Associations).Create(&r).Error if err != nil { return uuid.UUID{}, err } @@ -71,40 +60,8 @@ func (m *RelationshipDefinition) UpdateStatus(db *database.Handler, status entit return nil } -func (r *RelationshipDefinition) GetRelationshipDefinitionDB() (rdb RelationshipDefinitionDB) { - rdb.ID = r.ID - rdb.VersionMeta = r.VersionMeta - rdb.Kind = r.Kind - rdb.RelationshipType = r.RelationshipType - rdb.SubType = r.SubType - rdb.EvaluationQuery = r.EvaluationQuery - rdb.Metadata, _ = json.Marshal(r.Metadata) - rdb.ModelID = r.Model.ID - rdb.Selectors, _ = json.Marshal(r.Selectors) - return -} - func (c RelationshipDefinition) WriteComponentDefinition(relDirPath string) error { relPath := filepath.Join(relDirPath, c.Kind, string(c.Type())+".json") err := utils.WriteJSONToFile[RelationshipDefinition](relPath, c) return err } - -func (rdb *RelationshipDefinitionDB) GetRelationshipDefinition(m v1beta1.Model) (r RelationshipDefinition) { - r.ID = rdb.ID - r.VersionMeta = rdb.VersionMeta - r.Kind = rdb.Kind - r.RelationshipType = rdb.RelationshipType - r.SubType = rdb.SubType - r.EvaluationQuery = rdb.EvaluationQuery - if r.Metadata == nil { - r.Metadata = make(map[string]interface{}) - } - _ = json.Unmarshal(rdb.Metadata, &r.Metadata) - r.Model = m - if r.Selectors == nil { - r.Selectors = []map[string]interface{}{} - } - _ = json.Unmarshal(rdb.Selectors, &r.Selectors) - return -} diff --git a/models/meshmodel/core/v1beta1/category.go b/models/meshmodel/core/v1beta1/category.go index e8e9fe14..0b63e4bb 100644 --- a/models/meshmodel/core/v1beta1/category.go +++ b/models/meshmodel/core/v1beta1/category.go @@ -15,15 +15,13 @@ var categoryCreationLock sync.Mutex //Each model will perform a check and if the // swagger:response Category type Category struct { - ID uuid.UUID `json:"-" yaml:"-"` - Name string `json:"name"` - Metadata map[string]interface{} `json:"metadata" yaml:"metadata"` + ID uuid.UUID `json:"-"` + Name string `json:"name" gorm:"name"` + Metadata map[string]interface{} `json:"metadata" yaml:"metadata" gorm:"type:bytes;serializer:json"` } -type CategoryDB struct { - ID uuid.UUID `json:"-"` - Name string `json:"categoryName" gorm:"categoryName"` - Metadata []byte `json:"categoryMetadata" gorm:"categoryMetadata"` +func (c Category) TableName() string { + return "category_dbs" } // "Uncategorized" is assigned when Category is empty in the component definitions. @@ -50,7 +48,7 @@ func (cat *Category) Create(db *database.Handler, _ uuid.UUID) (uuid.UUID, error return uuid.UUID{}, err } catID := uuid.NewSHA1(uuid.UUID{}, byt) - var category CategoryDB + var category Category categoryCreationLock.Lock() defer categoryCreationLock.Unlock() err = db.First(&category, "id = ?", catID).Error @@ -59,12 +57,11 @@ func (cat *Category) Create(db *database.Handler, _ uuid.UUID) (uuid.UUID, error } if err == gorm.ErrRecordNotFound { //The category is already not present and needs to be inserted cat.ID = catID - catdb := cat.GetCategoryDB(db) - err = db.Create(&catdb).Error + err = db.Create(&cat).Error if err != nil { return uuid.UUID{}, err } - return catdb.ID, nil + return cat.ID, nil } return category.ID, nil } @@ -73,16 +70,3 @@ func (m *Category) UpdateStatus(db database.Handler, status entity.EntityStatus) return nil } -func (c *Category) GetCategoryDB(db *database.Handler) (catdb CategoryDB) { - catdb.ID = c.ID - catdb.Name = c.Name - catdb.Metadata, _ = json.Marshal(c.Metadata) - return -} - -func (cdb *CategoryDB) GetCategory(db *database.Handler) (cat Category) { - cat.ID = cdb.ID - cat.Name = cdb.Name - _ = json.Unmarshal(cdb.Metadata, &cat.Metadata) - return -} diff --git a/models/meshmodel/core/v1beta1/component.go b/models/meshmodel/core/v1beta1/component.go index 530204a8..aefafb60 100644 --- a/models/meshmodel/core/v1beta1/component.go +++ b/models/meshmodel/core/v1beta1/component.go @@ -1,13 +1,13 @@ package v1beta1 import ( - "encoding/json" "fmt" "path/filepath" "github.com/layer5io/meshkit/database" "github.com/layer5io/meshkit/models/meshmodel/entity" "github.com/layer5io/meshkit/utils" + "gorm.io/gorm/clause" "github.com/google/uuid" ) @@ -37,30 +37,23 @@ type ComponentEntity struct { } // swagger:response ComponentDefinition -// use NewComponent function for instantiating type ComponentDefinition struct { - ID uuid.UUID `json:"id,omitempty"` + ID uuid.UUID `json:"id"` VersionMeta DisplayName string `json:"displayName" gorm:"displayName"` Description string `json:"description" gorm:"description"` Format ComponentFormat `json:"format" yaml:"format"` - Model Model `json:"model"` - Metadata map[string]interface{} `json:"metadata" yaml:"metadata"` - // component corresponds to the specifications of underlying entity eg: Pod/Deployment.... - Component ComponentEntity `json:"component,omitempty" yaml:"component"` + ModelID uuid.UUID `json:"-" gorm:"index:idx_component_definition_dbs_model_id,column:model_id"` + Model Model `json:"model" gorm:"foreignKey:ModelID;references:ID"` + Metadata map[string]interface{} `json:"metadata" yaml:"metadata" gorm:"type:bytes;serializer:json"` + Component ComponentEntity `json:"component,omitempty" yaml:"component" gorm:"type:bytes;serializer:json"` } -type ComponentDefinitionDB struct { - ID uuid.UUID `json:"id"` - VersionMeta - DisplayName string `json:"displayName" gorm:"displayName"` - Description string `json:"description" gorm:"description"` - Format ComponentFormat `json:"format" yaml:"format"` - ModelID uuid.UUID `json:"-" gorm:"index:idx_component_definition_dbs_model_id,column:modelID"` - Metadata []byte `json:"metadata" yaml:"metadata"` - Component ComponentEntity `json:"component,omitempty" yaml:"component" gorm:"type:bytes;serializer:json"` +func (c ComponentDefinition) TableName() string { + return "component_definition_dbs" } + func (c ComponentDefinition) Type() entity.EntityType { return entity.ComponentDefinition } @@ -91,9 +84,10 @@ func (c *ComponentDefinition) Create(db *database.Handler, hostID uuid.UUID) (uu if !utils.IsSchemaEmpty(c.Component.Schema) { c.Metadata["hasInvalidSchema"] = true } - cdb := c.GetComponentDefinitionDB() - cdb.ModelID = mid - err = db.Create(&cdb).Error + + c.ModelID = mid + err = db.Omit(clause.Associations).Create(&c).Error + fmt.Println("ERROR REGISTER COMPONENT", err) return c.ID, err } @@ -101,35 +95,8 @@ func (m *ComponentDefinition) UpdateStatus(db *database.Handler, status entity.E return nil } -func (c *ComponentDefinition) GetComponentDefinitionDB() (cmd ComponentDefinitionDB) { - cmd.ID = c.ID - cmd.VersionMeta = c.VersionMeta - cmd.DisplayName = c.DisplayName - cmd.Description = c.Description - cmd.Format = c.Format - cmd.ModelID = c.Model.ID - cmd.Metadata, _ = json.Marshal(c.Metadata) - cmd.Component = c.Component - return -} - func (c ComponentDefinition) WriteComponentDefinition(componentDirPath string) error { componentPath := filepath.Join(componentDirPath, c.Component.Kind+".json") err := utils.WriteJSONToFile[ComponentDefinition](componentPath, c) return err } - -func (cmd *ComponentDefinitionDB) GetComponentDefinition(model Model) (c ComponentDefinition) { - c.ID = cmd.ID - c.VersionMeta = cmd.VersionMeta - c.DisplayName = cmd.DisplayName - c.Description = cmd.Description - c.Format = cmd.Format - c.Model = model - if c.Metadata == nil { - c.Metadata = make(map[string]interface{}) - } - _ = json.Unmarshal(cmd.Metadata, &c.Metadata) - c.Component = cmd.Component - return -} diff --git a/models/meshmodel/core/v1beta1/models.go b/models/meshmodel/core/v1beta1/models.go index 362156f1..82e6947e 100644 --- a/models/meshmodel/core/v1beta1/models.go +++ b/models/meshmodel/core/v1beta1/models.go @@ -11,6 +11,7 @@ import ( "github.com/layer5io/meshkit/models/meshmodel/entity" "github.com/layer5io/meshkit/utils" "gorm.io/gorm" + "gorm.io/gorm/clause" ) const SchemaVersion = "core.meshery.io/v1beta1" @@ -23,31 +24,23 @@ type ModelEntity struct { // swagger:response Model type Model struct { - ID uuid.UUID `json:"id,omitempty" yaml:"-"` + ID uuid.UUID `json:"id"` VersionMeta - Name string `json:"name"` - DisplayName string `json:"displayName" gorm:"modelDisplayName"` - Description string `json:"description" gorm:"description"` - Status entity.EntityStatus `json:"status" gorm:"status"` - Registrant Host `json:"registrant" gorm:"registrant"` // to be Connection - Category Category `json:"category"` - SubCategory string `json:"subCategory" gorm:"subCategory"` - Metadata map[string]interface{} `json:"metadata" yaml:"modelMetadata"` - Model ModelEntity `json:"model,omitempty" gorm:"model;type:bytes;serializer:json"` + Name string `json:"name" gorm:"modelName"` + DisplayName string `json:"displayName"` + Description string `json:"description" gorm:"description"` + Status entity.EntityStatus `json:"status" gorm:"status"` + RegistrantID uuid.UUID `json:"hostID" gorm:"column:host_id"` // make as a foreign refer to host's table + Registrant Host `json:"registrant" gorm:"foreignKey:RegistrantID;references:ID"` + CategoryID uuid.UUID `json:"-" gorm:"categoryID"` + Category Category `json:"category" gorm:"foreignKey:CategoryID;references:ID"` + SubCategory string `json:"subCategory" gorm:"subCategory"` + Metadata map[string]interface{} `json:"metadata" gorm:"type:bytes;serializer:json"` + Model ModelEntity `json:"model,omitempty" gorm:"model;type:bytes;serializer:json"` } -type ModelDB struct { - ID uuid.UUID `json:"id"` - VersionMeta - Name string `json:"modelName" gorm:"modelName"` - DisplayName string `json:"modelDisplayName"` - Description string `json:"description" gorm:"description"` - Status entity.EntityStatus `json:"status" gorm:"status"` - RegistrantID uuid.UUID `json:"hostID" gorm:"column:host_id"` // make as a foreign refer to host's table - CategoryID uuid.UUID `json:"-" gorm:"categoryID"` - SubCategory string `json:"subCategory" gorm:"subCategory"` - Metadata []byte `json:"modelMetadata" gorm:"modelMetadata"` - Model ModelEntity `json:"model,omitempty" gorm:"model;type:bytes;serializer:json"` +func (m Model) TableName() string { + return "model_dbs" } func (m Model) Type() entity.EntityType { @@ -75,7 +68,8 @@ func (m *Model) Create(db *database.Handler, hostID uuid.UUID) (uuid.UUID, error return uuid.UUID{}, err } modelID := uuid.NewSHA1(uuid.UUID{}, byt) - var model ModelDB + var model Model + fmt.Println("TEST MODEL NAME ", m.Name, m) if m.Name == "" { return uuid.UUID{}, fmt.Errorf("empty or invalid model name passed") } @@ -91,12 +85,12 @@ func (m *Model) Create(db *database.Handler, hostID uuid.UUID) (uuid.UUID, error return uuid.UUID{}, err } m.ID = modelID - mdb := m.GetModelDB() - mdb.CategoryID = id - mdb.RegistrantID = hostID - mdb.Status = entity.Enabled - err = db.Create(&mdb).Error + m.CategoryID = id + m.RegistrantID = hostID + m.Status = entity.Enabled + err = db.Omit(clause.Associations).Create(&m).Error if err != nil { + fmt.Println("MODEL REGISTER >>>>>", err) return uuid.UUID{}, err } // register model inside registries table @@ -104,52 +98,19 @@ func (m *Model) Create(db *database.Handler, hostID uuid.UUID) (uuid.UUID, error if err != nil { return uuid.UUID{}, err } - return mdb.ID, nil + return m.ID, nil } return model.ID, nil } func (m *Model) UpdateStatus(db *database.Handler, status entity.EntityStatus) error { - err := db.Model(&ModelDB{}).Where("id = ?", m.ID).Update("status", status).Error + err := db.Model(&Model{}).Where("id = ?", m.ID).Update("status", status).Error if err != nil { return entity.ErrUpdateEntityStatus(err, string(m.Type()), status) } return nil } -func (c *Model) GetModelDB() (cmd ModelDB) { - cmd.ID = c.ID - cmd.VersionMeta = c.VersionMeta - cmd.Name = c.Name - cmd.DisplayName = c.DisplayName - cmd.Description = c.Description - cmd.Status = c.Status - cmd.RegistrantID = c.Registrant.ID - cmd.CategoryID = c.Category.ID - cmd.SubCategory = c.SubCategory - cmd.Metadata, _ = json.Marshal(c.Metadata) - cmd.Model = c.Model - return -} - -// is reg should be passed as param? -func (cmd *ModelDB) GetModel(cat Category, reg Host) (c Model) { - c.ID = cmd.ID - c.VersionMeta = cmd.VersionMeta - c.Name = cmd.Name - c.DisplayName = cmd.DisplayName - c.Description = cmd.Description - c.Status = cmd.Status - c.Registrant = reg - c.Category = cat - c.SubCategory = cmd.SubCategory - _ = json.Unmarshal(cmd.Metadata, &c.Metadata) - c.Model = cmd.Model - // c.Components = make([]ComponentDefinitionDB, 0) - // c.Relationships = make([]RelationshipDefinitionDB, 0) - return -} - func (c Model) WriteModelDefinition(modelDefPath string) error { err := utils.CreateDirectory(modelDefPath) if err != nil { diff --git a/models/meshmodel/core/v1beta1/policy.go b/models/meshmodel/core/v1beta1/policy.go index 547c79a2..d0e53bf1 100644 --- a/models/meshmodel/core/v1beta1/policy.go +++ b/models/meshmodel/core/v1beta1/policy.go @@ -1,7 +1,6 @@ package v1beta1 import ( - "encoding/json" "fmt" "path/filepath" "time" @@ -9,30 +8,23 @@ import ( "github.com/google/uuid" "github.com/layer5io/meshkit/database" "github.com/layer5io/meshkit/utils" + "gorm.io/gorm/clause" "github.com/layer5io/meshkit/models/meshmodel/entity" ) +// swagger:response PolicyDefinition type PolicyDefinition struct { ID uuid.UUID `json:"-"` TypeMeta + ModelID uuid.UUID `json:"-" gorm:"column:modelID"` Model Model `json:"model"` SubType string `json:"subType" yaml:"subType"` - Expression map[string]interface{} `json:"expression" yaml:"expression"` + Expression map[string]interface{} `json:"expression" yaml:"expression" gorm:"type:bytes;serializer:json"` CreatedAt time.Time `json:"-"` UpdatedAt time.Time `json:"-"` } -type PolicyDefinitionDB struct { - ID uuid.UUID `json:"-"` - ModelID uuid.UUID `json:"-" gorm:"modelID"` - TypeMeta - SubType string `json:"subType" yaml:"subType"` - Expression []byte `json:"expression" yaml:"expression"` - CreatedAt time.Time `json:"-"` - UpdatedAt time.Time `json:"-"` -} - func (p PolicyDefinition) GetID() uuid.UUID { return p.ID } @@ -51,41 +43,18 @@ func (p *PolicyDefinition) Create(db *database.Handler, hostID uuid.UUID) (uuid. if err != nil { return uuid.UUID{}, err } - pdb := p.GetPolicyDefinitionDB() - pdb.ModelID = mid - err = db.Create(&pdb).Error + p.ModelID = mid + err = db.Omit(clause.Associations).Create(&p).Error if err != nil { return uuid.UUID{}, err } - return pdb.ID, nil + return p.ID, nil } func (m *PolicyDefinition) UpdateStatus(db *database.Handler, status entity.EntityStatus) error { return nil } -func (p *PolicyDefinition) GetPolicyDefinitionDB() (pdb PolicyDefinitionDB) { - pdb.ID = p.ID - pdb.TypeMeta = p.TypeMeta - pdb.SubType = p.SubType - pdb.ModelID = p.Model.ID - pdb.Expression, _ = json.Marshal(p.Expression) - return -} - -func (pdb *PolicyDefinitionDB) GetPolicyDefinition(m Model) (p PolicyDefinition) { - p.ID = pdb.ID - p.TypeMeta = pdb.TypeMeta - p.Model = m - p.SubType = pdb.SubType - if p.Expression == nil { - p.Expression = make(map[string]interface{}) - } - _ = json.Unmarshal(pdb.Expression, &p.Expression) - - return -} - func (p PolicyDefinition) WritePolicyDefinition(policyDirPath string) error { policyPath := filepath.Join(policyDirPath, p.Kind+".json") err := utils.WriteJSONToFile[PolicyDefinition](policyPath, p) diff --git a/models/meshmodel/registry/registry.go b/models/meshmodel/registry/registry.go index 206d6a15..519e4db2 100644 --- a/models/meshmodel/registry/registry.go +++ b/models/meshmodel/registry/registry.go @@ -53,13 +53,14 @@ func NewRegistryManager(db *database.Handler) (*RegistryManager, error) { err := rm.db.AutoMigrate( &Registry{}, &v1beta1.Host{}, - &v1beta1.ComponentDefinitionDB{}, - &v1alpha2.RelationshipDefinitionDB{}, - &v1beta1.PolicyDefinitionDB{}, - &v1beta1.ModelDB{}, - &v1beta1.CategoryDB{}, + &v1beta1.ComponentDefinition{}, + &v1alpha2.RelationshipDefinition{}, + &v1beta1.PolicyDefinition{}, + &v1beta1.Model{}, + &v1beta1.Category{}, ) if err != nil { + fmt.Println(err) return nil, err } return &rm, nil @@ -68,20 +69,22 @@ func (rm *RegistryManager) Cleanup() { _ = rm.db.Migrator().DropTable( &Registry{}, &v1beta1.Host{}, - &v1beta1.ComponentDefinitionDB{}, - &v1beta1.ModelDB{}, - &v1beta1.CategoryDB{}, - &v1alpha2.RelationshipDefinitionDB{}, + &v1beta1.ComponentDefinition{}, + &v1beta1.Model{}, + &v1beta1.Category{}, + &v1alpha2.RelationshipDefinition{}, ) } func (rm *RegistryManager) RegisterEntity(h v1beta1.Host, en entity.Entity) error { registrantID, err := h.Create(rm.db) if err != nil { + fmt.Println("TEST host register : ", err) return err } entityID, err := en.Create(rm.db, registrantID) if err != nil { + fmt.Println("TEST entity register : ", err) return err } diff --git a/models/meshmodel/registry/v1alpha2/relationship_filter.go b/models/meshmodel/registry/v1alpha2/relationship_filter.go index a752c54f..a9c71837 100644 --- a/models/meshmodel/registry/v1alpha2/relationship_filter.go +++ b/models/meshmodel/registry/v1alpha2/relationship_filter.go @@ -5,19 +5,10 @@ import ( "github.com/layer5io/meshkit/database" "github.com/layer5io/meshkit/models/meshmodel/core/v1alpha2" - "github.com/layer5io/meshkit/models/meshmodel/core/v1beta1" "github.com/layer5io/meshkit/models/meshmodel/entity" "gorm.io/gorm/clause" ) -type relationshipDefinitionWithModel struct { - RelationshipDefinitionDB v1alpha2.RelationshipDefinitionDB `gorm:"embedded"` - ModelDB v1beta1.ModelDB `gorm:"embedded"` - // acoount for overridn fields - // v1beta1.ModelDB.Version `json:"modelVersion"` - CategoryDB v1beta1.CategoryDB `gorm:"embedded"` -} - // For now, only filtering by Kind and SubType are allowed. // In the future, we will add support to query using `selectors` (using CUE) // TODO: Add support for Model @@ -42,16 +33,11 @@ func (rf *RelationshipFilter) Create(m map[string]interface{}) { rf.Kind = m["kind"].(string) } func (relationshipFilter *RelationshipFilter) Get(db *database.Handler) ([]entity.Entity, int64, int, error) { - // relationshipFilter, err := utils.Cast[RelationshipFilter](f) - // if err != nil { - // return nil, 0, 0, err - // } - var relationshipDefinitionsWithModel []relationshipDefinitionWithModel - finder := db.Model(&v1alpha2.RelationshipDefinitionDB{}). - Select("relationship_definition_dbs.*, model_dbs.*"). - Joins("JOIN model_dbs ON relationship_definition_dbs.model_id = model_dbs.id"). // - Joins("JOIN category_dbs ON model_dbs.category_id = category_dbs.id") // + var relationshipDefinitionsWithModel []v1alpha2.RelationshipDefinition + finder := db.Model(&v1alpha2.RelationshipDefinition{}).Preload("Model"). + Joins("JOIN model_dbs ON relationship_definition_dbs.model_id = model_dbs.id"). + Joins("JOIN category_dbs ON model_dbs.category_id = category_dbs.id") if relationshipFilter.Kind != "" { if relationshipFilter.Greedy { finder = finder.Where("relationship_definition_dbs.kind LIKE ?", "%"+relationshipFilter.Kind+"%") @@ -89,18 +75,14 @@ func (relationshipFilter *RelationshipFilter) Get(db *database.Handler) ([]entit finder = finder.Limit(relationshipFilter.Limit) } err := finder. - Scan(&relationshipDefinitionsWithModel).Error + Find(&relationshipDefinitionsWithModel).Error if err != nil { fmt.Println(err.Error()) //for debugging } defs := make([]entity.Entity, len(relationshipDefinitionsWithModel)) // remove this when reldef and reldefdb struct is consolidated. - for _, cm := range relationshipDefinitionsWithModel { - // Ensure correct reg is passed, rn it is dummy for sake of testing. - // In the first query above where we do seelection i think there changes will be requrired, an when that two def and defDB structs are consolidated, using association and preload i think we can do. - reg := v1beta1.Host{} - rd := cm.RelationshipDefinitionDB.GetRelationshipDefinition(cm.ModelDB.GetModel(cm.CategoryDB.GetCategory(db), reg)) + for _, rd := range relationshipDefinitionsWithModel { defs = append(defs, &rd) } // Should have count unique relationships (by model version, model name, and relationship's kind, type, subtype, version) diff --git a/models/meshmodel/registry/v1beta1/category_filter.go b/models/meshmodel/registry/v1beta1/category_filter.go index 545ad4cf..5adb78ce 100644 --- a/models/meshmodel/registry/v1beta1/category_filter.go +++ b/models/meshmodel/registry/v1beta1/category_filter.go @@ -25,9 +25,9 @@ func (cf *CategoryFilter) Create(m map[string]interface{}) { } func (cf *CategoryFilter) Get(db *database.Handler) ([]entity.Entity, int64, int, error) { - var catdb []v1beta1.CategoryDB + var catdb []v1beta1.Category var cat []entity.Entity - finder := db.Model(&catdb) + finder := db.Model(&catdb).Debug() // total count before pagination var count int64 @@ -60,10 +60,15 @@ func (cf *CategoryFilter) Get(db *database.Handler) ([]entity.Entity, int64, int finder.Count(&count) } - _ = finder.Find(&catdb).Error + err := finder.Find(&catdb).Error + if err != nil { + return cat, count, int(count), nil + } + for _, c := range catdb { - category := c.GetCategory(db) - cat = append(cat, &category) + // resolve for loop scope + _c := c + cat = append(cat, &_c) } // duplicate category ? return cat, count, int(count), nil diff --git a/models/meshmodel/registry/v1beta1/component_filter.go b/models/meshmodel/registry/v1beta1/component_filter.go index d030594b..3fcdc0a0 100644 --- a/models/meshmodel/registry/v1beta1/component_filter.go +++ b/models/meshmodel/registry/v1beta1/component_filter.go @@ -25,10 +25,10 @@ type ComponentFilter struct { } type componentDefinitionWithModel struct { - ComponentDefinitionDB v1beta1.ComponentDefinitionDB `gorm:"embedded"` - ModelDB v1beta1.ModelDB `gorm:"embedded"` - CategoryDB v1beta1.CategoryDB `gorm:"embedded"` - HostsDB v1beta1.Host `gorm:"embedded"` + ComponentDefinitionDB v1beta1.ComponentDefinition `gorm:"embedded"` + ModelDB v1beta1.Model `gorm:"embedded"` + CategoryDB v1beta1.Category `gorm:"embedded"` + HostsDB v1beta1.Host `gorm:"embedded"` } // Create the filter from map[string]interface{} @@ -39,19 +39,20 @@ func (cf *ComponentFilter) Create(m map[string]interface{}) { cf.Name = m["name"].(string) } -func (componentFilter *ComponentFilter) Get(db *database.Handler) ([]entity.Entity, int64, int, error) { - countUniqueComponents := func(components []componentDefinitionWithModel) int { - set := make(map[string]struct{}) - for _, compWithModel := range components { - key := compWithModel.ComponentDefinitionDB.Component.Kind + "@" + compWithModel.ComponentDefinitionDB.Version + "@" + compWithModel.ModelDB.Name + "@" + compWithModel.ModelDB.Version - if _, ok := set[key]; !ok { - set[key] = struct{}{} - } +func countUniqueComponents (components []componentDefinitionWithModel) int { + set := make(map[string]struct{}) + for _, compWithModel := range components { + key := compWithModel.ComponentDefinitionDB.Component.Kind + "@" + compWithModel.ComponentDefinitionDB.Component.Version + "@" + compWithModel.ModelDB.Name + "@" + compWithModel.ModelDB.Model.Version + if _, ok := set[key]; !ok { + set[key] = struct{}{} } - return len(set) } + return len(set) +} + +func (componentFilter *ComponentFilter) Get(db *database.Handler) ([]entity.Entity, int64, int, error) { var componentDefinitionsWithModel []componentDefinitionWithModel - finder := db.Model(&v1beta1.ComponentDefinitionDB{}). + finder := db.Model(&v1beta1.ComponentDefinition{}). Select("component_definition_dbs.*, model_dbs.*,category_dbs.*, hosts.*"). Joins("JOIN model_dbs ON component_definition_dbs.model_id = model_dbs.id"). Joins("JOIN category_dbs ON model_dbs.category_id = category_dbs.id"). @@ -108,19 +109,20 @@ func (componentFilter *ComponentFilter) Get(db *database.Handler) ([]entity.Enti } defs := make([]entity.Entity, len(componentDefinitionsWithModel)) - // remove this when compoentdef and componetdefdb struct is consolidated. + for _, cm := range componentDefinitionsWithModel { if componentFilter.Trim { cm.ComponentDefinitionDB.Component.Schema = "" } - // Ensure correct reg is passed, rn it is dummy for sake of testing. - // In the first query above where we do seelection i think there changes will be requrired, an when that two def and defDB structs are consolidated, using association and preload i think we can do. + reg := cm.HostsDB - cd := cm.ComponentDefinitionDB.GetComponentDefinition(cm.ModelDB.GetModel(cm.CategoryDB.GetCategory(db), reg)) + cd := cm.ComponentDefinitionDB + cd.Model = cm.ModelDB + cd.Model.Registrant = reg defs = append(defs, &cd) } - unique := countUniqueComponents(componentDefinitionsWithModel) + uniqueCount := countUniqueComponents(componentDefinitionsWithModel) - return defs, count, unique, nil + return defs, count, uniqueCount, nil } diff --git a/models/meshmodel/registry/v1beta1/model_filter.go b/models/meshmodel/registry/v1beta1/model_filter.go index cefe369d..c4386cd8 100644 --- a/models/meshmodel/registry/v1beta1/model_filter.go +++ b/models/meshmodel/registry/v1beta1/model_filter.go @@ -36,30 +36,21 @@ func (mf *ModelFilter) Create(m map[string]interface{}) { mf.Name = m["name"].(string) } -// I have removed Regsitry struct here check if filter by Registrant works. -type modelWithCategories struct { - ModelDB v1beta1.ModelDB `gorm:"embedded"` - CategoryDB v1beta1.CategoryDB `gorm:"embedded"` - Host v1beta1.Host `gorm:"embedded"` - // registry.Registry +func countUniqueModels(models []v1beta1.Model) int { + set := make(map[string]struct{}) + for _, model := range models { + key := model.Name + "@" + model.Model.Version + if _, ok := set[key]; !ok { + set[key] = struct{}{} + } + } + return len(set) } - func (mf *ModelFilter) Get(db *database.Handler) ([]entity.Entity, int64, int, error) { - countUniqueModels := func(models []modelWithCategories) int { - set := make(map[string]struct{}) - for _, model := range models { - key := model.ModelDB.Name + "@" + model.ModelDB.Version - if _, ok := set[key]; !ok { - set[key] = struct{}{} - } - } - return len(set) - } + var modelWithCategories []v1beta1.Model - var modelWithCategoriess []modelWithCategories - finder := db.Model(&v1beta1.ModelDB{}). - Select("model_dbs.*, category_dbs.*", "registries.*", "hosts.*"). + finder := db.Model(&v1beta1.Model{}).Preload("Category").Preload("Registrant"). Joins("JOIN category_dbs ON model_dbs.category_id = category_dbs.id"). Joins("JOIN registries ON registries.entity = model_dbs.id"). Joins("JOIN hosts ON hosts.id = registries.registrant_id") @@ -132,23 +123,15 @@ func (mf *ModelFilter) Get(db *database.Handler) ([]entity.Entity, int64, int, e // includeRelationships = mf.Relationships err := finder. - Scan(&modelWithCategoriess).Error + Find(&modelWithCategories).Error if err != nil { - fmt.Println(modelWithCategoriess) + fmt.Println(modelWithCategories) fmt.Println(err.Error()) //for debugging } defs := []entity.Entity{} - for _, modelDB := range modelWithCategoriess { - // host := rm.GetRegistrant(model) - // model.HostID = host.ID - // model.HostName = host.Hostname - // model.DisplayHostName = host.Hostname - // all these above accounted by having registrant as an atribute in the model schema. - reg := modelDB.Host - model := modelDB.ModelDB.GetModel(modelDB.CategoryDB.GetCategory(db), reg) - + for _, modelDB := range modelWithCategories { // is this required? not used by UI, confirm with Yash once // if includeComponents { // var components []v1beta1.ComponentDefinitionDB @@ -171,7 +154,11 @@ func (mf *ModelFilter) Get(db *database.Handler) ([]entity.Entity, int64, int, e // } // // model.Relationships = relationships // } - defs = append(defs, &model) + fmt.Println(modelDB.Registrant, modelDB.Category) + // resolve for loop scope + _modelDB := modelDB + + defs = append(defs, &_modelDB) } - return defs, count, countUniqueModels(modelWithCategoriess), nil + return defs, count, countUniqueModels(modelWithCategories), nil } diff --git a/models/meshmodel/registry/v1beta1/policy_filter.go b/models/meshmodel/registry/v1beta1/policy_filter.go index 6bba2f1c..c481e84f 100644 --- a/models/meshmodel/registry/v1beta1/policy_filter.go +++ b/models/meshmodel/registry/v1beta1/policy_filter.go @@ -23,16 +23,11 @@ func (pf *PolicyFilter) Create(m map[string]interface{}) { } } -type policyDefinitionWithModel struct { - PolicyDB v1beta1.PolicyDefinitionDB `gorm:"embedded"` - Model v1beta1.Model `gorm:"embedded"` -} - func (pf *PolicyFilter) Get(db *database.Handler) ([]entity.Entity, int64, int, error) { pl := []entity.Entity{} - var componentDefinitionsWithModel []policyDefinitionWithModel - finder := db.Model(&v1beta1.PolicyDefinitionDB{}). - Select("policy_definition_dbs.*, models_dbs.*"). + var policyDefinitionWithModel []v1beta1.PolicyDefinition + finder := db.Model(&v1beta1.PolicyDefinition{}). + Select("policy_definition_dbs.*"). Joins("JOIN model_dbs ON model_dbs.id = policy_definition_dbs.model_id") if pf.Kind != "" { finder = finder.Where("policy_definition_dbs.kind = ?", pf.Kind) @@ -47,13 +42,12 @@ func (pf *PolicyFilter) Get(db *database.Handler) ([]entity.Entity, int64, int, var count int64 finder.Count(&count) - err := finder.Scan(&componentDefinitionsWithModel).Error + err := finder.Scan(&policyDefinitionWithModel).Error if err != nil { return pl, 0, 0, err } - for _, cm := range componentDefinitionsWithModel { - policyDef := cm.PolicyDB.GetPolicyDefinition(cm.Model) - pl = append(pl, &policyDef) + for _, pm := range policyDefinitionWithModel { + pl = append(pl, &pm) } return pl, count, int(count), nil } From 3911988a9cd5a48f199761238fa3d4bbbaa961ba Mon Sep 17 00:00:00 2001 From: MUzairS15 Date: Tue, 30 Apr 2024 18:10:37 +0530 Subject: [PATCH 25/32] finalize changes Signed-off-by: MUzairS15 --- .../meshmodel/core/v1alpha2/relationship.go | 16 +++++----- models/meshmodel/core/v1beta1/component.go | 4 ++- models/meshmodel/core/v1beta1/models.go | 2 -- models/meshmodel/registry/registry.go | 3 -- .../registry/v1alpha2/relationship_filter.go | 12 ++++---- .../registry/v1beta1/category_filter.go | 1 + .../registry/v1beta1/component_filter.go | 10 +++---- .../registry/v1beta1/model_filter.go | 30 ++----------------- 8 files changed, 25 insertions(+), 53 deletions(-) diff --git a/models/meshmodel/core/v1alpha2/relationship.go b/models/meshmodel/core/v1alpha2/relationship.go index b46d8025..d4b65ba3 100644 --- a/models/meshmodel/core/v1alpha2/relationship.go +++ b/models/meshmodel/core/v1alpha2/relationship.go @@ -12,19 +12,21 @@ import ( "gorm.io/gorm/clause" ) +const SchemaVersion = "core.meshery.io/v1alpha2" + type RelationshipDefinition struct { ID uuid.UUID `json:"id"` v1beta1.VersionMeta Kind string `json:"kind,omitempty" yaml:"kind"` // The property has been named RelationshipType instead of Type to avoid collision from Type() function, which enables support for dynamic type. // Though, the column name and the json representation is "type". - RelationshipType string `json:"type" yaml:"type" gorm:"type"` - SubType string `json:"subType" yaml:"subType"` - EvaluationQuery string `json:"evaluationQuery" yaml:"evaluationQuery" gorm:"evaluationQuery"` - Metadata []byte `json:"metadata" yaml:"metadata"` - ModelID uuid.UUID `json:"-" gorm:"index:idx_relationship_definition_dbs_model_id,column:model_id"` - Model v1beta1.Model `json:"model" gorm:"foreignKey:ModelID;references:ID"` - Selectors []byte `json:"selectors" yaml:"selectors"` + RelationshipType string `json:"type" yaml:"type" gorm:"type"` + SubType string `json:"subType" yaml:"subType"` + EvaluationQuery string `json:"evaluationQuery" yaml:"evaluationQuery" gorm:"evaluationQuery"` + Metadata map[string]interface{} `json:"metadata" yaml:"metadata" gorm:"type:bytes;serializer:json"` + ModelID uuid.UUID `json:"-" gorm:"index:idx_relationship_definition_dbs_model_id,column:model_id"` + Model v1beta1.Model `json:"model" gorm:"foreignKey:ModelID;references:ID"` + Selectors []map[string]interface{} `json:"selectors" yaml:"selectors" gorm:"type:bytes;serializer:json"` } func (r RelationshipDefinition) TableName() string { diff --git a/models/meshmodel/core/v1beta1/component.go b/models/meshmodel/core/v1beta1/component.go index aefafb60..9ce205c2 100644 --- a/models/meshmodel/core/v1beta1/component.go +++ b/models/meshmodel/core/v1beta1/component.go @@ -87,7 +87,6 @@ func (c *ComponentDefinition) Create(db *database.Handler, hostID uuid.UUID) (uu c.ModelID = mid err = db.Omit(clause.Associations).Create(&c).Error - fmt.Println("ERROR REGISTER COMPONENT", err) return c.ID, err } @@ -96,6 +95,9 @@ func (m *ComponentDefinition) UpdateStatus(db *database.Handler, status entity.E } func (c ComponentDefinition) WriteComponentDefinition(componentDirPath string) error { + if c.Component.Kind == "" { + return nil + } componentPath := filepath.Join(componentDirPath, c.Component.Kind+".json") err := utils.WriteJSONToFile[ComponentDefinition](componentPath, c) return err diff --git a/models/meshmodel/core/v1beta1/models.go b/models/meshmodel/core/v1beta1/models.go index 82e6947e..b17db212 100644 --- a/models/meshmodel/core/v1beta1/models.go +++ b/models/meshmodel/core/v1beta1/models.go @@ -69,7 +69,6 @@ func (m *Model) Create(db *database.Handler, hostID uuid.UUID) (uuid.UUID, error } modelID := uuid.NewSHA1(uuid.UUID{}, byt) var model Model - fmt.Println("TEST MODEL NAME ", m.Name, m) if m.Name == "" { return uuid.UUID{}, fmt.Errorf("empty or invalid model name passed") } @@ -90,7 +89,6 @@ func (m *Model) Create(db *database.Handler, hostID uuid.UUID) (uuid.UUID, error m.Status = entity.Enabled err = db.Omit(clause.Associations).Create(&m).Error if err != nil { - fmt.Println("MODEL REGISTER >>>>>", err) return uuid.UUID{}, err } // register model inside registries table diff --git a/models/meshmodel/registry/registry.go b/models/meshmodel/registry/registry.go index 519e4db2..a5d1873e 100644 --- a/models/meshmodel/registry/registry.go +++ b/models/meshmodel/registry/registry.go @@ -60,7 +60,6 @@ func NewRegistryManager(db *database.Handler) (*RegistryManager, error) { &v1beta1.Category{}, ) if err != nil { - fmt.Println(err) return nil, err } return &rm, nil @@ -78,13 +77,11 @@ func (rm *RegistryManager) Cleanup() { func (rm *RegistryManager) RegisterEntity(h v1beta1.Host, en entity.Entity) error { registrantID, err := h.Create(rm.db) if err != nil { - fmt.Println("TEST host register : ", err) return err } entityID, err := en.Create(rm.db, registrantID) if err != nil { - fmt.Println("TEST entity register : ", err) return err } diff --git a/models/meshmodel/registry/v1alpha2/relationship_filter.go b/models/meshmodel/registry/v1alpha2/relationship_filter.go index a9c71837..9449b543 100644 --- a/models/meshmodel/registry/v1alpha2/relationship_filter.go +++ b/models/meshmodel/registry/v1alpha2/relationship_filter.go @@ -1,8 +1,6 @@ package v1alpha2 import ( - "fmt" - "github.com/layer5io/meshkit/database" "github.com/layer5io/meshkit/models/meshmodel/core/v1alpha2" "github.com/layer5io/meshkit/models/meshmodel/entity" @@ -57,7 +55,7 @@ func (relationshipFilter *RelationshipFilter) Get(db *database.Handler) ([]entit finder = finder.Where("model_dbs.name = ?", relationshipFilter.ModelName) } if relationshipFilter.Version != "" { - finder = finder.Where("model_dbs.version = ?", relationshipFilter.Version) + finder = finder.Where("model_dbs.model->>'version' = ?", relationshipFilter.Version) } if relationshipFilter.OrderOn != "" { if relationshipFilter.Sort == "desc" { @@ -77,13 +75,15 @@ func (relationshipFilter *RelationshipFilter) Get(db *database.Handler) ([]entit err := finder. Find(&relationshipDefinitionsWithModel).Error if err != nil { - fmt.Println(err.Error()) //for debugging + return nil, 0, 0, err } defs := make([]entity.Entity, len(relationshipDefinitionsWithModel)) - // remove this when reldef and reldefdb struct is consolidated. for _, rd := range relationshipDefinitionsWithModel { - defs = append(defs, &rd) + // resolve for loop scope + _rd := rd + + defs = append(defs, &_rd) } // Should have count unique relationships (by model version, model name, and relationship's kind, type, subtype, version) return defs, count, int(count), nil diff --git a/models/meshmodel/registry/v1beta1/category_filter.go b/models/meshmodel/registry/v1beta1/category_filter.go index 5adb78ce..cd79eaeb 100644 --- a/models/meshmodel/registry/v1beta1/category_filter.go +++ b/models/meshmodel/registry/v1beta1/category_filter.go @@ -68,6 +68,7 @@ func (cf *CategoryFilter) Get(db *database.Handler) ([]entity.Entity, int64, int for _, c := range catdb { // resolve for loop scope _c := c + cat = append(cat, &_c) } // duplicate category ? diff --git a/models/meshmodel/registry/v1beta1/component_filter.go b/models/meshmodel/registry/v1beta1/component_filter.go index 3fcdc0a0..cc3e8c77 100644 --- a/models/meshmodel/registry/v1beta1/component_filter.go +++ b/models/meshmodel/registry/v1beta1/component_filter.go @@ -1,8 +1,6 @@ package v1beta1 import ( - "fmt" - "github.com/layer5io/meshkit/database" "github.com/layer5io/meshkit/models/meshmodel/core/v1beta1" "github.com/layer5io/meshkit/models/meshmodel/entity" @@ -39,7 +37,7 @@ func (cf *ComponentFilter) Create(m map[string]interface{}) { cf.Name = m["name"].(string) } -func countUniqueComponents (components []componentDefinitionWithModel) int { +func countUniqueComponents(components []componentDefinitionWithModel) int { set := make(map[string]struct{}) for _, compWithModel := range components { key := compWithModel.ComponentDefinitionDB.Component.Kind + "@" + compWithModel.ComponentDefinitionDB.Component.Version + "@" + compWithModel.ModelDB.Name + "@" + compWithModel.ModelDB.Model.Version @@ -93,7 +91,7 @@ func (componentFilter *ComponentFilter) Get(db *database.Handler) ([]entity.Enti finder = finder.Where("category_dbs.name = ?", componentFilter.CategoryName) } if componentFilter.Version != "" { - finder = finder.Where("model_dbs.version = ?", componentFilter.Version) + finder = finder.Where("model_dbs.model->>'version' = ?", componentFilter.Version) } var count int64 @@ -105,7 +103,7 @@ func (componentFilter *ComponentFilter) Get(db *database.Handler) ([]entity.Enti err := finder. Scan(&componentDefinitionsWithModel).Error if err != nil { - fmt.Println(err.Error()) //for debugging + return nil, 0, 0, err } defs := make([]entity.Entity, len(componentDefinitionsWithModel)) @@ -114,7 +112,7 @@ func (componentFilter *ComponentFilter) Get(db *database.Handler) ([]entity.Enti if componentFilter.Trim { cm.ComponentDefinitionDB.Component.Schema = "" } - + reg := cm.HostsDB cd := cm.ComponentDefinitionDB cd.Model = cm.ModelDB diff --git a/models/meshmodel/registry/v1beta1/model_filter.go b/models/meshmodel/registry/v1beta1/model_filter.go index c4386cd8..dd217f79 100644 --- a/models/meshmodel/registry/v1beta1/model_filter.go +++ b/models/meshmodel/registry/v1beta1/model_filter.go @@ -1,8 +1,6 @@ package v1beta1 import ( - "fmt" - "github.com/layer5io/meshkit/database" "github.com/layer5io/meshkit/models/meshmodel/core/v1beta1" "github.com/layer5io/meshkit/models/meshmodel/entity" @@ -83,7 +81,7 @@ func (mf *ModelFilter) Get(db *database.Handler) ([]entity.Entity, int64, int, e finder = finder.Where("model_dbs.metadata->>'isAnnotation' = false") } if mf.Version != "" { - finder = finder.Where("model_dbs.version = ?", mf.Version) + finder = finder.Where("model_dbs.model->>'version' = ?", mf.Version) } if mf.Category != "" { finder = finder.Where("category_dbs.name = ?", mf.Category) @@ -125,36 +123,12 @@ func (mf *ModelFilter) Get(db *database.Handler) ([]entity.Entity, int64, int, e err := finder. Find(&modelWithCategories).Error if err != nil { - fmt.Println(modelWithCategories) - fmt.Println(err.Error()) //for debugging + return nil, 0, 0, err } defs := []entity.Entity{} for _, modelDB := range modelWithCategories { - // is this required? not used by UI, confirm with Yash once - // if includeComponents { - // var components []v1beta1.ComponentDefinitionDB - // finder := db.Model(&v1beta1.ComponentDefinitionDB{}). - // Select("component_definition_dbs.id, component_definition_dbs.kind,component_definition_dbs.display_name, component_definition_dbs.api_version, component_definition_dbs.metadata"). - // Where("component_definition_dbs.model_id = ?", model.ID) - // if err := finder.Scan(&components).Error; err != nil { - // fmt.Println(err) - // } - // // model.Components = components - // } - // is this required? not used by UI, confirm with Yash once - // if includeRelationships { - // var relationships []v1alpha2.RelationshipDefinitionDB - // finder := db.Model(&v1alpha2.RelationshipDefinitionDB{}). - // Select("relationship_definition_dbs.*"). - // Where("relationship_definition_dbs.model_id = ?", model.ID) - // if err := finder.Scan(&relationships).Error; err != nil { - // fmt.Println(err) - // } - // // model.Relationships = relationships - // } - fmt.Println(modelDB.Registrant, modelDB.Category) // resolve for loop scope _modelDB := modelDB From 8c73a0d7d03ab7b3639679b639831cde402c4bcb Mon Sep 17 00:00:00 2001 From: MUzairS15 Date: Tue, 30 Apr 2024 18:14:28 +0530 Subject: [PATCH 26/32] fix merge conflicts Signed-off-by: MUzairS15 --- models/meshmodel/registry/v1beta1/component_filter.go | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/models/meshmodel/registry/v1beta1/component_filter.go b/models/meshmodel/registry/v1beta1/component_filter.go index cc3e8c77..6c80d705 100644 --- a/models/meshmodel/registry/v1beta1/component_filter.go +++ b/models/meshmodel/registry/v1beta1/component_filter.go @@ -4,6 +4,7 @@ import ( "github.com/layer5io/meshkit/database" "github.com/layer5io/meshkit/models/meshmodel/core/v1beta1" "github.com/layer5io/meshkit/models/meshmodel/entity" + "gorm.io/gorm/clause" ) type ComponentFilter struct { @@ -94,6 +95,15 @@ func (componentFilter *ComponentFilter) Get(db *database.Handler) ([]entity.Enti finder = finder.Where("model_dbs.model->>'version' = ?", componentFilter.Version) } + if componentFilter.OrderOn != "" { + if componentFilter.Sort == "desc" { + finder = finder.Order(clause.OrderByColumn{Column: clause.Column{Name: componentFilter.OrderOn}, Desc: true}) + } else { + finder = finder.Order(componentFilter.OrderOn) + } + } else { + finder = finder.Order("display_name") + } var count int64 finder.Count(&count) finder = finder.Offset(componentFilter.Offset) From e02d271d1e7e8054e39ab51caecb22f2ead5ba2c Mon Sep 17 00:00:00 2001 From: MUzairS15 Date: Tue, 30 Apr 2024 18:49:19 +0530 Subject: [PATCH 27/32] remove uneeded constructs Signed-off-by: MUzairS15 --- models/meshmodel/core/policies/data_models.go | 20 +------------------ 1 file changed, 1 insertion(+), 19 deletions(-) diff --git a/models/meshmodel/core/policies/data_models.go b/models/meshmodel/core/policies/data_models.go index e37e4a50..4f7f9c04 100644 --- a/models/meshmodel/core/policies/data_models.go +++ b/models/meshmodel/core/policies/data_models.go @@ -1,21 +1,3 @@ package policies -// Define a simple policy struct -type RegoPolicy struct { - Rules string `json:"rules"` -} - -type RelationObject struct { - DestinationID string `json:"destination_id,omitempty"` - DestinationName string `json:"destination_name,omitempty"` - SourceId string `json:"source_id,omitempty"` - SourceName string `json:"source_name,omitempty"` - Port map[string]interface{} `json:"port,omitempty"` -} - -type NetworkPolicyRegoResponse struct { - ServicePodRelationships []RelationObject `json:"service_pod_relationships,omitempty"` - ServiceDeploymentRelationships []RelationObject `json:"service_deployment_relationships,omitempty"` -} - -// Add response struct based on schema for all relationships evaluations. binding, inventory, network... +// Add response struct based on schema for all relationships evaluations. binding, network, hierarchical From 6e5402c439e0bccad94cac14202b56e3c347226f Mon Sep 17 00:00:00 2001 From: MUzairS15 Date: Tue, 30 Apr 2024 18:49:48 +0530 Subject: [PATCH 28/32] run gofmt Signed-off-by: MUzairS15 --- .../core/policies/rego_policy_relationship.go | 2 +- models/meshmodel/core/v1alpha2/relationship.go | 14 +++++++------- .../core/v1beta1/application_component.go | 1 + models/meshmodel/core/v1beta1/category.go | 6 ++---- models/meshmodel/core/v1beta1/component.go | 1 - models/meshmodel/core/v1beta1/models.go | 2 +- .../registry/v1alpha2/relationship_filter.go | 2 +- .../meshmodel/registry/v1beta1/category_filter.go | 2 +- 8 files changed, 14 insertions(+), 16 deletions(-) diff --git a/models/meshmodel/core/policies/rego_policy_relationship.go b/models/meshmodel/core/policies/rego_policy_relationship.go index 73434618..789cddbb 100644 --- a/models/meshmodel/core/policies/rego_policy_relationship.go +++ b/models/meshmodel/core/policies/rego_policy_relationship.go @@ -58,7 +58,7 @@ func (r *Rego) RegoPolicyHandler(regoQueryString string, designFile []byte) (int rego.Query(regoQueryString), rego.Load([]string{r.policyDir}, func(abspath string, info fs.FileInfo, depth int) bool { extension := filepath.Ext(abspath) - return extension == "rego" + return extension == "rego" }), rego.Store(r.store), rego.Transaction(r.transaction), diff --git a/models/meshmodel/core/v1alpha2/relationship.go b/models/meshmodel/core/v1alpha2/relationship.go index d4b65ba3..58f15985 100644 --- a/models/meshmodel/core/v1alpha2/relationship.go +++ b/models/meshmodel/core/v1alpha2/relationship.go @@ -20,13 +20,13 @@ type RelationshipDefinition struct { Kind string `json:"kind,omitempty" yaml:"kind"` // The property has been named RelationshipType instead of Type to avoid collision from Type() function, which enables support for dynamic type. // Though, the column name and the json representation is "type". - RelationshipType string `json:"type" yaml:"type" gorm:"type"` - SubType string `json:"subType" yaml:"subType"` - EvaluationQuery string `json:"evaluationQuery" yaml:"evaluationQuery" gorm:"evaluationQuery"` - Metadata map[string]interface{} `json:"metadata" yaml:"metadata" gorm:"type:bytes;serializer:json"` - ModelID uuid.UUID `json:"-" gorm:"index:idx_relationship_definition_dbs_model_id,column:model_id"` - Model v1beta1.Model `json:"model" gorm:"foreignKey:ModelID;references:ID"` - Selectors []map[string]interface{} `json:"selectors" yaml:"selectors" gorm:"type:bytes;serializer:json"` + RelationshipType string `json:"type" yaml:"type" gorm:"type"` + SubType string `json:"subType" yaml:"subType"` + EvaluationQuery string `json:"evaluationQuery" yaml:"evaluationQuery" gorm:"evaluationQuery"` + Metadata map[string]interface{} `json:"metadata" yaml:"metadata" gorm:"type:bytes;serializer:json"` + ModelID uuid.UUID `json:"-" gorm:"index:idx_relationship_definition_dbs_model_id,column:model_id"` + Model v1beta1.Model `json:"model" gorm:"foreignKey:ModelID;references:ID"` + Selectors []map[string]interface{} `json:"selectors" yaml:"selectors" gorm:"type:bytes;serializer:json"` } func (r RelationshipDefinition) TableName() string { diff --git a/models/meshmodel/core/v1beta1/application_component.go b/models/meshmodel/core/v1beta1/application_component.go index c094f29b..b2b546b0 100644 --- a/models/meshmodel/core/v1beta1/application_component.go +++ b/models/meshmodel/core/v1beta1/application_component.go @@ -6,6 +6,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) + // To be removed when Design schema is refactored. // Component is the structure for the core Application Component diff --git a/models/meshmodel/core/v1beta1/category.go b/models/meshmodel/core/v1beta1/category.go index 0b63e4bb..8c870f24 100644 --- a/models/meshmodel/core/v1beta1/category.go +++ b/models/meshmodel/core/v1beta1/category.go @@ -15,8 +15,8 @@ var categoryCreationLock sync.Mutex //Each model will perform a check and if the // swagger:response Category type Category struct { - ID uuid.UUID `json:"-"` - Name string `json:"name" gorm:"name"` + ID uuid.UUID `json:"-"` + Name string `json:"name" gorm:"name"` Metadata map[string]interface{} `json:"metadata" yaml:"metadata" gorm:"type:bytes;serializer:json"` } @@ -34,7 +34,6 @@ func (cat Category) GetID() uuid.UUID { return cat.ID } - func (cat *Category) GetEntityDetail() string { return fmt.Sprintf("name: %s", cat.Name) } @@ -69,4 +68,3 @@ func (cat *Category) Create(db *database.Handler, _ uuid.UUID) (uuid.UUID, error func (m *Category) UpdateStatus(db database.Handler, status entity.EntityStatus) error { return nil } - diff --git a/models/meshmodel/core/v1beta1/component.go b/models/meshmodel/core/v1beta1/component.go index 9ce205c2..07958119 100644 --- a/models/meshmodel/core/v1beta1/component.go +++ b/models/meshmodel/core/v1beta1/component.go @@ -53,7 +53,6 @@ func (c ComponentDefinition) TableName() string { return "component_definition_dbs" } - func (c ComponentDefinition) Type() entity.EntityType { return entity.ComponentDefinition } diff --git a/models/meshmodel/core/v1beta1/models.go b/models/meshmodel/core/v1beta1/models.go index b17db212..294b4a0d 100644 --- a/models/meshmodel/core/v1beta1/models.go +++ b/models/meshmodel/core/v1beta1/models.go @@ -33,7 +33,7 @@ type Model struct { RegistrantID uuid.UUID `json:"hostID" gorm:"column:host_id"` // make as a foreign refer to host's table Registrant Host `json:"registrant" gorm:"foreignKey:RegistrantID;references:ID"` CategoryID uuid.UUID `json:"-" gorm:"categoryID"` - Category Category `json:"category" gorm:"foreignKey:CategoryID;references:ID"` + Category Category `json:"category" gorm:"foreignKey:CategoryID;references:ID"` SubCategory string `json:"subCategory" gorm:"subCategory"` Metadata map[string]interface{} `json:"metadata" gorm:"type:bytes;serializer:json"` Model ModelEntity `json:"model,omitempty" gorm:"model;type:bytes;serializer:json"` diff --git a/models/meshmodel/registry/v1alpha2/relationship_filter.go b/models/meshmodel/registry/v1alpha2/relationship_filter.go index 9449b543..ded04329 100644 --- a/models/meshmodel/registry/v1alpha2/relationship_filter.go +++ b/models/meshmodel/registry/v1alpha2/relationship_filter.go @@ -82,7 +82,7 @@ func (relationshipFilter *RelationshipFilter) Get(db *database.Handler) ([]entit for _, rd := range relationshipDefinitionsWithModel { // resolve for loop scope _rd := rd - + defs = append(defs, &_rd) } // Should have count unique relationships (by model version, model name, and relationship's kind, type, subtype, version) diff --git a/models/meshmodel/registry/v1beta1/category_filter.go b/models/meshmodel/registry/v1beta1/category_filter.go index cd79eaeb..07ff177c 100644 --- a/models/meshmodel/registry/v1beta1/category_filter.go +++ b/models/meshmodel/registry/v1beta1/category_filter.go @@ -68,7 +68,7 @@ func (cf *CategoryFilter) Get(db *database.Handler) ([]entity.Entity, int64, int for _, c := range catdb { // resolve for loop scope _c := c - + cat = append(cat, &_c) } // duplicate category ? From e7a46c658e5364f2b685beee9cd3cebceb6f85eb Mon Sep 17 00:00:00 2001 From: MUzairS15 Date: Fri, 3 May 2024 17:18:41 +0530 Subject: [PATCH 29/32] preload category and initialize slice with cap instead of len Signed-off-by: MUzairS15 --- models/meshmodel/registry/v1alpha2/relationship_filter.go | 4 ++-- models/meshmodel/registry/v1beta1/component_filter.go | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/models/meshmodel/registry/v1alpha2/relationship_filter.go b/models/meshmodel/registry/v1alpha2/relationship_filter.go index ded04329..e07b63b7 100644 --- a/models/meshmodel/registry/v1alpha2/relationship_filter.go +++ b/models/meshmodel/registry/v1alpha2/relationship_filter.go @@ -33,7 +33,7 @@ func (rf *RelationshipFilter) Create(m map[string]interface{}) { func (relationshipFilter *RelationshipFilter) Get(db *database.Handler) ([]entity.Entity, int64, int, error) { var relationshipDefinitionsWithModel []v1alpha2.RelationshipDefinition - finder := db.Model(&v1alpha2.RelationshipDefinition{}).Preload("Model"). + finder := db.Model(&v1alpha2.RelationshipDefinition{}).Preload("Model").Preload("Model.Category"). Joins("JOIN model_dbs ON relationship_definition_dbs.model_id = model_dbs.id"). Joins("JOIN category_dbs ON model_dbs.category_id = category_dbs.id") if relationshipFilter.Kind != "" { @@ -77,7 +77,7 @@ func (relationshipFilter *RelationshipFilter) Get(db *database.Handler) ([]entit if err != nil { return nil, 0, 0, err } - defs := make([]entity.Entity, len(relationshipDefinitionsWithModel)) + defs := make([]entity.Entity, 0, len(relationshipDefinitionsWithModel)) for _, rd := range relationshipDefinitionsWithModel { // resolve for loop scope diff --git a/models/meshmodel/registry/v1beta1/component_filter.go b/models/meshmodel/registry/v1beta1/component_filter.go index 6c80d705..3e41f874 100644 --- a/models/meshmodel/registry/v1beta1/component_filter.go +++ b/models/meshmodel/registry/v1beta1/component_filter.go @@ -126,6 +126,7 @@ func (componentFilter *ComponentFilter) Get(db *database.Handler) ([]entity.Enti reg := cm.HostsDB cd := cm.ComponentDefinitionDB cd.Model = cm.ModelDB + cd.Model.Category = cm.CategoryDB cd.Model.Registrant = reg defs = append(defs, &cd) } From cb90654121a7c21ba21606dab16e4536d926886a Mon Sep 17 00:00:00 2001 From: MUzairS15 Date: Fri, 3 May 2024 18:14:31 +0530 Subject: [PATCH 30/32] rmv logs Signed-off-by: MUzairS15 --- .../meshmodel/core/policies/rego_policy_relationship.go | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/models/meshmodel/core/policies/rego_policy_relationship.go b/models/meshmodel/core/policies/rego_policy_relationship.go index 789cddbb..423116d3 100644 --- a/models/meshmodel/core/policies/rego_policy_relationship.go +++ b/models/meshmodel/core/policies/rego_policy_relationship.go @@ -3,8 +3,6 @@ package policies import ( "context" "fmt" - "io/fs" - "path/filepath" "github.com/layer5io/meshkit/models/meshmodel/registry" "github.com/layer5io/meshkit/models/meshmodel/registry/v1alpha2" @@ -56,10 +54,7 @@ func (r *Rego) RegoPolicyHandler(regoQueryString string, designFile []byte) (int } regoEngine, err := rego.New( rego.Query(regoQueryString), - rego.Load([]string{r.policyDir}, func(abspath string, info fs.FileInfo, depth int) bool { - extension := filepath.Ext(abspath) - return extension == "rego" - }), + rego.Load([]string{r.policyDir}, nil), rego.Store(r.store), rego.Transaction(r.transaction), ).PrepareForEval(r.ctx) @@ -78,7 +73,6 @@ func (r *Rego) RegoPolicyHandler(regoQueryString string, designFile []byte) (int if err != nil { return nil, ErrEval(err) } - if !eval_result.Allowed() { if len(eval_result) > 0 { if len(eval_result[0].Expressions) > 0 { From 2b558708b20c8ecce5419c23d6556fd857213bee Mon Sep 17 00:00:00 2001 From: MUzairS15 Date: Sun, 5 May 2024 12:47:02 +0530 Subject: [PATCH 31/32] return comps and rels for model based on filter conds Signed-off-by: MUzairS15 --- models/meshmodel/core/v1beta1/models.go | 24 ++++++++------- .../registry/v1beta1/model_filter.go | 30 +++++++++++++++---- 2 files changed, 38 insertions(+), 16 deletions(-) diff --git a/models/meshmodel/core/v1beta1/models.go b/models/meshmodel/core/v1beta1/models.go index 294b4a0d..e0424527 100644 --- a/models/meshmodel/core/v1beta1/models.go +++ b/models/meshmodel/core/v1beta1/models.go @@ -26,17 +26,19 @@ type ModelEntity struct { type Model struct { ID uuid.UUID `json:"id"` VersionMeta - Name string `json:"name" gorm:"modelName"` - DisplayName string `json:"displayName"` - Description string `json:"description" gorm:"description"` - Status entity.EntityStatus `json:"status" gorm:"status"` - RegistrantID uuid.UUID `json:"hostID" gorm:"column:host_id"` // make as a foreign refer to host's table - Registrant Host `json:"registrant" gorm:"foreignKey:RegistrantID;references:ID"` - CategoryID uuid.UUID `json:"-" gorm:"categoryID"` - Category Category `json:"category" gorm:"foreignKey:CategoryID;references:ID"` - SubCategory string `json:"subCategory" gorm:"subCategory"` - Metadata map[string]interface{} `json:"metadata" gorm:"type:bytes;serializer:json"` - Model ModelEntity `json:"model,omitempty" gorm:"model;type:bytes;serializer:json"` + Name string `json:"name" gorm:"modelName"` + DisplayName string `json:"displayName"` + Description string `json:"description" gorm:"description"` + Status entity.EntityStatus `json:"status" gorm:"status"` + RegistrantID uuid.UUID `json:"hostID" gorm:"column:host_id"` // make as a foreign refer to host's table + Registrant Host `json:"registrant" gorm:"foreignKey:RegistrantID;references:ID"` + CategoryID uuid.UUID `json:"-" gorm:"categoryID"` + Category Category `json:"category" gorm:"foreignKey:CategoryID;references:ID"` + SubCategory string `json:"subCategory" gorm:"subCategory"` + Metadata map[string]interface{} `json:"metadata" gorm:"type:bytes;serializer:json"` + Model ModelEntity `json:"model,omitempty" gorm:"model;type:bytes;serializer:json"` + Components []ComponentDefinition `json:"components" gorm:"-"` + Relationships interface{} `json:"relationships" gorm:"-"` } func (m Model) TableName() string { diff --git a/models/meshmodel/registry/v1beta1/model_filter.go b/models/meshmodel/registry/v1beta1/model_filter.go index dd217f79..e8bf85d2 100644 --- a/models/meshmodel/registry/v1beta1/model_filter.go +++ b/models/meshmodel/registry/v1beta1/model_filter.go @@ -2,6 +2,7 @@ package v1beta1 import ( "github.com/layer5io/meshkit/database" + "github.com/layer5io/meshkit/models/meshmodel/core/v1alpha2" "github.com/layer5io/meshkit/models/meshmodel/core/v1beta1" "github.com/layer5io/meshkit/models/meshmodel/entity" "gorm.io/gorm/clause" @@ -44,6 +45,7 @@ func countUniqueModels(models []v1beta1.Model) int { } return len(set) } + func (mf *ModelFilter) Get(db *database.Handler) ([]entity.Entity, int64, int, error) { var modelWithCategories []v1beta1.Model @@ -57,7 +59,7 @@ func (mf *ModelFilter) Get(db *database.Handler) ([]entity.Entity, int64, int, e var count int64 // include components and relationships in response body - // var includeComponents, includeRelationships bool + var includeComponents, includeRelationships bool if mf.Greedy { if mf.Name != "" && mf.DisplayName != "" { @@ -116,9 +118,8 @@ func (mf *ModelFilter) Get(db *database.Handler) ([]entity.Entity, int64, int, e finder = finder.Where("model_dbs.status = ?", mf.Status) } - // is this required? not used by UI, confirm with Yash once - // includeComponents = mf.Components - // includeRelationships = mf.Relationships + includeComponents = mf.Components + includeRelationships = mf.Relationships err := finder. Find(&modelWithCategories).Error @@ -131,7 +132,26 @@ func (mf *ModelFilter) Get(db *database.Handler) ([]entity.Entity, int64, int, e for _, modelDB := range modelWithCategories { // resolve for loop scope _modelDB := modelDB - + if includeComponents { + var components []v1beta1.ComponentDefinition + finder := db.Model(&v1beta1.ComponentDefinition{}). + Select("component_definition_dbs.id, component_definition_dbs.component, component_definition_dbs.display_name, component_definition_dbs.metadata"). + Where("component_definition_dbs.model_id = ?", _modelDB.ID) + if err := finder.Scan(&components).Error; err != nil { + return nil, 0, 0, err + } + _modelDB.Components = components + } + if includeRelationships { + var relationships []v1alpha2.RelationshipDefinition + finder := db.Model(&v1alpha2.RelationshipDefinition{}). + Select("relationship_definition_dbs.*"). + Where("relationship_definition_dbs.model_id = ?", _modelDB.ID) + if err := finder.Scan(&relationships).Error; err != nil { + return nil, 0, 0, err + } + _modelDB.Relationships = relationships + } defs = append(defs, &_modelDB) } return defs, count, countUniqueModels(modelWithCategories), nil From fb758835eaa6b407f7d341df5aa3fb4b4509476b Mon Sep 17 00:00:00 2001 From: MUzairS15 Date: Sun, 5 May 2024 19:13:04 +0530 Subject: [PATCH 32/32] initialize result slice Signed-off-by: MUzairS15 --- models/meshmodel/registry/v1beta1/component_filter.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/models/meshmodel/registry/v1beta1/component_filter.go b/models/meshmodel/registry/v1beta1/component_filter.go index 3e41f874..b9b401e8 100644 --- a/models/meshmodel/registry/v1beta1/component_filter.go +++ b/models/meshmodel/registry/v1beta1/component_filter.go @@ -116,7 +116,7 @@ func (componentFilter *ComponentFilter) Get(db *database.Handler) ([]entity.Enti return nil, 0, 0, err } - defs := make([]entity.Entity, len(componentDefinitionsWithModel)) + defs := make([]entity.Entity, 0, len(componentDefinitionsWithModel)) for _, cm := range componentDefinitionsWithModel { if componentFilter.Trim {