Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

migrategroups: update migrate groups #2329

Merged
merged 1 commit into from
Dec 14, 2023
Merged

migrategroups: update migrate groups #2329

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
135 changes: 98 additions & 37 deletions cmd/migrategroups/migrategroups/migrategroups.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,10 +6,12 @@ import (
"encoding/json"
"errors"

"github.com/redhatinsights/edge-api/pkg/clients/inventory"
"github.com/redhatinsights/edge-api/pkg/clients/inventorygroups"
"github.com/redhatinsights/edge-api/pkg/db"
"github.com/redhatinsights/edge-api/pkg/models"
"github.com/redhatinsights/edge-api/pkg/routes/common"
"github.com/redhatinsights/edge-api/pkg/services/utility"
feature "github.com/redhatinsights/edge-api/unleash/features"

"github.com/redhatinsights/platform-go-middlewares/identity"
Expand All @@ -26,30 +28,26 @@ var ErrMigrationFeatureNotAvailable = errors.New("groups migrations is not avail
// ErrOrgIDIsMandatory error returned when the org_id with empty value is passed
var ErrOrgIDIsMandatory = errors.New("org_id is mandatory")

// ErrInventoryGroupAlreadyExist error returned when trying to migrate an already existing group
var ErrInventoryGroupAlreadyExist = errors.New("inventory group already exist")
omaciel marked this conversation as resolved.
Show resolved Hide resolved

// DefaultDataLimit the default data limit to use when collecting data
var DefaultDataLimit = 100
var DefaultDataLimit = 1000

// DefaultMaxDataPageNumber the default data pages to handle as preventive way to enter an indefinite loop
var DefaultMaxDataPageNumber = 100
// two pages should be sufficient as DefaultDataLimit is large enough
var DefaultMaxDataPageNumber = 2

// DefaultIdentityType the default identity type used in header when requesting inventory groups end-point
var DefaultIdentityType = "System"
var DefaultIdentityType = "User"

// AuthTypeBASIC the BASIC identity type used in header when requesting inventory groups end-point
var AuthTypeBASIC = "basic-auth"

// OrgsGroupsFilter the filter added to filter an organization groups (if the org_id is defined in the map as a key)
var OrgsGroupsFilter = map[string][]interface{}{
"11789772": {"device_groups.name LIKE ?", "%-Store-%"},
}
// simple map item: org_id: {"device_groups.name LIKE ?", "%-Store-%"},
// to use a filter by any org, use a wildcard : "*": {"device_groups.name LIKE ?", "%-Store-%"},
var OrgsGroupsFilter = map[string][]interface{}{}

// DefaultOrgsIDS if the slice is not empty, only the organizations with this ids will be taken into account when migrating
var DefaultOrgsIDS = []string{
"11789772",
}
var DefaultOrgsIDS = []string{}

// OrgCandidate the org candidate queried from the database
type OrgCandidate struct {
Expand All @@ -61,45 +59,90 @@ type OrgCandidate struct {
// GetNewInventoryGroupClient the function to get the client to inventory groups end-point, for testing convenience
var GetNewInventoryGroupClient = inventorygroups.InitClient

func newInventoryGroupsOrgClient(orgID string) (inventorygroups.ClientInterface, error) {
// create a new inventory-groups client and set organization identity in the initialization context
// GetNewInventoryClient the function to get the client to inventory end-point
var GetNewInventoryClient = inventory.InitClient

type InventoryOrgClients struct {
InventoryClient inventory.ClientInterface
InventoryGroupsClient inventorygroups.ClientInterface
}

// newInventoryOrgClients create a new inventory groups client
func newInventoryOrgClients(orgID string) (*InventoryOrgClients, error) {
// create a new organization identity in the initialization context
ident := identity.XRHID{Identity: identity.Identity{
OrgID: orgID,
Type: DefaultIdentityType,
AuthType: AuthTypeBASIC,
Internal: identity.Internal{OrgID: orgID},
User: identity.User{OrgAdmin: true, Email: "[email protected]", FirstName: "edge-groups-migrator"},
}}
jsonIdent, err := json.Marshal(&ident)
if err != nil {
return nil, err
}
base64Identity := base64.StdEncoding.EncodeToString(jsonIdent)

ctx := context.Background()
ctx = context.WithValue(ctx, request_id.RequestIDKey, uuid.NewString())
ctx = common.SetOriginalIdentity(ctx, base64Identity)
client := GetNewInventoryGroupClient(ctx, log.NewEntry(log.StandardLogger()))

return client, nil
clientLog := log.WithFields(log.Fields{
"org_id": orgID,
"context": "org-groups-migration",
})
clients := InventoryOrgClients{
InventoryClient: GetNewInventoryClient(ctx, clientLog),
InventoryGroupsClient: GetNewInventoryGroupClient(ctx, clientLog),
}
return &clients, nil
}

func createInventoryGroup(client inventorygroups.ClientInterface, edgeGroup models.DeviceGroup) error {
func getInventoryGroupHostsToAdd(clients *InventoryOrgClients, edgeGroup models.DeviceGroup) ([]string, error) {
logger := log.WithFields(log.Fields{
"context": "org-group-migration",
"org_id": edgeGroup.OrgID,
"group_name": edgeGroup.Name,
})
groupsHosts := make([]string, 0, len(edgeGroup.Devices))
// filter group devices to have only those with uuids
for _, device := range edgeGroup.Devices {
if device.UUID == "" {
continue
}
groupsHosts = append(groupsHosts, device.UUID)
}
hostIDS := make([]string, 0, len(groupsHosts))
if len(groupsHosts) > 0 {
// check all devices in inventory, to filter the ones that does not exist and the ones that are already in other groups
result, err := clients.InventoryClient.ReturnDeviceListByID(groupsHosts)
if err != nil {
logger.WithField("error", err.Error()).Info("error occurred while getting group devices from inventory")
return hostIDS, err
}
for _, inventoryDevice := range result.Result {
if len(inventoryDevice.Groups) == 0 {
hostIDS = append(hostIDS, inventoryDevice.ID)
}
}
}

return hostIDS, nil
}

func createInventoryGroup(clients *InventoryOrgClients, edgeGroup models.DeviceGroup) error {
hostIDS, err := getInventoryGroupHostsToAdd(clients, edgeGroup)
if err != nil {
return err
}

logger := log.WithFields(log.Fields{
"context": "org-group-migration",
"org_id": edgeGroup.OrgID,
"group_name": edgeGroup.Name,
"hosts": groupsHosts,
"hosts": hostIDS,
})

logger.Info("inventory group create started")
inventoryGroup, err := client.CreateGroup(edgeGroup.Name, groupsHosts)
inventoryGroup, err := clients.InventoryGroupsClient.CreateGroup(edgeGroup.Name, hostIDS)
if err != nil {
logger.WithField("error", err.Error()).Info("error occurred while creating inventory group")
return err
Expand All @@ -116,7 +159,7 @@ func createInventoryGroup(client inventorygroups.ClientInterface, edgeGroup mode
return nil
}

func migrateGroup(client inventorygroups.ClientInterface, edgeGroup models.DeviceGroup) error {
func migrateGroup(clients *InventoryOrgClients, edgeGroup models.DeviceGroup) error {
logger := log.WithFields(log.Fields{
"context": "org-group-migration",
"org_id": edgeGroup.OrgID,
Expand All @@ -125,16 +168,15 @@ func migrateGroup(client inventorygroups.ClientInterface, edgeGroup models.Devic
logger.Info("group migration started")

// check if group exist in inventory group
if _, err := client.GetGroupByName(edgeGroup.Name); err != nil && err != inventorygroups.ErrGroupNotFound {
if _, err := clients.InventoryGroupsClient.GetGroupByName(edgeGroup.Name); err != nil && err != inventorygroups.ErrGroupNotFound {
logger.WithField("error", err.Error()).Error("unknown error occurred while getting inventory group")
return err
} else if err == nil {
// inventory group should not exist to continue
logger.Error("group already exists")
return ErrInventoryGroupAlreadyExist
logger.Error("edge group name already exists in inventory groups, migration skipped")
return nil
}

if err := createInventoryGroup(client, edgeGroup); err != nil {
if err := createInventoryGroup(clients, edgeGroup); err != nil {
logger.WithField("error", err.Error()).Error("error occurred while creating inventory group")
return err
}
Expand All @@ -147,21 +189,23 @@ func migrateOrgGroups(orgID string, gormDB *gorm.DB) error {
return ErrOrgIDIsMandatory
}

logger := log.WithFields(log.Fields{"context": "org-groups-migration", "org_id": orgID})
logger := log.WithFields(log.Fields{"context": "org-group-migration", "org_id": orgID})
logger.Info("organization groups migration started")

client, err := newInventoryGroupsOrgClient(orgID)
inventoryOrgClients, err := newInventoryOrgClients(orgID)
if err != nil {
logger.WithField("error", err.Error()).Error("error occurred while creating organization inventory-groups client")
logger.WithField("error", err.Error()).Error("error occurred while creating organization inventory clients")
return err
}

// get all org groups
var orgGroupsToMigrate []models.DeviceGroup
baseQuery := db.OrgDB(orgID, gormDB, "device_groups").Debug().Where("device_groups.uuid IS NULL OR device_groups.uuid = ''")
if orgGroupsFilter, ok := OrgsGroupsFilter[orgID]; ok && len(orgGroupsFilter) > 0 {
query, args := orgGroupsFilter[0], orgGroupsFilter[1:]
baseQuery = baseQuery.Where(query, args...)
for _, filterKey := range []string{orgID, "*"} {
omaciel marked this conversation as resolved.
Show resolved Hide resolved
if orgGroupsFilter, ok := OrgsGroupsFilter[filterKey]; ok && len(orgGroupsFilter) > 0 {
query, args := orgGroupsFilter[0], orgGroupsFilter[1:]
baseQuery = baseQuery.Where(query, args...)
}
}

if err := baseQuery.Preload("Devices").Order("created_at").Find(&orgGroupsToMigrate).Error; err != nil {
Expand All @@ -171,12 +215,12 @@ func migrateOrgGroups(orgID string, gormDB *gorm.DB) error {
logger = log.WithField("groups_count", len(orgGroupsToMigrate))

for _, group := range orgGroupsToMigrate {
if err := migrateGroup(client, group); err != nil {
if err := migrateGroup(inventoryOrgClients, group); err != nil {
return err
}
}

logger.Info("organization groups migration finished")
logger.WithField("groups-count", len(orgGroupsToMigrate)).Info("organization groups migration finished")
return nil
}

Expand All @@ -189,7 +233,9 @@ func getAllOrgs(gormDB *gorm.DB) ([]OrgCandidate, error) {
Where("device_groups.uuid IS NULL OR device_groups.uuid = ''"). // consider only orgs with empty inventory group id
Where("device_groups.deleted_at IS NULL"). // with non deleted groups
Where("devices.deleted_at IS NULL"). // with non deleted devices
Where("devices.id IS NOT NULL") // we take only groups with hosts
Where("devices.id IS NOT NULL"). // we take only groups with hosts
Group("device_groups.org_id").
Order("device_groups.org_id")

if len(DefaultOrgsIDS) > 0 {
baseQuery = baseQuery.Where("device_groups.org_id IN (?)", DefaultOrgsIDS)
Expand Down Expand Up @@ -217,6 +263,8 @@ func MigrateAllGroups(gormDB *gorm.DB) error {

page := 0
orgsCount := 0
// create a map to track the orgs already processed
orgsMap := make(map[string]bool)
for page < DefaultMaxDataPageNumber {
orgsToMigrate, err := getAllOrgs(gormDB)
if err != nil {
Expand All @@ -232,13 +280,26 @@ func MigrateAllGroups(gormDB *gorm.DB) error {
"groups_count": orgToMigrate.GroupsCount,
"devices_count": orgToMigrate.DevicesCount,
})
if orgsMap[orgToMigrate.OrgID] {
// migrate org only once, an org may return back in the query because:
// a- not all groups has been migrated when an org filter exists in OrgsGroupsFilter
// b- when a group with same name exists in inventory groups
continue
}
if utility.EnforceEdgeGroups(orgToMigrate.OrgID) {
// do not migrate orgs that are enforced to use edge groups
orgLogger.Error("enforce-edge-groups is enabled for this organization, migration is skipped")
continue
}

orgLogger.Info("starting migration of organization groups")
err := migrateOrgGroups(orgToMigrate.OrgID, gormDB)
if err != nil {
orgLogger.WithField("error", err.Error()).Error("error occurred while migrating organization groups")
omaciel marked this conversation as resolved.
Show resolved Hide resolved
return err
}

// register org_id to make sure the org is not processed many times
orgsMap[orgToMigrate.OrgID] = true
}
orgsCount += len(orgsToMigrate)
page++
Expand Down
Loading
Loading