diff --git a/go/consensus/cometbft/apps/registry/admission.go b/go/consensus/cometbft/apps/registry/admission.go new file mode 100644 index 00000000000..5365bfc463b --- /dev/null +++ b/go/consensus/cometbft/apps/registry/admission.go @@ -0,0 +1,237 @@ +package registry + +import ( + beacon "github.com/oasisprotocol/oasis-core/go/beacon/api" + "github.com/oasisprotocol/oasis-core/go/common/node" + "github.com/oasisprotocol/oasis-core/go/consensus/cometbft/api" + registryState "github.com/oasisprotocol/oasis-core/go/consensus/cometbft/apps/registry/state" + registry "github.com/oasisprotocol/oasis-core/go/registry/api" +) + +// policyFn is an admission policy verification function. +type policyFn func( + *api.Context, + *registryState.MutableState, + *node.Node, + *registry.Runtime, + beacon.EpochTime, +) error + +// admissionPolicyFns are the global admission policy verification functions. +var admissionPolicyFns = []policyFn{ + // Whitelist. + verifyRuntimeWhitelistAdmissionPolicy, + // Per-role. + verifyRuntimePerRoleAdmissionPolicy, +} + +// rolePolicyFn is a per-role admission policy verification function. +type rolePolicyFn func( + *api.Context, + *registryState.MutableState, + *node.Node, + *registry.Runtime, + beacon.EpochTime, + node.RolesMask, + *registry.PerRoleAdmissionPolicy, +) error + +// perRoleAdmissionPolicyFns are the per-role admission policy verification functions. +var perRoleAdmissionPolicyFns = []rolePolicyFn{ + // Whitelist. + verifyRuntimePerRoleWhitelistAdmissionPolicy, +} + +func verifyRuntimeAdmissionPolicy( + ctx *api.Context, + state *registryState.MutableState, + newNode *node.Node, + rt *registry.Runtime, + epoch beacon.EpochTime, +) error { + // Process all admission policy functions. + for _, policyFn := range admissionPolicyFns { + if err := policyFn(ctx, state, newNode, rt, epoch); err != nil { + return err + } + } + + return nil +} + +func verifyRuntimeWhitelistAdmissionPolicy( + ctx *api.Context, + state *registryState.MutableState, + newNode *node.Node, + rt *registry.Runtime, + epoch beacon.EpochTime, +) error { + if rt.AdmissionPolicy.EntityWhitelist == nil { + return nil // No whitelist policy specified. + } + + wcfg, entIsWhitelisted := rt.AdmissionPolicy.EntityWhitelist.Entities[newNode.EntityID] + if !entIsWhitelisted { + ctx.Logger().Debug("RegisterNode: node's entity not in a runtime's whitelist", + "entity_id", newNode.EntityID, + "runtime_id", rt.ID, + "node_id", newNode.ID, + ) + return registry.ErrForbidden + } + if len(wcfg.MaxNodes) == 0 { + return nil // Any amount of nodes allowed. + } + + // Map is present and non-empty, check per-role restrictions + // on the maximum number of nodes per entity. + + // Iterate over all valid roles (each entry in the map can + // only have a single role). + for _, role := range node.Roles() { + if !newNode.HasRoles(role) { + // Skip unset roles. + continue + } + + maxNodes, exists := wcfg.MaxNodes[role] + if !exists { + // No such role found in whitelist. + ctx.Logger().Debug("RegisterNode: runtime's whitelist does not allow nodes with given role", + "role", role.String(), + "runtime_id", rt.ID, + "node_id", newNode.ID, + ) + return registry.ErrForbidden + } + if maxNodes == 0 { + // No nodes of this type are allowed. + ctx.Logger().Debug("RegisterNode: runtime's whitelist does not allow nodes with given role", + "role", role.String(), + "runtime_id", rt.ID, + ) + return registry.ErrForbidden + } + + if err := verifyNodeCountWithRoleForRuntime(ctx, state, newNode, rt, epoch, role, int(maxNodes)); err != nil { + return err + } + } + + return nil +} + +// verifyNodeCountWithRoleForRuntime verifies that the number of nodes registered by the specified +// entity for the specified runtime with the specified role is at most the specified maximum. +func verifyNodeCountWithRoleForRuntime( + ctx *api.Context, + state *registryState.MutableState, + newNode *node.Node, + rt *registry.Runtime, + epoch beacon.EpochTime, + role node.RolesMask, + maxNodes int, +) error { + // Count existing nodes owned by entity. + nodes, err := state.GetEntityNodes(ctx, newNode.EntityID) + if err != nil { + ctx.Logger().Error("RegisterNode: failed to query entity nodes", + "err", err, + "entity_id", newNode.EntityID, + ) + return err + } + + var curNodes int + for _, n := range nodes { + if n.ID.Equal(newNode.ID) || n.IsExpired(uint64(epoch)) || !n.HasRuntime(rt.ID) { + // Skip existing node when re-registering. Also skip + // expired nodes and nodes that haven't registered + // for the same runtime. + continue + } + + if n.HasRoles(role) { + curNodes++ + } + + // The check is inside the for loop, so we can stop as + // soon as possible once we're over the limit. + if curNodes+1 > maxNodes { + // Too many nodes with given role already registered. + ctx.Logger().Debug("RegisterNode: too many nodes with given role already registered for runtime", + "role", role.String(), + "runtime_id", rt.ID, + "node_id", newNode.ID, + "num_registered_nodes", curNodes, + ) + return registry.ErrForbidden + } + } + + return nil +} + +func verifyRuntimePerRoleAdmissionPolicy( + ctx *api.Context, + state *registryState.MutableState, + newNode *node.Node, + rt *registry.Runtime, + epoch beacon.EpochTime, +) error { + if len(rt.AdmissionPolicy.PerRole) == 0 { + return nil // No per-role policy specified. + } + + // Iterate over all valid roles (each entry in the map can only have a single role). + for _, role := range node.Roles() { + if !newNode.HasRoles(role) { + // Skip unset roles. + continue + } + + rolePolicy, ok := rt.AdmissionPolicy.PerRole[role] + if !ok { + // Skip roles for which a per-role policy is not set. + continue + } + + for _, policyFn := range perRoleAdmissionPolicyFns { + if err := policyFn(ctx, state, newNode, rt, epoch, role, &rolePolicy); err != nil { + return err + } + } + } + + return nil +} + +func verifyRuntimePerRoleWhitelistAdmissionPolicy( + ctx *api.Context, + state *registryState.MutableState, + newNode *node.Node, + rt *registry.Runtime, + epoch beacon.EpochTime, + role node.RolesMask, + rolePolicy *registry.PerRoleAdmissionPolicy, +) error { + if rolePolicy.EntityWhitelist == nil { + return nil // No per-role whitelist specified. + } + + wcfg, entIsWhitelisted := rolePolicy.EntityWhitelist.Entities[newNode.EntityID] + if !entIsWhitelisted { + ctx.Logger().Debug("RegisterNode: node's entity not in a runtime's per-role whitelist", + "entity_id", newNode.EntityID, + "runtime_id", rt.ID, + "node_id", newNode.ID, + "node_roles", newNode.Roles, + ) + return registry.ErrForbidden + } + if wcfg.MaxNodes == 0 { + return nil // Any amount of nodes allowed. + } + + return verifyNodeCountWithRoleForRuntime(ctx, state, newNode, rt, epoch, role, int(wcfg.MaxNodes)) +} diff --git a/go/consensus/cometbft/apps/registry/transactions.go b/go/consensus/cometbft/apps/registry/transactions.go index a0999f86db0..eaead30dc6e 100644 --- a/go/consensus/cometbft/apps/registry/transactions.go +++ b/go/consensus/cometbft/apps/registry/transactions.go @@ -235,89 +235,10 @@ func (app *registryApplication) registerNode( // nolint: gocyclo } } - // Check runtime's whitelist. + // Verify admission policies for all node's runtimes are satisfied. for _, rt := range paidRuntimes { - if rt.AdmissionPolicy.EntityWhitelist == nil { - continue - } - wcfg, entIsWhitelisted := rt.AdmissionPolicy.EntityWhitelist.Entities[newNode.EntityID] - if !entIsWhitelisted { - ctx.Logger().Debug("RegisterNode: node's entity not in a runtime's whitelist", - "entity_id", newNode.EntityID, - "runtime_id", rt.ID, - "node_id", newNode.ID, - ) - return registry.ErrForbidden - } - if len(wcfg.MaxNodes) == 0 { - continue - } - - // Map is present and non-empty, check per-role restrictions - // on the maximum number of nodes per entity. - - // Iterate over all valid roles (each entry in the map can - // only have a single role). - for _, role := range node.Roles() { - if !newNode.HasRoles(role) { - // Skip unset roles. - continue - } - - maxNodes, exists := wcfg.MaxNodes[role] - if !exists { - // No such role found in whitelist. - ctx.Logger().Debug("RegisterNode: runtime's whitelist does not allow nodes with given role", - "role", role.String(), - "runtime_id", rt.ID, - "node_id", newNode.ID, - ) - return registry.ErrForbidden - } - if maxNodes == 0 { - // No nodes of this type are allowed. - ctx.Logger().Debug("RegisterNode: runtime's whitelist does not allow nodes with given role", - "role", role.String(), - "runtime_id", rt.ID, - ) - return registry.ErrForbidden - } - - // Count existing nodes owned by entity. - nodes, grr := state.GetEntityNodes(ctx, newNode.EntityID) - if grr != nil { - ctx.Logger().Error("RegisterNode: failed to query entity nodes", - "err", grr, - "entity_id", newNode.EntityID, - ) - return grr - } - var curNodes uint16 - for _, n := range nodes { - if n.ID.Equal(newNode.ID) || n.IsExpired(uint64(epoch)) || !n.HasRuntime(rt.ID) { - // Skip existing node when re-registering. Also skip - // expired nodes and nodes that haven't registered - // for the same runtime. - continue - } - - if n.HasRoles(role) { - curNodes++ - } - - // The check is inside the for loop, so we can stop as - // soon as possible once we're over the limit. - if curNodes+1 > maxNodes { - // Too many nodes with given role already registered. - ctx.Logger().Error("RegisterNode: too many nodes with given role already registered for runtime", - "role", role.String(), - "runtime_id", rt.ID, - "node_id", newNode.ID, - "num_registered_nodes", curNodes, - ) - return registry.ErrForbidden - } - } + if err := verifyRuntimeAdmissionPolicy(ctx, state, newNode, rt, epoch); err != nil { + return err } } diff --git a/go/consensus/cometbft/apps/registry/transactions_test.go b/go/consensus/cometbft/apps/registry/transactions_test.go index 4371487c248..e15bcf809ff 100644 --- a/go/consensus/cometbft/apps/registry/transactions_test.go +++ b/go/consensus/cometbft/apps/registry/transactions_test.go @@ -303,6 +303,167 @@ func TestRegisterNode(t *testing.T) { false, false, }, + // Compute node on whitelist. + { + "ComputeNodeOnWhitelist", + func(tcd *testCaseData) { + rt := registry.Runtime{ + Versioned: cbor.NewVersioned(registry.LatestRuntimeDescriptorVersion), + ID: common.NewTestNamespaceFromSeed([]byte("consensus/cometbft/apps/registry: runtime: ComputeNodeOnWhitelist"), 0), + Kind: registry.KindCompute, + GovernanceModel: registry.GovernanceEntity, + AdmissionPolicy: registry.RuntimeAdmissionPolicy{ + EntityWhitelist: ®istry.EntityWhitelistRuntimeAdmissionPolicy{ + Entities: map[signature.PublicKey]registry.EntityWhitelistConfig{ + tcd.node.EntityID: {}, + }, + }, + }, + } + _ = state.SetRuntime(ctx, &rt, false) + + tcd.node.AddRoles(node.RoleComputeWorker) + tcd.node.Runtimes = []*node.Runtime{ + {ID: rt.ID}, + } + }, + nil, + true, + true, + }, + // Compute node not on whitelist. + { + "ComputeNodeNotOnWhitelist", + func(tcd *testCaseData) { + // Generate a random entity ID. + sig := memorySigner.NewTestSigner("consensus/cometbft/apps/registry: random signer 1") + + rt := registry.Runtime{ + Versioned: cbor.NewVersioned(registry.LatestRuntimeDescriptorVersion), + ID: common.NewTestNamespaceFromSeed([]byte("consensus/cometbft/apps/registry: runtime: ComputeNodeNotOnWhitelist"), 0), + Kind: registry.KindCompute, + GovernanceModel: registry.GovernanceEntity, + AdmissionPolicy: registry.RuntimeAdmissionPolicy{ + EntityWhitelist: ®istry.EntityWhitelistRuntimeAdmissionPolicy{ + Entities: map[signature.PublicKey]registry.EntityWhitelistConfig{ + sig.Public(): {}, + }, + }, + }, + } + _ = state.SetRuntime(ctx, &rt, false) + + tcd.node.AddRoles(node.RoleComputeWorker) + tcd.node.Runtimes = []*node.Runtime{ + {ID: rt.ID}, + } + }, + nil, + false, + false, + }, + // Observer node on per-role whitelist. + { + "ObserverNodeOnPerRoleWhitelist", + func(tcd *testCaseData) { + rt := registry.Runtime{ + Versioned: cbor.NewVersioned(registry.LatestRuntimeDescriptorVersion), + ID: common.NewTestNamespaceFromSeed([]byte("consensus/cometbft/apps/registry: runtime: ObserverNodeOnPerRoleWhitelist"), 0), + Kind: registry.KindCompute, + GovernanceModel: registry.GovernanceEntity, + AdmissionPolicy: registry.RuntimeAdmissionPolicy{ + PerRole: map[node.RolesMask]registry.PerRoleAdmissionPolicy{ + node.RoleObserver: { + EntityWhitelist: ®istry.EntityWhitelistRoleAdmissionPolicy{ + Entities: map[signature.PublicKey]registry.EntityWhitelistRoleConfig{ + tcd.node.EntityID: {}, + }, + }, + }, + }, + }, + } + _ = state.SetRuntime(ctx, &rt, false) + + tcd.node.AddRoles(node.RoleObserver) + tcd.node.Runtimes = []*node.Runtime{ + {ID: rt.ID}, + } + }, + nil, + true, + true, + }, + // Observer node not on per-role whitelist. + { + "ObserverNodeNotOnPerRoleWhitelist", + func(tcd *testCaseData) { + // Generate a random entity ID. + sig := memorySigner.NewTestSigner("consensus/cometbft/apps/registry: random signer 1") + + rt := registry.Runtime{ + Versioned: cbor.NewVersioned(registry.LatestRuntimeDescriptorVersion), + ID: common.NewTestNamespaceFromSeed([]byte("consensus/cometbft/apps/registry: runtime: ObserverNodeNotOnPerRoleWhitelist"), 0), + Kind: registry.KindCompute, + GovernanceModel: registry.GovernanceEntity, + AdmissionPolicy: registry.RuntimeAdmissionPolicy{ + PerRole: map[node.RolesMask]registry.PerRoleAdmissionPolicy{ + node.RoleObserver: { + EntityWhitelist: ®istry.EntityWhitelistRoleAdmissionPolicy{ + Entities: map[signature.PublicKey]registry.EntityWhitelistRoleConfig{ + sig.Public(): {}, + }, + }, + }, + }, + }, + } + _ = state.SetRuntime(ctx, &rt, false) + + tcd.node.AddRoles(node.RoleObserver) + tcd.node.Runtimes = []*node.Runtime{ + {ID: rt.ID}, + } + }, + nil, + false, + false, + }, + // Compute node not on per-role whitelist, but per-role whitelist only set for observer nodes. + { + "ComputeNodeNotOnPerRoleWhitelist", + func(tcd *testCaseData) { + // Generate a random entity ID. + sig := memorySigner.NewTestSigner("consensus/cometbft/apps/registry: random signer 1") + + rt := registry.Runtime{ + Versioned: cbor.NewVersioned(registry.LatestRuntimeDescriptorVersion), + ID: common.NewTestNamespaceFromSeed([]byte("consensus/cometbft/apps/registry: runtime: ComputeNodeNotOnPerRoleWhitelist"), 0), + Kind: registry.KindCompute, + GovernanceModel: registry.GovernanceEntity, + AdmissionPolicy: registry.RuntimeAdmissionPolicy{ + PerRole: map[node.RolesMask]registry.PerRoleAdmissionPolicy{ + node.RoleObserver: { // Note: The node will register with compute role. + EntityWhitelist: ®istry.EntityWhitelistRoleAdmissionPolicy{ + Entities: map[signature.PublicKey]registry.EntityWhitelistRoleConfig{ + sig.Public(): {}, + }, + }, + }, + }, + }, + } + _ = state.SetRuntime(ctx, &rt, false) + + tcd.node.AddRoles(node.RoleComputeWorker) + tcd.node.Runtimes = []*node.Runtime{ + {ID: rt.ID}, + } + }, + nil, + true, + true, + }, // Updating a node should be allowed. { "UpdateValidator", diff --git a/go/registry/api/api.go b/go/registry/api/api.go index 370baed6b7d..e00aeab3527 100644 --- a/go/registry/api/api.go +++ b/go/registry/api/api.go @@ -1180,6 +1180,30 @@ func VerifyRuntime( // nolint: gocyclo } } } + // Ensure valid per-role policy if present. + if perRole := rt.AdmissionPolicy.PerRole; perRole != nil { + for role, cfg := range perRole { + if !role.IsSingleRole() { + logger.Debug("RegisterRuntime: non-single role in per-role admission policy", + "role", role, + ) + return fmt.Errorf("%w: non-single role in per-role admission policy", ErrInvalidArgument) + } + + // Ensure valid whitelist if present. + if cfg.EntityWhitelist != nil { + for ent := range cfg.EntityWhitelist.Entities { + // Entity ID should be valid. + if !ent.IsValid() { + logger.Debug("RegisterRuntime: invalid entity ID in per-role whitelist", + "entity_id", ent, + ) + return fmt.Errorf("%w: invalid entity ID in per-role entity whitelist", ErrInvalidArgument) + } + } + } + } + } return nil } diff --git a/go/registry/api/runtime.go b/go/registry/api/runtime.go index 6e5672abd5f..02528078528 100644 --- a/go/registry/api/runtime.go +++ b/go/registry/api/runtime.go @@ -225,6 +225,24 @@ type EntityWhitelistConfig struct { type RuntimeAdmissionPolicy struct { AnyNode *AnyNodeRuntimeAdmissionPolicy `json:"any_node,omitempty"` EntityWhitelist *EntityWhitelistRuntimeAdmissionPolicy `json:"entity_whitelist,omitempty"` + + // PerRole is a per-role admission policy that must be satisfied in addition to the global + // admission policy for a specific role. + PerRole map[node.RolesMask]PerRoleAdmissionPolicy `json:"per_role,omitempty"` +} + +// PerRoleAdmissionPolicy is a per-role admission policy. +type PerRoleAdmissionPolicy struct { + EntityWhitelist *EntityWhitelistRoleAdmissionPolicy `json:"entity_whitelist,omitempty"` +} + +// EntityWhitelistRoleAdmissionPolicy is a per-role entity whitelist policy. +type EntityWhitelistRoleAdmissionPolicy struct { + Entities map[signature.PublicKey]EntityWhitelistRoleConfig `json:"entities"` +} + +type EntityWhitelistRoleConfig struct { + MaxNodes uint16 `json:"max_nodes,omitempty"` } // SchedulingConstraints are the node scheduling constraints.