From 4a377122005f6104c4e9d50de16da46ce14e6b14 Mon Sep 17 00:00:00 2001 From: Mahmoud Ismail Date: Thu, 9 Mar 2023 15:21:24 +0100 Subject: [PATCH] [CLOUD-86] Introduce an explicit allInOne parameter to identify single_node RonDB clusters (#243) --- CHANGELOG.md | 2 + hopsworksai/internal/api/apis_test.go | 3 + hopsworksai/internal/api/model.go | 5 +- hopsworksai/internal/api/model_test.go | 55 --------- hopsworksai/internal/structure/cluster.go | 2 +- .../internal/structure/cluster_test.go | 1 + hopsworksai/resource_cluster.go | 19 ++- hopsworksai/resource_cluster_test.go | 112 ++---------------- 8 files changed, 31 insertions(+), 168 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f1156f7..513211b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,7 @@ ## 1.4.0 (Unreleased) NOTES: +Old terraform providers would fail to create new single_node RonDB clusters, so you they need to update to this version. ENHANCEMENTS: * dependencies: Bump hashicorp/terraform-plugin-log from 0.7.0 to 0.8.0 @@ -8,6 +9,7 @@ ENHANCEMENTS: * dependencies: Bump hashicorp/terraform-plugin-docs from 0.13.0 to 0.14.1 * datasource/dataSourceAWSInstanceProfilePolicy: Add possibility to limit permissions to region, user ecr account and hopsworks.ai ecr account. * remove unusued AWS permissions from instance profile policy +* resource/hopsworksai_cluster: introduce an explicit allInOne parameter instead of detecting allInOne RonDB implicitly BUG FIXES: diff --git a/hopsworksai/internal/api/apis_test.go b/hopsworksai/internal/api/apis_test.go index 8b245e3..1c3ad6d 100644 --- a/hopsworksai/internal/api/apis_test.go +++ b/hopsworksai/internal/api/apis_test.go @@ -531,6 +531,7 @@ func TestNewClusterAWS(t *testing.T) { } ], "ronDB": { + "allInOne": false, "configuration": { "ndbdDefault": { "replicationFactor": 2 @@ -726,6 +727,7 @@ func TestNewClusterAZURE(t *testing.T) { } ], "ronDB": { + "allInOne": false, "configuration": { "ndbdDefault": { "replicationFactor": 2 @@ -2443,6 +2445,7 @@ func TestNewClusterAWS_HA(t *testing.T) { } ], "ronDB": { + "allInOne": false, "configuration": { "ndbdDefault": { "replicationFactor": 2 diff --git a/hopsworksai/internal/api/model.go b/hopsworksai/internal/api/model.go index 1eb22ab..506ff44 100644 --- a/hopsworksai/internal/api/model.go +++ b/hopsworksai/internal/api/model.go @@ -123,6 +123,7 @@ type RonDBNodeConfiguration struct { } type RonDBConfiguration struct { + AllInOne bool `json:"allInOne"` Configuration RonDBBaseConfiguration `json:"configuration"` ManagementNodes RonDBNodeConfiguration `json:"mgmd"` DataNodes RonDBNodeConfiguration `json:"ndbd"` @@ -130,10 +131,6 @@ type RonDBConfiguration struct { APINodes RonDBNodeConfiguration `json:"api"` } -func (rondb *RonDBConfiguration) IsSingleNodeSetup() bool { - return rondb.Configuration.NdbdDefault.ReplicationFactor == 1 && rondb.DataNodes.Count == 1 && rondb.MYSQLNodes.Count == 1 -} - type SpotConfiguration struct { MaxPrice int `json:"maxPrice"` FallBackOnDemand bool `json:"fallBackOnDemand"` diff --git a/hopsworksai/internal/api/model_test.go b/hopsworksai/internal/api/model_test.go index 4dbc1c8..559cc6e 100644 --- a/hopsworksai/internal/api/model_test.go +++ b/hopsworksai/internal/api/model_test.go @@ -410,58 +410,3 @@ func TestBackupStateString(t *testing.T) { } } } - -func TestIsSingleRonDBSetup(t *testing.T) { - input := &RonDBConfiguration{ - Configuration: RonDBBaseConfiguration{ - NdbdDefault: RonDBNdbdDefaultConfiguration{ - ReplicationFactor: 1, - }, - }, - ManagementNodes: RonDBNodeConfiguration{ - NodeConfiguration: NodeConfiguration{ - InstanceType: "mgm-node-1", - DiskSize: 30, - }, - Count: 1, - }, - DataNodes: RonDBNodeConfiguration{ - NodeConfiguration: NodeConfiguration{ - InstanceType: "data-node-1", - DiskSize: 512, - }, - Count: 1, - }, - MYSQLNodes: RonDBNodeConfiguration{ - NodeConfiguration: NodeConfiguration{ - InstanceType: "mysqld-node-1", - DiskSize: 100, - }, - Count: 1, - }, - } - - if !input.IsSingleNodeSetup() { - t.Fatalf("expected RonDB %#v to be single node setup but got false", *input) - } - - input.Configuration.NdbdDefault.ReplicationFactor = 2 - - if input.IsSingleNodeSetup() { - t.Fatalf("expected RonDB %#v to be a cluster setup but got false", *input) - } - - input.Configuration.NdbdDefault.ReplicationFactor = 1 - input.MYSQLNodes.Count = 2 - - if input.IsSingleNodeSetup() { - t.Fatalf("expected RonDB %#v to be a cluster setup but got false", *input) - } - - input.Configuration.NdbdDefault.ReplicationFactor = 2 - input.DataNodes.Count = 2 - - if input.IsSingleNodeSetup() { - t.Fatalf("expected RonDB %#v to be a cluster setup but got false", *input) - } -} diff --git a/hopsworksai/internal/structure/cluster.go b/hopsworksai/internal/structure/cluster.go index 112e3bd..98952ef 100644 --- a/hopsworksai/internal/structure/cluster.go +++ b/hopsworksai/internal/structure/cluster.go @@ -252,7 +252,7 @@ func flattenRonDB(ronDB *api.RonDBConfiguration) []map[string]interface{} { return nil } - if ronDB.IsSingleNodeSetup() { + if ronDB.AllInOne { singleNode := map[string]interface{}{ "instance_type": ronDB.DataNodes.InstanceType, diff --git a/hopsworksai/internal/structure/cluster_test.go b/hopsworksai/internal/structure/cluster_test.go index 7c06937..f6d6f70 100644 --- a/hopsworksai/internal/structure/cluster_test.go +++ b/hopsworksai/internal/structure/cluster_test.go @@ -1947,6 +1947,7 @@ func TestFlattenRonDBNodeConfiguration(t *testing.T) { func TestFlattenRonDB_single_node(t *testing.T) { input := &api.RonDBConfiguration{ + AllInOne: true, Configuration: api.RonDBBaseConfiguration{ NdbdDefault: api.RonDBNdbdDefaultConfiguration{ ReplicationFactor: 1, diff --git a/hopsworksai/resource_cluster.go b/hopsworksai/resource_cluster.go index f996fc1..c674af3 100644 --- a/hopsworksai/resource_cluster.go +++ b/hopsworksai/resource_cluster.go @@ -1323,6 +1323,7 @@ func createClusterBaseRequest(d *schema.ResourceData) (*api.CreateCluster, error defaultRonDB := defaultRonDBConfiguration() if singleRonDB, ok := d.GetOk("rondb.0.single_node"); ok { createCluster.RonDB = &api.RonDBConfiguration{ + AllInOne: true, Configuration: api.RonDBBaseConfiguration{ NdbdDefault: api.RonDBNdbdDefaultConfiguration{ ReplicationFactor: 1, @@ -1340,6 +1341,12 @@ func createClusterBaseRequest(d *schema.ResourceData) (*api.CreateCluster, error }, Count: 1, }, + APINodes: api.RonDBNodeConfiguration{ + NodeConfiguration: api.NodeConfiguration{ + DiskSize: defaultRonDB.APINodes.DiskSize, + }, + Count: 0, + }, DataNodes: api.RonDBNodeConfiguration{ NodeConfiguration: structure.ExpandNode(singleRonDB.([]interface{})[0].(map[string]interface{})), Count: 1, @@ -1357,6 +1364,7 @@ func createClusterBaseRequest(d *schema.ResourceData) (*api.CreateCluster, error } createCluster.RonDB = &api.RonDBConfiguration{ + AllInOne: false, Configuration: api.RonDBBaseConfiguration{ NdbdDefault: api.RonDBNdbdDefaultConfiguration{ ReplicationFactor: replicationFactor, @@ -1374,15 +1382,18 @@ func createClusterBaseRequest(d *schema.ResourceData) (*api.CreateCluster, error if n, ok := d.GetOk("rondb.0.api_nodes"); ok { createCluster.RonDB.APINodes = structure.ExpandRonDBNodeConfiguration(n.([]interface{})[0].(map[string]interface{})) + } else { + createCluster.RonDB.APINodes = api.RonDBNodeConfiguration{ + NodeConfiguration: api.NodeConfiguration{ + DiskSize: defaultRonDB.APINodes.DiskSize, + }, + Count: 0, + } } if createCluster.RonDB.DataNodes.Count%createCluster.RonDB.Configuration.NdbdDefault.ReplicationFactor != 0 { return nil, fmt.Errorf("number of RonDB data nodes must be multiples of RonDB replication factor") } - - if createCluster.RonDB.IsSingleNodeSetup() { - return nil, fmt.Errorf("your configuration creates a single rondb node, you should use singe_node configuration block instead or modifiy the configuration to run RonDB in a cluster mode") - } } } diff --git a/hopsworksai/resource_cluster_test.go b/hopsworksai/resource_cluster_test.go index c74934b..55d158f 100644 --- a/hopsworksai/resource_cluster_test.go +++ b/hopsworksai/resource_cluster_test.go @@ -2161,14 +2161,6 @@ func TestClusterCreate_RonDB_invalidReplicationFactor(t *testing.T) { testClusterCreate_RonDB_invalidReplicationFactor(t, api.AZURE) } -func TestClusterCreate_AWS_RonDB_cluster_with_single_node_config(t *testing.T) { - testClusterCreate_RonDB_cluster_with_single_node_config(t, api.AWS) -} - -func TestClusterCreate_AZURE_RonDB_cluster_with_single_node_config(t *testing.T) { - testClusterCreate_RonDB_cluster_with_single_node_config(t, api.AZURE) -} - func testClusterCreate_RonDB(t *testing.T, cloud api.CloudProvider) { state := map[string]interface{}{ "name": "cluster", @@ -2337,102 +2329,6 @@ func testClusterCreate_RonDB(t *testing.T, cloud api.CloudProvider) { r.Apply(t, context.TODO()) } -func testClusterCreate_RonDB_cluster_with_single_node_config(t *testing.T, cloud api.CloudProvider) { - state := map[string]interface{}{ - "name": "cluster", - "head": []interface{}{ - map[string]interface{}{ - "disk_size": 512, - }, - }, - "ssh_key": "my-key", - "workers": []interface{}{ - map[string]interface{}{ - "disk_size": 256, - "count": 2, - }, - }, - "rondb": []interface{}{ - map[string]interface{}{ - "configuration": []interface{}{ - map[string]interface{}{ - "ndbd_default": []interface{}{ - map[string]interface{}{ - "replication_factor": 1, - }, - }, - "general": []interface{}{ - map[string]interface{}{ - "benchmark": []interface{}{ - map[string]interface{}{ - "grant_user_privileges": false, - }, - }, - }, - }, - }, - }, - "management_nodes": []interface{}{ - map[string]interface{}{ - "instance_type": "mgm-node-1", - "disk_size": 30, - "count": 1, - }, - }, - "data_nodes": []interface{}{ - map[string]interface{}{ - "instance_type": "data-node-1", - "disk_size": 512, - "count": 1, - }, - }, - "mysql_nodes": []interface{}{ - map[string]interface{}{ - "instance_type": "mysqld-node-1", - "disk_size": 100, - "count": 1, - }, - }, - }, - }, - } - - if cloud == api.AWS { - state["aws_attributes"] = []interface{}{ - map[string]interface{}{ - "region": "region-1", - "bucket": []interface{}{ - map[string]interface{}{ - "name": "bucket-1", - }, - }, - "instance_profile_arn": "profile-1", - }, - } - } else if cloud == api.AZURE { - state["azure_attributes"] = []interface{}{ - map[string]interface{}{ - "location": "location-1", - "resource_group": "resource-group-1", - "user_assigned_managed_identity": "user-identity-1", - "container": []interface{}{ - map[string]interface{}{ - "storage_account": "storage-account-1", - }, - }, - }, - } - } - - r := test.ResourceFixture{ - Resource: clusterResource(), - OperationContextFunc: clusterResource().CreateContext, - State: state, - ExpectError: "your configuration creates a single rondb node, you should use singe_node configuration block instead or modifiy the configuration to run RonDB in a cluster mode", - } - r.Apply(t, context.TODO()) -} - func testGetRonDBConfig(reqBody io.Reader, cloud api.CloudProvider) (*api.RonDBConfiguration, error) { var output *api.RonDBConfiguration if cloud == api.AZURE { @@ -2524,6 +2420,7 @@ func testClusterCreate_RonDB_single_node(t *testing.T, cloud api.CloudProvider) } defaultRonDB := defaultRonDBConfiguration() expected := api.RonDBConfiguration{ + AllInOne: true, Configuration: api.RonDBBaseConfiguration{ NdbdDefault: api.RonDBNdbdDefaultConfiguration{ ReplicationFactor: 1, @@ -2541,6 +2438,12 @@ func testClusterCreate_RonDB_single_node(t *testing.T, cloud api.CloudProvider) }, Count: 1, }, + APINodes: api.RonDBNodeConfiguration{ + NodeConfiguration: api.NodeConfiguration{ + DiskSize: defaultRonDB.APINodes.DiskSize, + }, + Count: 0, + }, DataNodes: api.RonDBNodeConfiguration{ NodeConfiguration: api.NodeConfiguration{ InstanceType: "single-node-1", @@ -5740,6 +5643,7 @@ func TestClusterUpdate_modifyInstancetype_rondb_single_node(t *testing.T) { "provider": "AZURE", "version": "v1", "ronDB": { + "allInOne": true, "configuration": { "ndbdDefault": { "replicationFactor": 1