Refactored names

app/cli/cliapp.py

@@ -1,13 +1,13 @@
 import inspect
 import os
-import pkgutil
 import sys
+import pkgutil
 import traceback
 from typing import List
 
 import click
 
-from common.utils import get_logger, setup_log, Singleton, path_extend
+from clap.utils import get_logger, setup_log, Singleton, path_extend
 
 if 'CLAP_PATH' not in os.environ:
     os.environ['CLAP_PATH'] = path_extend('~', '.clap')
@@ -17,13 +17,17 @@ class Defaults(metaclass=Singleton):
     def __init__(self):
         self.verbosity: int = 0
         self.clap_path: str = path_extend(os.environ.get('CLAP_PATH'))
-        self.configs_path: str = path_extend(os.environ.get('CLAP_PATH'), 'configs')
-        self.private_path: str = path_extend(os.environ.get('CLAP_PATH'), 'private')
-        self.storage_path: str = path_extend(os.environ.get('CLAP_PATH'), 'storage')
+        self.configs_path: str = path_extend(
+            os.environ.get('CLAP_PATH'), 'configs')
+        self.private_path: str = path_extend(
+            os.environ.get('CLAP_PATH'), 'private')
+        self.storage_path: str = path_extend(
+            os.environ.get('CLAP_PATH'), 'storage')
 
 
 class clap_command(click.Group):
     def __init__(self, group):
+        super().__init__()
         self.__dict__ = group.__dict__.copy()
 
 
@@ -32,6 +36,11 @@ def __init__(self, group):
 
 
 def find_commands(paths: List[str] = None) -> List[clap_command]:
+    """
+
+    :param paths:
+    :return:
+    """
     paths = paths or [os.path.join(os.path.dirname(__file__), 'modules')]
     modules = []
     for importer, package_name, _ in pkgutil.iter_modules(paths):
@@ -40,9 +49,8 @@ def find_commands(paths: List[str] = None) -> List[clap_command]:
             continue
         try:
             mod = importer.find_module(package_name).load_module(package_name)
-            # TODO Remove this!!
             modules += [obj for _, obj in inspect.getmembers(
-                mod, predicate=lambda x: 'clap_command' in str(type(x)))]
+                mod, predicate=lambda x: type(x) is clap_command)]
         except Exception as e:
             print(f'Error importing module with name `{package_name}`')
             raise e
@@ -79,4 +87,4 @@ def main(args):
 
 if __name__ == '__main__':
     import sys
-    main(sys.argv)
+    main(sys.argv)

app/cli/modules/cluster.py

@@ -7,13 +7,12 @@
 from dataclasses import asdict
 from typing import List
 
-from common.cluster_manager import ClusterManager, ClusterConfigDatabase, ClusterRepositoryController
-from common.configs import ConfigurationDatabase
-from common.executor import ShellInvoker, SSHCommandExecutor, AnsiblePlaybookExecutor
-from common.node import NodeRepositoryController
-from common.node_manager import NodeManager
-from common.repository import RepositoryFactory
-from common.utils import path_extend, float_time_to_string, get_logger, \
+from clap.cluster_manager import ClusterManager, ClusterConfigDatabase, ClusterRepositoryController
+from clap.configs import ConfigurationDatabase
+from clap.executor import ShellInvoker, SSHCommandExecutor, AnsiblePlaybookExecutor
+from clap.node_manager import NodeManager, NodeRepositoryController
+from clap.repository import RepositoryFactory
+from clap.utils import path_extend, float_time_to_string, get_logger, \
     Singleton, defaultdict_to_dict
 from providers.provider_ansible_aws import AnsibleAWSProvider
 from app.cli.cliapp import clap_command, Defaults

app/cli/modules/node.py

@@ -5,12 +5,11 @@
 from collections import defaultdict
 from dataclasses import asdict
 from typing import List
-from common.configs import ConfigurationDatabase
-from common.executor import ShellInvoker, SSHCommandExecutor, AnsiblePlaybookExecutor
-from common.node import NodeRepositoryController
-from common.node_manager import NodeManager
-from common.repository import RepositoryFactory
-from common.utils import path_extend, float_time_to_string, get_logger, \
+from clap.configs import ConfigurationDatabase
+from clap.executor import ShellInvoker, SSHCommandExecutor, AnsiblePlaybookExecutor
+from clap.node_manager import NodeManager, NodeRepositoryController
+from clap.repository import RepositoryFactory
+from clap.utils import path_extend, float_time_to_string, get_logger, \
     Singleton, defaultdict_to_dict
 from providers.provider_ansible_aws import AnsibleAWSProvider
 from app.cli.cliapp import clap_command, Defaults

app/cli/modules/role.py

@@ -2,10 +2,10 @@
 import yaml
 from collections import defaultdict
 from dataclasses import asdict
-from common.node import NodeRepositoryController
-from common.repository import RepositoryFactory
-from common.role_manager import RoleManager
-from common.utils import path_extend, get_logger, Singleton, defaultdict_to_dict
+from clap.node_manager import NodeRepositoryController
+from clap.repository import RepositoryFactory
+from clap.role_manager import RoleManager
+from clap.utils import path_extend, get_logger, Singleton, defaultdict_to_dict
 from app.cli.cliapp import clap_command, Defaults
 
 logger = get_logger(__name__)

common/abstract_provider.py → clap/abstract_provider.py

@@ -1,9 +1,9 @@
 from abc import abstractmethod, ABC
 from typing import List, Dict
 
-from common.node import NodeDescriptor
-from common.configs import InstanceInfo
-from common.utils import get_logger
+from clap.node import NodeDescriptor
+from clap.configs import InstanceInfo
+from clap.utils import get_logger
 
 logger = get_logger(__name__)
 

common/cluster_manager.py → clap/cluster_manager.py

@@ -7,12 +7,12 @@
 
 import dacite
 
-from common.configs import ConfigurationDatabase, InstanceInfo
-from common.executor import SSHCommandExecutor, AnsiblePlaybookExecutor
-from common.node_manager import NodeManager
-from common.repository import Repository, InvalidEntryError
-from common.role_manager import RoleManager
-from common.utils import yaml_load, get_logger, get_random_object
+from clap.configs import ConfigurationDatabase, InstanceInfo
+from clap.executor import SSHCommandExecutor, AnsiblePlaybookExecutor
+from clap.node_manager import NodeManager, NodeRepositoryController
+from clap.repository import Repository, InvalidEntryError
+from clap.role_manager import RoleManager
+from clap.utils import yaml_load, get_logger, get_random_object
 
 logger = get_logger(__name__)
 
@@ -557,61 +557,3 @@ def is_alive(self, cluster_id: str, retries: int = 5,
             nodes, retries=retries, wait_timeout=wait_timeout,
             update_timeout=update_timeout, max_workers=max_workers,
             test_command=test_command)
-
-
-if __name__ == '__main__':
-    import json
-    from dataclasses import asdict
-    from common.repository import RepositoryFactory
-    from common.utils import setup_log
-    from common.node import NodeRepositoryController
-    from providers.provider_ansible_aws import AnsibleAWSProvider
-
-    setup_log(verbosity_level=1)
-    node_repository_path = '/home/lopani/.clap/storage/nodes.db'
-    private_path = '/home/lopani/.clap/private'
-    cluster_file_path = '/home/lopani/.clap/configs/clusters/spits-cluster.yml'
-    cluster_repository_path = '/home/lopani/.clap/storage/clusters.db'
-
-    config_db = ConfigurationDatabase(
-        providers_file='/home/lopani/.clap/configs/providers.yaml',
-        logins_file='/home/lopani/.clap/configs/logins.yaml',
-        instances_file='/home/lopani/.clap/configs/instances.yaml'
-    )
-
-    repository = RepositoryFactory().get_repository('sqlite', node_repository_path)
-    repository_controller = NodeRepositoryController(repository)
-
-    ansible_aws = AnsibleAWSProvider(private_path)
-    node_manager = NodeManager(
-        repository_controller, {'aws': ansible_aws}, private_path)
-
-    role_manager = RoleManager(
-        repository_controller,
-        '/home/lopani/.clap/groups',
-        '/home/lopani/.clap/groups/actions.d',
-        '/home/lopani/.clap/private'
-    )
-
-    cluster_db = ClusterConfigDatabase([cluster_file_path])
-    print(cluster_db.clusters)
-
-    repository = RepositoryFactory().get_repository('sqlite', cluster_repository_path)
-    cluster_repository_controller = ClusterRepositoryController(repository)
-
-    cluster_manager = ClusterManager(
-        node_manager, role_manager, config_db,
-        cluster_repository_controller, private_path)
-
-    cluster_config = cluster_db.clusters['spits-cluster']
-    cluster = cluster_manager.start_cluster(cluster_config, max_workers=4)
-    # cluster = cluster_manager.get_all_clusters()[0]
-    print(f"CLUSTER: {cluster}")
-    all_cluster_node_types = cluster_manager.get_cluster_nodes_types(cluster.cluster_id)
-    print(f"All cluster node types: {all_cluster_node_types}")
-    all_cluster_nodes = cluster_manager.get_all_cluster_nodes(cluster.cluster_id)
-    print(f"All cluster nodes: {all_cluster_nodes}")
-    cluster_manager.setup_cluster(cluster.cluster_id, max_workers=1)
-    print('setup ok')
-    stopped_nodes = cluster_manager.stop_cluster(cluster.cluster_id)
-    print(f'Stopped nodes: {stopped_nodes}')

common/configs.py → clap/configs.py

@@ -3,7 +3,7 @@
 
 import dacite
 
-from common.utils import yaml_load, get_logger
+from clap.utils import yaml_load, get_logger
 
 logger = get_logger(__name__)
 
@@ -190,20 +190,3 @@ def load_all(self):
 
         logger.debug(f"Loaded {len(self.instance_descriptors)} instance descriptors: "
                      f"{', '.join(sorted(self.instance_descriptors.keys()))}")
-
-
-# if __name__ == '__main__':
-#     from dataclasses import asdict
-#     from common.utils import setup_log
-#     setup_log(verbosity_level=4)
-#
-#     c = ConfigurationDatabase(
-#         providers_file='/home/lopani/.clap/configs/providers.yaml',
-#         logins_file='/home/lopani/.clap/configs/logins.yaml',
-#         instances_file='/home/lopani/.clap/configs/instances.yaml'
-#     )
-#
-#     for iid, instance in c.instance_descriptors.items():
-#         a = asdict(instance)
-#         x = dacite.from_dict(InstanceInfo, a)
-#         print(x)

common/executor.py → clap/executor.py

@@ -12,8 +12,8 @@
 import paramiko
 import yaml
 
-from common.node import NodeDescriptor
-from common.utils import path_extend, get_logger, tmpdir, defaultdict_to_dict
+from clap.node import NodeDescriptor
+from clap.utils import path_extend, get_logger, tmpdir, defaultdict_to_dict
 
 logger = get_logger(__name__)
 
@@ -288,25 +288,3 @@ def run(self):
         except subprocess.CalledProcessError:
             logger.error(f"Invalid connection ip: {self.node.ip}. "
                          f"Check if `{self.node.node_id}` is alive first...")
-
-
-# if __name__ == '__main__':
-    # import os
-    # import common.config
-    # import common.node
-    # import common.repository
-    # import json
-    # config = common.config.Config()
-    # rpath = os.path.join(config.storage_path, 'nodes.db')
-    # repository = common.repository.SQLiteRepository(rpath)
-    # node_controller = common.node.NodeRepositoryController(repository)
-    # node = node_controller.get_nodes_by_id(['node-5'])[0]
-    # print(node)
-
-    # c = SSHCommandExecutor('ls -lha ~', [node], config.private_path)
-    # results = c.run()
-    # print(json.dumps(results, sort_keys=True, indent=4))
-
-    # p = AnsiblePlaybookExecutor('/home/lopani/.clap/groups/debug.yaml', config.private_path, hosts_node_map=[node])
-    # results = p.run()
-    # print(json.dumps(results, sort_keys=True, indent=4))

clap/node.py

@@ -0,0 +1,44 @@
+from dataclasses import dataclass, field
+
+from typing import List, Any, Dict, Optional
+
+from clap.configs import InstanceInfo
+from clap.utils import get_logger
+
+logger = get_logger(__name__)
+
+
+class NodeStatus:
+    UNKNOWN = 'unknown'
+    STARTED = 'started'
+    UNREACHABLE = 'unreachable'
+    REACHABLE = 'reachable'
+    PAUSED = 'paused'
+    STOPPED = 'stopped'
+
+
+class NodeType:
+    TYPE_CLOUD = 'cloud'
+    TYPE_LOCAL = 'local'
+
+
+class NodeLifecycle:
+    NORMAL = 'normal'
+    PREEMPTIBLE = 'preemptible'
+
+
+@dataclass
+class NodeDescriptor:
+    node_id: str
+    configuration: InstanceInfo
+    nickname: Optional[str] = ''
+    ip: Optional[str] = ''
+    type: Optional[str] = NodeType.TYPE_CLOUD
+    cloud_instance_id: Optional[str] = ''
+    cloud_lifecycle: Optional[str] = NodeLifecycle.NORMAL
+    status: Optional[str] = NodeStatus.UNKNOWN
+    creation_time: Optional[float] = 0.0
+    update_time: Optional[float] = 0.0
+    roles: List[str] = field(default_factory=list)
+    tags: Dict[str, str] = field(default_factory=dict)
+    extra: Dict[str, Any] = field(default_factory=dict)