Initial commit

Author: Sylvain Lebresne

.gitignore

@@ -0,0 +1 @@
+*.pyc

README

@@ -0,0 +1,100 @@
+CCM (for Cassandra Cluster Manager)
+===================================
+
+A script to create, launch and remove a Apache Cassandra cluster on localhost.
+
+The goal of ccm is to make is easy to create, manage and destroy a small
+cluster on a local box. It is meant for quick testing on a Cassandra cluster.
+
+
+Install
+-------
+
+As far as I know, this uses only standard python modules, so as long as you
+have python installed, you should be good to go. Simply clone this repository
+where you want.
+
+Once cloned, you'll probably want to create a symbolic link to the ccm
+executable script somewhere in your path (The following examples assume that much).
+
+ccm is only for cluster on localhost so if you want more than one node, you
+will likely need multiple loopback interface aliases. On mac os x for
+instance, you can create such aliases with
+  sudo ifconfig lo0 alias 127.0.0.2 up
+  sudo ifconfig lo0 alias 127.0.0.3 up
+  ...
+
+I'll assume you have at least 127.0.0.1, 127.0.0.2 and 127.0.0.3 set up in
+the next section.
+
+
+Usage
+-----
+
+ccm works from a Cassandra source tree. So in the following example, I assume
+that 'ccm' is in the path and that current directory is a Cassandra source
+directory (either 0.7 or trunk, this doesn't work with 0.6, though that could
+be added easily enough if there is some interest). It also assumes that
+Cassandra has been compiled (with 'ant build').
+
+ccm work with the notion of a current cluster. To create a cluster and
+'switch' to it:
+> ccm create test
+
+Then add some node:
+> ccm add node1 -i 127.0.0.1 -j 7100 -s
+> ccm add node2 -i 127.0.0.2 -j 7200 -s
+
+This add 2 nodes on 127.0.0.1 and 127.0.0.2 using default thrift and storage
+port using jmx port 7100 and 7200 (JMX binds itself to all interfaces by
+default, so you want 2 separate ports here).
+Moreover, those are set as seeds ('-s' flag; you need at least one seed node).
+
+You can then start the whole cluster:
+> ccm start
+
+You can check that everything is working ok:
+> ccm node1 ring
+
+which simply call nodetool ring on node1.
+
+You can now bootstrap a new node and start it with:
+> ccm add node3 -i 127.0.0.3 -j 7300 -b
+> ccm node3 start
+
+This will wait for node3 to be fully bootstrapped, so this will take around 90
+seconds. You can use --no-wait to avoid this.
+
+ccm then provide a few conveniences, like flushing a full cluster:
+> ccm flush
+or a single node:
+> ccm node2 flush
+
+You can watch the log file of a given node with:
+> ccm node1 showlog
+(this exec 'less' on the log file)
+
+And you can remove the whole cluster with:
+> ccm remove
+
+There is a bunch of other commands (some of nodetool command are provided, just so that
+you don't have to remember the IP addresses and port number). Just try 'ccm'
+to get a list of available command. Then each command options are documented:
+for instance 'ccm add -h' describe the option for 'ccm add'.
+
+
+Where are things stored
+-----------------------
+
+By default, ccm store all the node data and configuration file under ~/.ccm/cluster_name/.
+This can be overriden using the --config-dir option with each command.
+
+
+Notes
+-----
+
+I use this script almost daily for quick Cassandra test, but this is *not*
+heavily tested, so you have been warned. I do welcome suggestion however.
+
+
+Sylvain Lebresne <[email protected]>

__init__.py

@@ -0,0 +1,74 @@
+#!/usr/bin/python
+
+import os, sys
+
+L = os.path.realpath(__file__).split(os.path.sep)[:-1]
+root = os.path.sep.join(L)
+sys.path.append(os.path.join(root, 'cmds'))
+import command, common
+from cluster_cmds import *
+from cluster_cass_cmds import *
+from node_cmds import *
+from node_cass_cmds import *
+
+def get_command(kind, cmd):
+    cmd_name = kind.lower().capitalize() + cmd.lower().capitalize() + "Cmd"
+    try:
+        klass = globals()[cmd_name]
+    except KeyError:
+        return None
+    if not issubclass(klass, command.Cmd):
+        return None
+    return klass()
+
+def print_global_usage():
+    print "Usage:"
+    print "  ccm <cluster_cmd> [options]"
+    print "  ccm <node_name> <node_cmd> [options]"
+    print ""
+    print "Where <node_name> is the name of a node of the current cluster, <cluster_cmd> is one of"
+    for cmd_name in cluster_cmds():
+        cmd = get_command("cluster", cmd_name)
+        if not cmd:
+            print "Internal error, unknown command {0}".format(cmd_name)
+            exit(1)
+        print "  {0:14} {1}".format(cmd_name, cmd.description())
+    print "and <node_cmd> is one of"
+    for cmd_name in node_cmds():
+        cmd = get_command("node", cmd_name)
+        if not cmd:
+            print "Internal error, unknown command {0}".format(cmd_name)
+            exit(1)
+        print "  {0:14} {1}".format(cmd_name, cmd.description())
+    exit(1)
+
+if len(sys.argv) <= 1:
+    print "Missing arguments"
+    print_global_usage()
+
+arg1 = sys.argv[1].lower()
+
+if arg1 in cluster_cmds():
+    kind = 'cluster'
+    cmd = arg1
+    cmd_args = sys.argv[2:]
+else:
+    if len(sys.argv) <= 2:
+        print "Missing arguments"
+        print_global_usage()
+    kind = 'node'
+    node = arg1
+    cmd = sys.argv[2]
+    cmd_args = [node] + sys.argv[3:]
+
+cmd = get_command(kind, cmd)
+if not cmd:
+    print "Unknown node or command: {0}".format(arg1)
+    exit(1)
+
+parser = cmd.get_parser()
+
+(options, args) = parser.parse_args(cmd_args)
+cmd.validate(parser, options, args)
+
+cmd.run()

ccm

@@ -0,0 +1,88 @@
+# ccm clusters
+
+import common, yaml, os
+from node import Node
+
+class Cluster():
+    def __init__(self, path, name):
+        self.name = name
+        self.nodes = {}
+        self.seeds = []
+        self.path = path
+
+    def save(self):
+        node_list = [ node.name for node in self.nodes.values() ]
+        seed_list = [ node.name for node in self.seeds ]
+        filename = os.path.join(self.path, self.name, 'cluster.conf')
+        with open(filename, 'w') as f:
+            yaml.dump({ 'name' : self.name, 'nodes' : node_list, 'seeds' : seed_list }, f)
+
+    @staticmethod
+    def load(path, name):
+        cluster_path = os.path.join(path, name)
+        filename = os.path.join(cluster_path, 'cluster.conf')
+        with open(filename, 'r') as f:
+            data = yaml.load(f)
+        try:
+            cluster = Cluster(path, data['name'])
+            node_list = data['nodes']
+            seed_list = data['seeds']
+        except KeyError as k:
+            raise common.LoadError("Error Loading " + filename + ", missing property:" + k)
+
+        for node_name in node_list:
+            cluster.nodes[node_name] = Node.load(cluster_path, node_name, cluster)
+        for seed_name in seed_list:
+            cluster.seeds.append(cluster.nodes[seed_name])
+        return cluster
+
+    def add(self, node, is_seed):
+        self.nodes[node.name] = node
+        if is_seed:
+            self.seeds.append(node)
+
+    def get_path(self):
+        return os.path.join(self.path, self.name)
+
+    def get_seeds(self):
+        return [ s.network_interfaces['storage'][0] for s in self.seeds ]
+
+    def show(self, verbose):
+        if len(self.nodes.values()) == 0:
+            print "No node in this cluster yet"
+            return
+        for node in self.nodes.values():
+            if (verbose):
+                node.show(show_cluster=False)
+                print ""
+            else:
+                node.show(only_status=True)
+
+    # update_pids() should be called after this
+    def start(self, cassandra_dir):
+        started = []
+        for node in self.nodes.values():
+            if not node.is_running():
+                p = node.start(cassandra_dir)
+                started.append((node, p))
+        return started
+
+    def update_pids(self, started):
+        for node, p in started:
+            try:
+                node.update_pid(p)
+            except StartError as e:
+                print str(e)
+
+    def stop(self):
+        not_running = []
+        for node in self.nodes.values():
+            if not node.stop():
+                not_running.append(node)
+        return not_running
+
+
+    def nodetool(self, cassandra_dir, nodetool_cmd):
+        for node in self.nodes.values():
+            if node.is_running():
+                node.nodetool(cassandra_dir, nodetool_cmd)

ccm_lib/__init__.py

@@ -0,0 +1,95 @@
+#
+# Cassandra Cluster Management lib
+#
+
+import os, common, shutil, re
+from cluster import Cluster
+from node import Node
+
+USER_HOME = os.path.expanduser('~')
+
+CASSANDRA_BIN_DIR= "bin"
+CASSANDRA_CONF_DIR= "conf"
+
+CASSANDRA_CONF = "cassandra.yaml"
+LOG4J_CONF = "log4j-server.properties"
+CASSANDRA_ENV = "cassandra-env.sh"
+CASSANDRA_SH = "cassandra.in.sh"
+
+class LoadError(Exception):
+    pass
+
+def get_default_path():
+    default_path = os.path.join(USER_HOME, '.ccm')
+    if not os.path.exists(default_path):
+        os.mkdir(default_path)
+    return default_path
+
+def parse_interface(itf, default_port):
+    i = itf.split(':')
+    if len(i) == 1:
+        return (i[0].strip(), default_port)
+    elif len(i) == 2:
+        return (i[0].strip(), int(i[1].strip()))
+    else:
+        raise ValueError("Invalid interface definition: " + itf)
+
+def current_cluster_name(path):
+    try:
+        with open(os.path.join(path, 'CURRENT'), 'r') as f:
+            return f.readline().strip()
+    except IOError:
+        return None
+
+def load_current_cluster(path):
+    name = current_cluster_name(path)
+    if name is None:
+        print 'No currently active cluster (use ccm cluster switch)'
+        exit(1)
+    try:
+        return Cluster.load(path, name)
+    except common.LoadError as e:
+        print str(e)
+        exit(1)
+
+# may raise OSError if dir exists
+def create_cluster(path, name):
+    dir_name = os.path.join(path, name)
+    os.mkdir(dir_name)
+    cluster = Cluster(path, name)
+    cluster.save()
+    return cluster
+
+def switch_cluster(path, new_name):
+    with open(os.path.join(path, 'CURRENT'), 'w') as f:
+        f.write(new_name + '\n')
+
+def replace_in_file(file, regexp, replace):
+    replaces_in_file(file, [(regexp, replace)])
+
+def replaces_in_file(file, replacement_list):
+    rs = [ (re.compile(regexp), repl) for (regexp, repl) in replacement_list]
+    file_tmp = file + ".tmp"
+    with open(file, 'r') as f:
+        with open(file_tmp, 'w') as f_tmp:
+            for line in f:
+                for r, replace in rs:
+                    match = r.search(line)
+                    if match:
+                        line = replace + "\n"
+                f_tmp.write(line)
+    shutil.move(file_tmp, file)
+
+def make_cassandra_env(cassandra_dir, node_path):
+    sh_file = os.path.join(CASSANDRA_BIN_DIR, CASSANDRA_SH)
+    orig = os.path.join(cassandra_dir, sh_file)
+    dst = os.path.join(node_path, sh_file)
+    shutil.copy(orig, dst)
+    replacements = [
+        ('CASSANDRA_HOME=', '\tCASSANDRA_HOME=%s' % cassandra_dir),
+        ('CASSANDRA_CONF=', '\tCASSANDRA_CONF=%s' % os.path.join(node_path, 'conf'))
+    ]
+    common.replaces_in_file(dst, replacements)
+    env = os.environ.copy()
+    env['CASSANDRA_INCLUDE'] = os.path.join(dst)
+    return env