adding odl-pipeline

Change-Id: I1c08883f0d68a61ce9e10c5596aec1a259eed71f
Signed-off-by: Nikolas Hermanns <[email protected]>

.gitignore

@@ -1,5 +1,15 @@
+*.project
+*.pydevproject
+*.pyc
 *~
 .*.sw?
 /docs_build/
 /docs_output/
 /releng/
+*.tar.gz
+*.qcow2
+*.img
+odl-pipeline/build/*
+odl-pipeline/trash/*
+odl-pipeline/lib/tmp
+odl-pipeline/disks/*

odl-pipeline/lib/deployment_cloner.sh

@@ -0,0 +1,6 @@
+#!/bin/bash
+DIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )
+set -e
+export PYTHONPATH=$PYTHONPATH:$DIR
+mkdir -p $DIR/tmp
+python $DIR/deployment_cloner/deployment_cloner.py $@

odl-pipeline/lib/deployment_cloner/deployment_cloner.py

@@ -0,0 +1,65 @@
+#!/bin/python
+from utils import utils_yaml
+from utils.utils_log import for_all_methods, log_enter_exit
+from utils.service import Service
+from utils.node_manager import NodeManager
+from utils.processutils import execute
+
+
+@for_all_methods(log_enter_exit)
+class DeploymentCloner(Service):
+
+    undercloud_root_dir = '~/DeploymentCloner/'
+
+    def create_cli_parser(self, parser):
+        parser.add_argument('--undercloud-ip', help="ip of undercloud",
+                            required=True)
+        parser.add_argument('--dest-dir', help="where everything should go to",
+                            required=True)
+        return parser
+
+    def undercloud_dict(self, undercloud_ip):
+        return {'address': undercloud_ip,
+                'user': 'stack'}
+
+    def run(self, sys_args, config):
+        dest_dir = sys_args.dest_dir if sys_args.dest_dir[:-1] == '/'\
+            else (sys_args.dest_dir + '/')
+        self.node_manager = NodeManager()
+        underlcloud = self.node_manager.add_node(
+            'undercloud',
+            self.undercloud_dict(sys_args.undercloud_ip))
+        # copy all files to undercloud
+        underlcloud.copy('to', '.', self.undercloud_root_dir)
+        # generate the undercloud yaml
+        underlcloud.execute('cd %s; ./tripleo_manager.sh --out ./cloner-info/'
+                            % self.undercloud_root_dir, log_true=True)
+        underlcloud.copy('from', dest_dir,
+                         self.undercloud_root_dir + '/cloner-info/')
+        node_yaml_path = dest_dir + '/cloner-info/node.yaml'
+        node_yaml = utils_yaml.read_dict_from_yaml(node_yaml_path)
+        for name, node in node_yaml['servers'].iteritems():
+            node['vNode-name'] = self.get_virtual_node_name_from_mac(
+                node['orig-ctl-mac'])
+        utils_yaml.write_dict_to_yaml(node_yaml, node_yaml_path)
+        # TODO copy qcow and tar it
+
+    def get_virtual_node_name_from_mac(self, mac):
+        vNode_names, _ = execute('virsh list|awk \'{print $2}\'', shell=True)
+        for node in vNode_names.split('\n'):
+            if 'baremetal' in node:
+                admin_net_mac, _ = execute(
+                    'virsh domiflist %s |grep admin |awk \'{print $5}\''
+                    % node, shell=True)
+                if admin_net_mac.replace('\n', '') == mac:
+                    return node
+        raise Exception('Could not find corresponding virtual node for MAC: %s'
+                        % mac)
+
+
+def main():
+    main = DeploymentCloner()
+    main.start()
+
+if __name__ == '__main__':
+    main()

odl-pipeline/lib/flash-all-bridges.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+export bridges="admin|private|public|storage"
+for br in $(ifconfig |grep -v br-external |grep "^br" |grep -E $bridges |awk '{print $1}');do
+  sudo ip addr flush dev $br;
+done

odl-pipeline/lib/odl_reinstaller.sh

@@ -0,0 +1,6 @@
+#!/bin/bash
+DIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )
+set -e
+export PYTHONPATH=$PYTHONPATH:$DIR
+mkdir -p $DIR/tmp
+python $DIR/odl_reinstaller/odl_reinstaller.py $@

odl-pipeline/lib/odl_reinstaller/install_odl.pp

@@ -0,0 +1,15 @@
+include ::tripleo::packages
+
+if count(hiera('ntp::servers')) > 0 {
+  include ::ntp
+}
+
+class {"opendaylight":
+  extra_features => any2array(hiera('opendaylight::extra_features', 'odl-netvirt-openstack')),
+  odl_rest_port  => hiera('opendaylight::odl_rest_port'),
+  enable_l3      => hiera('opendaylight::enable_l3', 'no'),
+  #tarball_url    =>  'file:///home/heat-admin/distribution-karaf-0.6.0-SNAPSHOT.tar.gz',
+  #unitfile_url   =>  'file:///home/heat-admin/opendaylight-unitfile.tar.gz' 
+}
+
+

odl-pipeline/lib/odl_reinstaller/odl_reinstaller.py

@@ -0,0 +1,86 @@
+#!/bin/python
+import os
+from utils.utils_log import LOG, for_all_methods, log_enter_exit
+from utils.service import Service
+from utils.node_manager import NodeManager
+from utils.ssh_util import SSH_CONFIG
+
+
+@for_all_methods(log_enter_exit)
+class ODLReInstaller(Service):
+
+    def run(self, sys_args, config):
+        SSH_CONFIG['ID_RSA_PATH'] = sys_args.id_rsa
+        # copy ODL to all nodes where it need to be copied
+        self.nodes = NodeManager(config['servers']).get_nodes()
+        for node in self.nodes:
+            LOG.info('Disconnecting OpenVSwitch from controller on node %s'
+                     % node.name)
+            node.execute('ovs-vsctl del-controller br-int', as_root=True)
+
+        for node in self.nodes:
+            if 'ODL' in node.config:
+                tar_tmp_path = '/tmp/odl-artifact/'
+                if node.config['ODL'].get('active'):
+                    tarball_name = os.path.basename(sys_args.odl_artifact)
+                    node.copy('to', sys_args.odl_artifact,
+                              '/tmp/odl-artifact/' + tarball_name)
+                    node.execute('rm -rf /opt/opendaylight/*', as_root=True)
+                    node.execute('mkdir -p /opt/opendaylight/*', as_root=True)
+                    LOG.info('Extracting %s to /opt/opendaylight/ on node %s'
+                             % (tarball_name, node.name))
+                    node.execute('tar -zxf %s --strip-components=1 -C '
+                                 '/opt/opendaylight/'
+                                 % (tar_tmp_path + tarball_name), as_root=True)
+                    node.execute('chown -R odl:odl /opt/opendaylight',
+                                 as_root=True)
+                    node.execute('rm -rf ' + tar_tmp_path, as_root=True)
+                    LOG.info('Installing and Starting Opendaylight on node %s'
+                             % node.name)
+                    node.copy('to', 'odl_reinstaller/install_odl.pp',
+                              tar_tmp_path)
+                    node.execute('puppet apply --modulepath='
+                                 '/etc/puppet/modules/ %sinstall_odl.pp '
+                                 '--verbose --debug --trace '
+                                 '--detailed-exitcodes'
+                                 % tar_tmp_path, check_exit_code=[2],
+                                 as_root=True)
+        # --detailed-exitcodes: Provide extra information about the run via
+        # exit codes. If enabled, 'puppet apply' will use the following exit
+        # codes:
+        # 0: The run succeeded with no changes or failures; the system was
+        #    already in the desired state.
+        # 1: The run failed.
+        # 2: The run succeeded, and some resources were changed.
+        # 4: The run succeeded, and some resources failed.
+        # 6: The run succeeded, and included both changes and failures.
+
+        for node in self.nodes:
+            LOG.info('Connecting OpenVSwitch to controller on node %s'
+                     % node.name)
+            ovs_controller = node.config.get('ovs-controller')
+            if ovs_controller:
+                node.execute('ovs-vsctl set-controller br-int %s'
+                             % ovs_controller, as_root=True)
+
+    def create_cli_parser(self, parser):
+        parser.add_argument('-c', '--config',
+                            help=("Give the path to the node config file "
+                                  "(node.yaml)"),
+                            required=True)
+        parser.add_argument('--odl-artifact',
+                            help=("Path to Opendaylight tarball"),
+                            required=True)
+        parser.add_argument('--id-rsa',
+                            help=("Path to the identity file which can "
+                                  "be used to connect to the overcloud"),
+                            required=True)
+        return parser
+
+
+def main():
+    main = ODLReInstaller()
+    main.start()
+
+if __name__ == '__main__':
+    main()

odl-pipeline/lib/setup_jenkins_networks.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+set -e
+cd "$( dirname "${BASH_SOURCE[0]}" )"
+sudo ifdown enp0s4 2&>1 >> /dev/null /dev/null || true
+sudo ifdown enp0s6 2&>1 >> /dev/null /dev/null || true
+sudo cp ../templates/ifcfg-* /etc/network/interfaces.d/
+sudo ifup enp0s4
+sudo ifup enp0s6

odl-pipeline/lib/test_environment.sh

@@ -0,0 +1,6 @@
+#!/bin/bash
+DIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )
+set -e
+export PYTHONPATH=$PYTHONPATH:$DIR
+mkdir -p $DIR/tmp
+python $DIR/test_environment/test_environment.py $@

odl-pipeline/lib/test_environment/test_environment.py

@@ -0,0 +1,131 @@
+#!/bin/python
+import os
+from utils.utils_log import LOG, for_all_methods, log_enter_exit
+from utils.service import Service
+from utils.processutils import execute
+from utils import utils_yaml
+from utils.shutil import shutil
+MAX_NODES = 5
+
+
+@for_all_methods(log_enter_exit)
+class TestEnvironment(Service):
+
+    NODE_NAME = 'baremetal'
+    TEMPLATES = '../templates'
+    BRIGES = ['admin', 'private', 'public', 'storage']
+
+    def run(self, sys_args, config):
+        self.BUILD_DIR = '../build/apex-%s' % sys_args.env_number
+        self.env = sys_args.env_number
+        self.cleanup()
+        if sys_args.cleanup:
+            return
+        if not sys_args.cloner_info or not sys_args.snapshot_disks:
+            LOG.error('--cloner-info, --snapshot-disks have to be given if not'
+                      ' only cleanup.')
+            exit(1)
+        node_info = utils_yaml.read_dict_from_yaml(sys_args.cloner_info +
+                                                   '/node.yaml')
+        nodes = node_info['servers']
+        number_of_nodes = len(nodes)
+        disk_home = self.BUILD_DIR + '/disks/'
+        shutil.mkdir_if_not_exsist(disk_home)
+        # Create Snapshots
+        for i in range(number_of_nodes):
+            disk_name = '%s%s.qcow2' % (self.NODE_NAME, i)
+            self.create_snapshot('%s/%s' % (sys_args.snapshot_disks,
+                                            disk_name),
+                                 '%s/%s' % (disk_home, disk_name))
+
+        # Create Bridges if not existing
+        for net in self.BRIGES:
+            bridge_name = '%s-%s' % (net, self.env)
+            if not self.check_if_br_exists(bridge_name):
+                LOG.info('Creating bridge %s' % bridge_name)
+                execute('ovs-vsctl add-br %s' % bridge_name, as_root=True)
+
+        # Create virtual Nodes
+        dom_template = self.TEMPLATES + '/nodes/baremetalX.xml'
+        dom_config = self.BUILD_DIR + '/nodes/baremetalX.xml'
+        shutil.mkdir_if_not_exsist(self.BUILD_DIR + '/nodes/')
+        LOG.info('Creating virtual Nodes')
+        for name, node in nodes.iteritems():
+            orig_node_name = node['vNode-name']
+            node_name = orig_node_name + '-' + self.env
+            LOG.info('Create node %s' % node_name)
+            type = node['type']
+            if type == 'compute':
+                cpu = 4
+                mem = 4
+            elif type == 'controller':
+                cpu = 8
+                mem = 10
+            else:
+                raise Exception('Unknown node type! %s' % type)
+            shutil.copy('to', dom_template, dom_config)
+            shutil.replace_string_in_file(dom_config, 'NaMe', node_name)
+            disk_full_path = os.path.abspath('%s/%s.qcow2' % (disk_home,
+                                                              orig_node_name))
+            shutil.replace_string_in_file(dom_config, 'DiSk', disk_full_path)
+            shutil.replace_string_in_file(dom_config, 'vCpU', str(cpu))
+            shutil.replace_string_in_file(dom_config, 'MeMoRy', str(mem))
+            shutil.replace_string_in_file(dom_config, 'InDeX', self.env)
+
+            execute('virsh define ' + dom_config)
+            execute('virsh start ' + node_name)
+
+            cores_per_environment = 8
+            cores = '%s-%s' % (int(self.env) * 8, int(self.env) * 8 +
+                               cores_per_environment - 1)
+            LOG.info('Pining vCPU of node %s to cores %s' % (node_name, cores))
+            for i in range(cpu):
+                execute('virsh vcpupin %(node)s %(nodes_cpu)s %(host_cpu)s' %
+                        {'node': node_name,
+                         'nodes_cpu': i,
+                         'host_cpu': cores})
+
+    def check_if_br_exists(self, bridge):
+        _, (_, rc) = execute('ovs-vsctl br-exists %s' % bridge,
+                             check_exit_code=[0, 2], as_root=True)
+        return True if rc == 0 else False
+
+    def create_snapshot(self, orig, path):
+        LOG.info('Creating snapshot of %s in %s' % (orig, path))
+        execute('qemu-img create -f qcow2 -b %s %s' % (orig, path),
+                as_root=True)
+
+    def cleanup(self):
+        for i in range(MAX_NODES):
+            rv, (_, rc) = execute('virsh destroy %(name)s%(i)s-%(env)s'
+                                  % {'i': i, 'env': self.env,
+                                     'name': self.NODE_NAME},
+                                  check_exit_code=[0, 1])
+            if rc == 0:
+                LOG.info(rv)
+            rv, (_, rc) = execute('virsh undefine %(name)s%(i)s-%(env)s'
+                                  % {'i': i, 'env': self.env,
+                                     'name': self.NODE_NAME},
+                                  check_exit_code=[0, 1])
+            if rc == 0:
+                LOG.info(rv)
+        execute('rm -rf ' + self.BUILD_DIR)
+
+    def create_cli_parser(self, parser):
+        parser.add_argument('--env-number', help="Number of the environment",
+                            required=True)
+        parser.add_argument('--cloner-info', help="Path to the cloner-info",
+                            required=False)
+        parser.add_argument('--snapshot-disks', help="Path to the snapshots",
+                            required=False)
+        parser.add_argument('--cleanup', help="Only Cleanup",
+                            required=False, action='store_true')
+        return parser
+
+
+def main():
+    main = TestEnvironment()
+    main.start()
+
+if __name__ == '__main__':
+    main()

odl-pipeline/lib/tripleo_manager.sh

@@ -0,0 +1,9 @@
+#!/bin/bash
+DIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )
+if [ -e ~/stackrc ];then
+  . ~/stackrc
+fi
+set -e
+export PYTHONPATH=$PYTHONPATH:$DIR
+mkdir -p $DIR/tmp
+python $DIR/tripleo_manager/tripleo_manager.py $@