opencontrail4-update.groovy

/**
 * Update pipeline for OpenContrail 4X versions
 *
 * Expected parameters:
 *   SALT_MASTER_CREDENTIALS        Credentials to the Salt API.
 *   SALT_MASTER_URL                Full Salt API address [http://10.10.10.1:8000].
 *   STAGE_CONTROLLERS_UPDATE       Run update on OpenContrail controller and analytic nodes (bool)
 *   STAGE_COMPUTES_UPDATE          Run update OpenContrail components on compute nodes (bool)
 *
 **/

common = new com.mirantis.mk.Common()
salt = new com.mirantis.mk.Salt()
python = new com.mirantis.mk.Python()

def pepperEnv = "pepperEnv"
def askConfirmation = (env.getProperty('ASK_CONFIRMATION') ?: true).toBoolean()
def supportedOcTargetVersions = ['4.0', '4.1']
def neutronServerPkgs = 'neutron-plugin-contrail,contrail-heat,python-contrail'
def config4Services = ['zookeeper', 'contrail-webui-middleware', 'contrail-webui', 'contrail-api', 'contrail-schema', 'contrail-svc-monitor', 'contrail-device-manager', 'contrail-config-nodemgr', 'contrail-database']
def dashboardPanelPkg = 'openstack-dashboard-contrail-panels'
def targetOcVersion

def cmpMinions
def cmpMinionsFirstSubset
def cmpMinionsSecondSubset
def cmpTargetAll
def cmpTargetFirstSubset
def cmpTargetSecondSubset

def checkContrailServices(pepperEnv, oc_version, target) {

    def checkCmd

    if (oc_version.startsWith('4')) {

        checkCmd = "doctrail all contrail-status | grep -v == | grep -v FOR | grep -v \\* | grep -v \'disabled on boot\' | grep -v nodemgr | grep -v active | grep -v backup | grep -v -F /var/crashes/"

        if (oc_version == '4.1') {
            def targetMinions = salt.getMinions(pepperEnv, target)
            def collectorMinionsInTarget = targetMinions.intersect(salt.getMinions(pepperEnv, 'I@opencontrail:collector'))

            if (collectorMinionsInTarget.size() != 0) {
                def cassandraConfigYaml = readYaml text: salt.getFileContent(pepperEnv, 'I@opencontrail:control:role:primary', '/etc/cassandra/cassandra.yaml')

                def currentCassandraNativeTransportPort = cassandraConfigYaml['native_transport_port'] ?: "9042"
                def currentCassandraRpcPort = cassandraConfigYaml['rpc_port'] ?: "9160"

                def cassandraNativeTransportPort = getValueForPillarKey(pepperEnv, "I@opencontrail:control:role:primary", "opencontrail:database:bind:port_configdb")
                def cassandraCassandraRpcPort = getValueForPillarKey(pepperEnv, "I@opencontrail:control:role:primary", "opencontrail:database:bind:rpc_port_configdb")

                if (currentCassandraNativeTransportPort != cassandraNativeTransportPort) {
                    checkCmd += ' | grep -v \'contrail-collector.*(Database:Cassandra connection down)\''
                }

                if (currentCassandraRpcPort != cassandraCassandraRpcPort) {
                    checkCmd += ' | grep -v \'contrail-alarm-gen.*(Database:Cassandra\\[\\] connection down)\''
                }
            }
        }

    } else {
        checkCmd = "contrail-status | grep -v == | grep -v FOR | grep -v \'disabled on boot\' | grep -v nodemgr | grep -v active | grep -v backup | grep -v -F /var/crashes/"
    }

    salt.commandStatus(pepperEnv, target, checkCmd, null, false, true, null, true, 500)
}

def getValueForPillarKey(pepperEnv, target, pillarKey) {
    def out = salt.getReturnValues(salt.getPillar(pepperEnv, target, pillarKey))
    if (out == '') {
        throw new Exception("Cannot get value for ${pillarKey} key on ${target} target")
    }
    return out.toString()
}

def cmpNodesUpdate(pepperEnv, target) {

    def cmpPkgs = 'contrail-lib contrail-nodemgr contrail-utils contrail-vrouter-agent contrail-vrouter-utils python-contrail python-contrail-vrouter-api python-opencontrail-vrouter-netns contrail-vrouter-dkms'
    def aptCmd = "export DEBIAN_FRONTEND=noninteractive; apt install -o Dpkg::Options::=\"--force-confold\" ${cmpPkgs} -y;"
    def kernelModuleReloadCmd = 'service contrail-vrouter-agent stop; service contrail-vrouter-nodemgr stop; rmmod vrouter; sync && echo 3 > /proc/sys/vm/drop_caches && echo 1 > /proc/sys/vm/compact_memory; service contrail-vrouter-agent start; service contrail-vrouter-nodemgr start'
    def out

    try {
        salt.runSaltProcessStep(pepperEnv, target, 'saltutil.refresh_pillar', [], null, true)
        salt.runSaltProcessStep(pepperEnv, target, 'saltutil.sync_all', [], null, true)
        salt.runSaltProcessStep(pepperEnv, target, 'file.remove', ["/etc/apt/sources.list.d/mcp_opencontrail.list"], null, true)
        salt.enforceState(pepperEnv, target, 'linux.system.repo')
    } catch (Exception er) {
        common.errorMsg("Opencontrail component on ${target} probably failed to be replaced. Please check availability of contrail packages before continuing.")
        throw er
    }

    out = salt.runSaltCommand(pepperEnv, 'local', ['expression': target, 'type': 'compound'], 'cmd.shell', null, aptCmd, null)
    salt.printSaltCommandResult(out)

    try {
        salt.enforceState(pepperEnv, target, 'opencontrail')
    } catch (Exception er) {
        common.errorMsg("Opencontrail state was executed on ${target} and failed please fix it manually.")
    }

    salt.runSaltProcessStep(pepperEnv, target, 'cmd.shell', [kernelModuleReloadCmd], null, true)
    salt.commandStatus(pepperEnv, target, 'contrail-status | grep -v == | grep -v active | grep -v -F /var/crashes/', null, false)
    out = salt.runSaltCommand(pepperEnv, 'local', ['expression': target, 'type': 'compound'], 'cmd.shell', null, "contrail-status", null)
    salt.printSaltCommandResult(out)
}

timeout(time: 12, unit: 'HOURS') {
    node() {

        stage('Setup virtualenv for Pepper') {
            python.setupPepperVirtualenv(pepperEnv, SALT_MASTER_URL, SALT_MASTER_CREDENTIALS)
        }

        if (STAGE_CONTROLLERS_UPDATE.toBoolean() == true) {

            stage('Sync Salt data') {

                // Sync data on minions
                salt.runSaltProcessStep(pepperEnv, 'I@keystone:server:role:primary or I@opencontrail:database or I@neutron:server or I@horizon:server', 'saltutil.refresh_pillar', [], null, true)
                salt.runSaltProcessStep(pepperEnv, 'I@keystone:server:role:primary or I@opencontrail:database or I@neutron:server or I@horizon:server', 'saltutil.sync_all', [], null, true)
            }

            stage('Verify OpenContrail version compatibility') {

                // Verify specified target OpenContrail version before update
                targetOcVersion = getValueForPillarKey(pepperEnv, "I@opencontrail:control:role:primary", "_param:opencontrail_version")
                if (!supportedOcTargetVersions.contains(targetOcVersion)) {
                    throw new Exception("Specified OpenContrail version ${targetOcVersion} is not supported by update pipeline. Supported versions: ${supportedOcTargetVersions}")
                }
            }

            stage('Opencontrail controllers health check') {
                try {
                    salt.enforceState(pepperEnv, 'I@opencontrail:control or I@opencontrail:collector', 'opencontrail.upgrade.verify', true, true)
                } catch (Exception er) {
                    common.errorMsg("OpenContrail controllers health check stage found issues with services. Please take a look at the logs above.")
                    throw er
                }
            }

            stage('Update system repositories') {
                try {
                    salt.runSaltProcessStep(pepperEnv, 'I@opencontrail:control or I@opencontrail:collector', 'file.remove', ["/etc/apt/sources.list.d/mcp_opencontrail.list"], null, true)
                    salt.enforceState(pepperEnv, 'I@opencontrail:control or I@opencontrail:collector or I@neutron:server or I@horizon:server', 'linux.system.repo')

                } catch (Exception er) {
                    common.errorMsg("System repositories failed to be updated on I@opencontrail:control, I@opencontrail:collector, I@neutron:server or I@horizon:server nodes.")
                    throw er
                }
            }

            stage('OpenContrail controllers update') {

                // Make sure that dedicated opencontrail user is created
                salt.enforceState(pepperEnv, 'I@keystone:server:role:primary', 'keystone.client.server')

                // Stop neutron-server to prevent creation of new objects in contrail
                salt.runSaltProcessStep(pepperEnv, 'I@neutron:server', 'service.stop', ['neutron-server'])

                // Backup Zookeeper data
                salt.enforceState(pepperEnv, 'I@zookeeper:backup:server', 'zookeeper.backup')
                salt.enforceState(pepperEnv, 'I@zookeeper:backup:client', 'zookeeper.backup')

                try {
                    salt.cmdRun(pepperEnv, 'I@opencontrail:control', "su root -c '/usr/local/bin/zookeeper-backup-runner.sh'")
                } catch (Exception er) {
                    common.errorMsg('Zookeeper failed to backup. Please fix it before continuing.')
                    throw er
                }

                // Backup Cassandra DB
                salt.enforceState(pepperEnv, 'I@cassandra:backup:server', 'cassandra.backup')
                salt.enforceState(pepperEnv, 'I@cassandra:backup:client', 'cassandra.backup')

                try {
                    salt.cmdRun(pepperEnv, 'I@cassandra:backup:client', "su root -c '/usr/local/bin/cassandra-backup-runner-call.sh'")
                } catch (Exception er) {
                    common.errorMsg('Cassandra failed to backup. Please fix it before continuing.')
                    throw er
                }

                try {
                    // Get docker images info
                    controllerImage = getValueForPillarKey(pepperEnv, "I@opencontrail:control:role:primary", "docker:client:compose:opencontrail:service:controller:image")
                    analyticsImage = getValueForPillarKey(pepperEnv, "I@opencontrail:collector:role:primary", "docker:client:compose:opencontrail:service:analytics:image")
                    analyticsdbImage = getValueForPillarKey(pepperEnv, "I@opencontrail:collector:role:primary", "docker:client:compose:opencontrail:service:analyticsdb:image")

                    // Pull new docker images
                    salt.runSaltProcessStep(pepperEnv, 'I@opencontrail:control', 'dockerng.pull', [controllerImage])
                    salt.runSaltProcessStep(pepperEnv, 'I@opencontrail:collector', 'dockerng.pull', [analyticsImage])
                    salt.runSaltProcessStep(pepperEnv, 'I@opencontrail:collector', 'dockerng.pull', [analyticsdbImage])

                } catch (Exception er) {
                    common.errorMsg("OpenContrail docker images failed be upgraded.")
                    throw er
                }

                try {
                    salt.runSaltProcessStep(pepperEnv, 'I@opencontrail:collector', 'cmd.shell', ['cd /etc/docker/compose/opencontrail/; docker-compose down'], null, true)

                    salt.runSaltProcessStep(pepperEnv, 'I@opencontrail:collector', 'state.sls', ['opencontrail', 'exclude=opencontrail.client'])
                    salt.runSaltProcessStep(pepperEnv, 'I@opencontrail:collector', 'state.sls', ['opencontrail.client'])

                    salt.enforceState(pepperEnv, 'I@opencontrail:collector', 'docker.client')
                    if (targetOcVersion == '4.1') {
                        sleep(15)
                        salt.runSaltProcessStep(pepperEnv, 'I@opencontrail:collector', 'cmd.shell', ["doctrail analyticsdb systemctl restart confluent-kafka"], null, true)
                    }
                    checkContrailServices(pepperEnv, targetOcVersion, 'I@opencontrail:collector')
                } catch (Exception er) {
                    common.errorMsg("OpenContrail Analytic nodes failed to be upgraded.")
                    throw er
                }

                try {
                    salt.runSaltProcessStep(pepperEnv, 'I@opencontrail:control:role:secondary', 'cmd.shell', ['cd /etc/docker/compose/opencontrail/; docker-compose down'], null, true)
                    for (service in config4Services) {
                        salt.runSaltProcessStep(pepperEnv, 'I@opencontrail:control:role:primary', 'cmd.shell', ["doctrail controller systemctl stop ${service}"], null, true)
                    }
                    salt.runSaltProcessStep(pepperEnv, 'I@opencontrail:control:role:secondary', 'state.sls', ['opencontrail', 'exclude=opencontrail.client'])

                    salt.enforceState(pepperEnv, 'I@opencontrail:control:role:secondary', 'docker.client')
                    checkContrailServices(pepperEnv, targetOcVersion, 'I@opencontrail:control:role:secondary')

                    sleep(120)

                    salt.runSaltProcessStep(pepperEnv, 'I@opencontrail:control:role:primary', 'cmd.shell', ['cd /etc/docker/compose/opencontrail/; docker-compose down'], null, true)
                    salt.runSaltProcessStep(pepperEnv, 'I@opencontrail:control:role:primary', 'state.sls', ['opencontrail', 'exclude=opencontrail.client'])

                    salt.enforceState(pepperEnv, 'I@opencontrail:control:role:primary', 'docker.client')
                    checkContrailServices(pepperEnv, targetOcVersion, 'I@opencontrail:control:role:primary')
                } catch (Exception er) {
                    common.errorMsg("OpenContrail Controller nodes failed to be upgraded.")
                    throw er
                }

                // Run opencontrail.client state once contrail-api is ready to service requests from clients
                salt.runSaltProcessStep(pepperEnv, 'I@opencontrail:control or I@opencontrail:collector', 'state.sls', ['opencontrail.client'])

                try {
                    salt.runSaltProcessStep(pepperEnv, 'I@neutron:server', 'pkg.install', [neutronServerPkgs])
                    salt.runSaltProcessStep(pepperEnv, 'I@horizon:server', 'pkg.install', [dashboardPanelPkg])
                    salt.runSaltProcessStep(pepperEnv, 'I@neutron:server', 'service.start', ['neutron-server'])
                    salt.enforceState(pepperEnv, 'I@horizon:server', 'horizon')
                } catch (Exception er) {
                    common.errorMsg("Update of packages on neutron and horizon nodes has been failed")
                    throw er
                }
            }
        }

        if (STAGE_COMPUTES_UPDATE.toBoolean() == true) {

            try {
                stage('List targeted compute servers') {
                    cmpMinions = salt.getMinions(pepperEnv, COMPUTE_TARGET_SERVERS)
                    cmpMinionsFirstSubset = cmpMinions[0..<Integer.valueOf(COMPUTE_TARGET_SUBSET_LIVE)]
                    cmpMinionsSecondSubset = cmpMinions - cmpMinionsFirstSubset

                    if (cmpMinions.isEmpty()) {
                        throw new Exception("No minions were found by specified target")
                    }

                    common.infoMsg("Found nodes: ${cmpMinions}")
                    common.infoMsg("Selected sample nodes: ${cmpMinionsFirstSubset}")

                    cmpTargetAll = cmpMinions.join(' or ')
                    cmpTargetFirstSubset = cmpMinionsFirstSubset.join(' or ')
                    cmpTargetSecondSubset = cmpMinionsSecondSubset.join(' or ')
                }

                stage('Compute nodes health check') {
                    try {
                        salt.enforceState(pepperEnv, cmpTargetAll, 'opencontrail.upgrade.verify', true, true)
                    } catch (Exception er) {
                        common.errorMsg("Opencontrail compute nodes health check stage found issues with services. Please take a look at the logs above.")
                        throw er
                    }
                }

                stage('Confirm update on sample nodes') {
                    if (askConfirmation) {
                        input message: "Do you want to continue with the Opencontrail components update on compute sample nodes? ${cmpTargetFirstSubset}"
                    }
                }

                stage("Opencontrail compute update on sample nodes") {

                    cmpNodesUpdate(pepperEnv, cmpTargetFirstSubset)
                }

                stage('Confirm update on all remaining target nodes') {
                    if (askConfirmation) {
                        input message: "Do you want to continue with the Opencontrail components update on all targeted compute nodes? Node list: ${cmpTargetSecondSubset}"
                    }
                }

                stage("Opencontrail compute update on all targeted nodes") {

                    cmpNodesUpdate(pepperEnv, cmpTargetSecondSubset)
                }

            } catch (Throwable e) {
                // If there was an error or exception thrown, the build failed
                currentBuild.result = "FAILURE"
                currentBuild.description = currentBuild.description ? e.message + " " + currentBuild.description : e.message
                throw e
            }
        }
    }
}