Jenkinsfile.cd

#!groovy

@Library('SovrinHelpers') _

def name = 'indy-node'

def nodeTestUbuntu = {
    try {
        echo 'Ubuntu Test: Checkout csm'
        checkout scm

        echo 'Ubuntu Test: Build docker image'
        def testEnv = dockerHelpers.build(name)

        testEnv.inside('--network host') {
            echo 'Ubuntu Test: Install dependencies'
            testHelpers.install()

            echo 'Ubuntu Test: Test'
            testHelpers.testRunner([resFile: "test-result-node.${NODE_NAME}.txt", testDir: 'indy_node'])
            //testHelpers.testJUnit(resFile: "test-result-node.${NODE_NAME}.xml")
        }
    }
    finally {
        echo 'Ubuntu Test: Cleanup'
        step([$class: 'WsCleanup'])
    }
}

def commonTestUbuntu = {
    try {
        echo 'Ubuntu Test: Checkout csm'
        checkout scm

        echo 'Ubuntu Test: Build docker image'
        def testEnv = dockerHelpers.build(name)

        testEnv.inside {
            echo 'Ubuntu Test: Install dependencies'
            testHelpers.install()

            echo 'Ubuntu Test: Test'
            testHelpers.testJUnit([resFile: "test-result-common.${NODE_NAME}.xml", testDir: 'indy_common'])
        }
    }
    finally {
        echo 'Ubuntu Test: Cleanup'
        step([$class: 'WsCleanup'])
    }
}

def buildDebUbuntu = { repoName, releaseVersion, sourcePath ->
    def volumeName = "$name-deb-u1604"
    if (env.BRANCH_NAME != '' && env.BRANCH_NAME != 'master') {
        volumeName = "${volumeName}.${BRANCH_NAME}"
    }
    if (sh(script: "docker volume ls -q | grep -q '^$volumeName\$'", returnStatus: true) == 0) {
        sh "docker volume rm $volumeName"
    }
    dir('build-scripts/ubuntu-1604') {
        sh "./build-$name-docker.sh \"$sourcePath\" $releaseVersion $volumeName"
        sh "./build-3rd-parties-docker.sh $volumeName"
    }
    return "$volumeName"
}

def systemTests = { component ->

    String prefix = "System Tests ($component)"
    String defaultNetwork = 'pool-network'
    int nodesNum = 4

    String indySDKVersion
    String indyCryptoVersion

    def dockerClean = { networkName ->
        networkName = networkName ?: defaultNetwork

        try {
            sh "docker ps -q --filter network=$networkName | xargs -r docker rm -f"
        } catch (Exception exc) {
            echo "$prefix: failed to remove docker containers in $networkName network: $exc"
            throw exc
        }

        try {
            sh "docker network ls -q --filter name=$networkName | xargs -r docker network rm"
        } catch (Exception exc) {
            echo "$prefix: failed to remove docker $networkName network: $exc"
            throw exc
        }

        sh "docker container prune -f"
        sh "docker network prune -f"
    }

    node("ubuntu") {
        stage("Cleanup docker") {
            dockerClean()
        }

        stage("Checkout SCM") {
            checkout scm
        }

        stage("Get versions for indy-crypto and indy-sdk") {
            def commonSedExprs = "-e 'tx' -e 'd' -e ':x' -e 'p' -e 'q'"
            def indyPlenumVersion = sh(returnStdout: true, script: """
                cat ./setup.py | sed -n -e 's/.*indy-plenum\\(-dev\\)\\?==\\([^"\\x27]\\+\\).*/\\2/' $commonSedExprs
            """).trim()
            echo "indy-plenum version: $indyPlenumVersion"
            indySDKVersion = sh(returnStdout: true, script: """
                cat ./setup.py | sed -n -e 's/.*python3-indy==\\([^"\\x27]\\+\\).*/\\1/' $commonSedExprs
            """).trim()
            echo "indy-sdk version: $indySDKVersion"
            indyCryptoVersion = sh(returnStdout: true, script: """
                wget -q https://raw.githubusercontent.com/hyperledger/indy-plenum/$indyPlenumVersion-${component == 'stable' ? 'rc' : component}/setup.py -O - | sed -n -e 's/.*indy-crypto==\\([^"\\x27]\\+\\).*/\\1/' $commonSedExprs
            """).trim()
            echo "indy-crypto version: $indyCryptoVersion"

            if (!(indySDKVersion && indyCryptoVersion)) {
                error "Failed to get versions for indy-crypto and indy-sdk"
            }
        }

        try {
            dir("environment/docker/pool"){
                stage("Patch core dockerfile") {
                    // TODO update dockerfile instead of patching
                    sh """
                        sed -i 's/repo.sovrin.org\\/deb xenial .*\\+"/repo.sovrin.org\\/deb xenial $component"/g' core.ubuntu.dockerfile
                        sed -i 's/sdk\\/deb xenial .\\+"/sdk\\/deb xenial stable"/g' core.ubuntu.dockerfile
                        sed -i 's/\\(apt-get install -y indy-node libindy\\)/&=$indySDKVersion python3-indy-crypto=$indyCryptoVersion libindy-crypto=$indyCryptoVersion/g' core.ubuntu.dockerfile
                        sed -i 's/\\(RUN pip3 install python3-indy\\)/&==$indySDKVersion pytest/g' core.ubuntu.dockerfile
                    """
                }

                stage("Run pool") {
                    echo "$prefix: Running nodes..."
                    sh "./pool_start.sh $nodesNum"
                    pool_nodes = sh(script: 'docker ps', returnStdout: true)
                    echo "$prefix: $pool_nodes"
                }

                stage("Ensure node services started") {
                    // TODO explore and fix the reason why they (mostly Node1) not always active
                    for (nodeNum = 1; nodeNum <= nodesNum; nodeNum++) {
                        for (service in ['indy-node', 'indy-node-control']) {
                            sh """
                                for i in 1 2 3; do
                                    echo "Ensure $service on node${nodeNum} is started (try #\$i)"
                                    docker exec -t -u 0 node${nodeNum} systemctl start $service
                                    if [ "\$(docker exec -u 0 node${nodeNum} systemctl is-active $service)" = "active" ]; then
                                        exit 0
                                    fi
                                done

                                echo 'Failed to start $service on node${nodeNum}'
                                exit 1
                            """
                        }
                    }
                }


                stage("Prepare and run client") {
                    sh '''
                        sed -i 's/\\(docker exec -it\\)/#&/g' client_start.sh
                        ./client_for_pool_start.sh
                    '''
                }
            }

            stage("Prepare tests") {
                // https://github.com/moby/moby/issues/34096
                // https://github.com/moby/moby/issues/34096
                // inconsistent docker behaviour, depends on version
                // thus, ensure ownership for copied files
                testHelpers.getSystemTests(targetDir: './system_tests')
                sh '''
                    docker cp ./system_tests/system indyclient:/home/indy
                    docker cp node1:/var/lib/indy/sandbox/pool_transactions_genesis ./
                    docker cp ./pool_transactions_genesis indyclient:/home/indy/system/docker_genesis
                    docker exec -t -u 0 indyclient chown -R indy:indy /home/indy/system
                '''
            }

            String testReportFileNameXml = "system_tests_report.${component}.xml"
            String testReportFileNamePlain = "system_tests_report.${component}.txt"
            try {
                stage("Run tests") {
                    sh """
                        docker exec -t --user indy indyclient bash -c "\
                            set -o pipefail; \
                            cd /home/indy/system && mkdir reports && \
                            pytest -v --junit-xml=./reports/$testReportFileNameXml test_ledger.py 2>&1 | \
                            tee ./reports/$testReportFileNamePlain"
                    """
                }
            } finally {
                stage("Upload test teports") {
                    sh "docker cp indyclient:/home/indy/system/reports/ ./system_tests"

                    dir("system_tests/reports"){
                        archiveArtifacts artifacts: testReportFileNamePlain, allowEmptyArchive: true
                        junit testResults: testReportFileNameXml, allowEmptyResults: true
                    }
                }
            }

        } catch (Exception exc) {
            echo "$prefix: fail: $exc"
            throw exc
        } finally {
            stage('Cleanup') {
                try {
                    dockerClean()
                } catch(Exception exc) {
                    // pass
                }

                cleanWs()
            }
        }
    }
}

options = new TestAndPublishOptions()
options.enable([StagesEnum.PACK_RELEASE_COPY, StagesEnum.PACK_RELEASE_COPY_ST])
options.setCopyWithDeps(true)
options.setSystemTestsCb(systemTests)
options.setPrContexts([env.INDY_GITHUB_PR_REQUIRED_CONTEXT ?: "ci/hyperledger-jenkins/pr-merge"])

testAndPublish(name, [ubuntu: [node: nodeTestUbuntu, common: commonTestUbuntu]], true, options, [ubuntu: buildDebUbuntu])