integ-test.jenkinsfile

lib = library(identifier: "jenkins@20211118", retriever: legacySCM(scm))

pipeline {
    options {
        timeout(time: 3, unit: 'HOURS')
    }
    agent none
    environment {
        BUILD_MANIFEST = "build-manifest.yml"
        DEFAULT_BUILD_JOB_NAME = "distribution-build-opensearch"
    }
    parameters {
        string(
            name: 'TEST_MANIFEST',
            description: 'Test manifest under the manifests folder, e.g. 2.0.0/opensearch-2.0.0-test.yml.',
            trim: true
        )
        string(
            name: 'BUILD_MANIFEST_URL',
            description: 'The build manifest URL, e.g. https://ci.opensearch.org/ci/dbc/distribution-build-opensearch/1.2.2/98/linux/x64/builds/opensearch/manifest.yml.',
            trim: true
        )
        string(
            name: 'AGENT_LABEL',
            description: 'The agent label where the tests should be executed. For x64 use Jenkins-Agent-AL2-X64-C54xlarge-Docker-Host,for arm64 use Jenkins-Agent-AL2-Arm64-C6g4xlarge-Docker-Host',
            trim: true
        )
    }
    stages {
        stage('verify-parameters') {
            agent { label AGENT_LABEL }
            steps {
                script {
                    currentBuild.description = BUILD_MANIFEST_URL
                    if (AGENT_LABEL == '') {
                        currentBuild.result = 'ABORTED'
                        error("Integration Tests failed to start. Missing parameter: AGENT_LABEL.")
                    }
                    if (TEST_MANIFEST == '' || !fileExists("manifests/${TEST_MANIFEST}")) {
                        currentBuild.result = 'ABORTED'
                        error("Integration Tests failed to start. Test manifest was not provided or not found in manifests/${TEST_MANIFEST}.")
                    }
                    /*
                    Rebuilding of this job will result in considering upstream build as self($JOB_NAME) See https://issues.jenkins.io/browse/JENKINS-61590 for bug
                    Either trigger from expected upstream job or run a new build
                    */
                    env.BUILD_JOB_NAME = currentBuild.upstreamBuilds ? 
                        currentBuild.upstreamBuilds[0].fullProjectName : 
                        env.DEFAULT_BUILD_JOB_NAME
                }
            }
        }
        stage('detect docker image + args') {
            agent {
                docker {
                    label AGENT_LABEL
                    image 'alpine:3'
                    alwaysPull true
                }
            }
            steps {
                script { 
                    DOCKER_AGENT = detectTestDockerAgent()
                }
            }
        }
        stage('integ-test') {
            // Required running on agent directly here to trigger docker stages in agent node, not trigger docker within docker container
            // Can only be run in runner that is at least 50GB per container
            agent { label AGENT_LABEL }
            steps {
                script {

                    downloadBuildManifest(
                        url: BUILD_MANIFEST_URL,
                        path: BUILD_MANIFEST
                    )

                    // Stash the current working directory files, aka opensearch-build repo
                    // Unstash later in each triggered stage to run integTest
                    stash includes: '**', name: 'opensearch-build-repo'

                    def buildManifestObj = lib.jenkins.BuildManifest.new(readYaml(file: BUILD_MANIFEST))
                    def componentList = buildManifestObj.getNames()
                    echo "componentList: ${componentList}"

                    componentTests = [:]

                    for (component in componentList) {
                        // Must use local variable due to groovy for loop and closure scope
                        // Or the stage will be fixed to the last item in return when new stages are triggered here
                        // https://web.archive.org/web/20181121065904/http://blog.freeside.co/2013/03/29/groovy-gotcha-for-loops-and-closure-scope/
                        def local_component = component

                        echo "Add Component: ${local_component}"
                        componentTests["Run Integtest ${local_component}"] = {
                            // Using scripted pipelines to trigger dynamic parallel stages
                            timeout(time: 2, unit: 'HOURS') {
                                node(AGENT_LABEL) {
                                    docker.image(DOCKER_AGENT.image).inside(DOCKER_AGENT.args) {
                                        try {
                                            stage("Run Integtest ${local_component}") {
                                                echo "Component Name: ${local_component}"
                                                unstash 'opensearch-build-repo'
                                                // Jenkins tend to not clean up workspace at times even though ws clean is called
                                                // Due to docker is mounting the agent directory so it can communicated with the agent
                                                // This sometimes causes the workspace to retain last run test-results and ends with build failures
                                                // https://github.com/opensearch-project/opensearch-build/blob/6ed1ce3c583233eae4fe1027969d778cfc7660f7/src/test_workflow/test_recorder/test_recorder.py#L99
                                                sh("rm -rf test-results")
                                                runIntegTestScript(
                                                    jobName: BUILD_JOB_NAME,
                                                    componentName: "${local_component}",
                                                    buildManifest: BUILD_MANIFEST,
                                                    testManifest: "manifests/${TEST_MANIFEST}"
                                                )
                                            }
                                        } catch (e) {
                                            echo "Error running integtest for component ${local_component}"
                                            throw e
                                        } finally {
                                            uploadTestResults(
                                                buildManifestFileName: BUILD_MANIFEST,
                                                jobName: JOB_NAME
                                            )
                                            postCleanup()
                                        }
                                    }
                                }
                            }
                        }
                    }
                    parallel componentTests
                }
            }
            post {
                always {
                    postCleanup()
                }
            }
        }
    }

    post {
        success {
            node(AGENT_LABEL) {
                script {
                    def stashed = lib.jenkins.Messages.new(this).get(['integ-test'])
                    publishNotification(
                        icon: ':white_check_mark:',
                        message: 'Integration Tests Successful',
                        extra: stashed,
                        credentialsId: 'jenkins-integ-test-webhook',
                        manifest: TEST_MANIFEST,
                    )

                    postCleanup()
                }
            }
        }
        failure {
            node(AGENT_LABEL) {
                script  {
                    def stashed = lib.jenkins.Messages.new(this).get(['integ-test'])
                    publishNotification(
                        icon: ':warning:',
                        message: 'Failed Integration Tests',
                        extra: stashed,
                        credentialsId: 'jenkins-integ-test-webhook',
                        manifest: TEST_MANIFEST,
                    )

                    postCleanup()
                }
            }
        }
    }
}