Test cleaning pipeline for resubmission (#1292)

.github/workflows/test-environment-cleaner-validation.yml

@@ -0,0 +1,27 @@
+name: Validate test environment cleaner
+
+on:
+  pull_request:
+    paths:
+      - 'jenkins_pipelines/scripts/test_environment_cleaner/**'
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Validate api manager script
+        uses: actions/checkout@v2
+
+      - name: Set up Python 3.11
+        uses: actions/setup-python@v2
+        with:
+          python-version: '3.11'
+
+      - name: Install dependencies
+        run: |
+          python3.11 -m pip install --upgrade pip
+          pip3.11 install -r jenkins_pipelines/scripts/test_environment_cleaner/requirements.txt
+
+      - name: Run tests
+        run: cd jenkins_pipelines/scripts/test_environment_cleaner; ./run_tests.sh

jenkins_pipelines/environments/common/pipeline-build-validation-cleanup.groovy

@@ -0,0 +1,245 @@
+def run(params) {
+    timestamps {
+        // Define paths and environment variables for reusability
+        GString TestEnvironmentCleanerProgram = "${WORKSPACE}/susemanager-ci/jenkins_pipelines/scripts/test_environment_cleaner/test_environment_cleaner_program/TestEnvironmentCleaner.py"
+        GString resultdir = "${WORKSPACE}/results"
+        GString resultdirbuild = "${resultdir}/${BUILD_NUMBER}"
+        GString exports = "export BUILD_NUMBER=${BUILD_NUMBER}; export BUILD_VALIDATION=true; "
+        String container_repository = params.container_repository ?: null
+        String product_version = null
+        String controllerHostname = null
+        String serverHostname = null
+        GString targetedTfFile = "${WORKSPACE}/../${params.targeted_project}/results/sumaform/main.tf"
+        GString targetedTfStateFile = "${WORKSPACE}/../${params.targeted_project}/results/sumaform/terraform.tfstate"
+        GString targetedTerraformDirPath = "${WORKSPACE}/../${params.targeted_project}/results/sumaform/"
+        GString localSumaformDirPath = "${resultdir}/sumaform/"
+        GString localTfStateFile = "${localSumaformDirPath}/terraform.tfstate"
+        GString logFile = "${resultdirbuild}/sumaform.log"
+
+        // Construct the --tf-resources-to-delete argument dynamically
+        ArrayList defaultResourcesToDelete = []
+        if (params.clean_proxy) {
+            defaultResourcesToDelete.add('proxy')
+        }
+        if (params.clean_monitoring_server) {
+            defaultResourcesToDelete.add('monitoring-server')
+        }
+        if (params.clean_retail) {
+            defaultResourcesToDelete.add('retail')
+        }
+
+        String defaultResourcesToDeleteArgs = defaultResourcesToDelete.isEmpty() ? '' : "--default-resources-to-delete ${defaultResourcesToDelete.join(' ')}"
+
+        GString commonParams = "--outputdir ${resultdir} --tf ${targetedTfFile} --gitfolder ${resultdir}/sumaform"
+
+        // Define shared environment variables for terraform calls
+        GString environmentVars = """
+                set -x
+                source /home/jenkins/.credentials
+                export TF_VAR_CONTAINER_REPOSITORY=${container_repository}
+                export TERRAFORM=${terraform_bin}
+                export TERRAFORM_PLUGINS=${terraform_bin_plugins}
+            """
+
+        try {
+            stage('Clone terracumber, susemanager-ci and sumaform') {
+
+                // Prevent rebuild option
+                if (currentBuild.getBuildCauses().toString().contains("RebuildCause")) {
+                    error "Rebuild is blocked for this job."
+                }
+
+                // Create a directory to store the build results (if it does not exist). Needed for first run
+                sh "mkdir -p ${resultdir}"
+                git url: params.terracumber_gitrepo, branch: params.terracumber_ref
+                dir("susemanager-ci") {
+                    checkout scm
+                }
+
+                // Clone sumaform
+                sh "set +x; source /home/jenkins/.credentials set -x; ${WORKSPACE}/terracumber-cli ${commonParams} --gitrepo ${params.sumaform_gitrepo} --gitref ${params.sumaform_ref} --runstep gitsync"
+
+                // Set product version
+                if (params.targeted_project.contains("5.0")) {
+                    product_version = '5.0'
+                } else if (params.targeted_project.contains("4.3")) {
+                    product_version = '4.3'
+                } else if (params.targeted_project.contains("uyuni")) {
+                    product_version = 'uyuni'
+                } else if (params.targeted_project.contains("5.1")) {
+                    product_version = '5.1'
+                } else if (params.targeted_project.contains("head")) {
+                    product_version = 'head'
+                }
+                else {
+                    // Use the `error` step instead of `throw`
+                    error("Error: targeted_project must contain either 'head', '5.1', '5.0', '4.3' or uyuni.")
+                }
+            }
+
+            stage('Confirm Environment Cleanup') {
+                // Ask the user what environment they are cleaning, ensuring the answer matches params.targeted_project
+                def environmentChoice = input(
+                        message: 'What environment are you cleaning?',
+                        parameters: [
+                                string(name: 'Environment_Name', description: 'Enter the name of the environment you are cleaning.')
+                        ]
+                )
+
+                // Validate that the user entered the correct environment
+                if (environmentChoice != params.targeted_project) {
+                    error("The environment name entered does not match the targeted project. Aborting pipeline.")
+                }
+            }
+
+            stage("Copy terraform files from ${params.targeted_project}"){
+                // Copy tfstate and terraform directory to the result directory
+                sh """
+                    cp ${targetedTfStateFile} ${localTfStateFile}
+                    cp -r ${targetedTerraformDirPath}.terraform ${localSumaformDirPath}
+                """
+
+            }
+
+            stage("Extract the controller and server hostname") {
+                try {
+                    controllerHostname = sh(
+                            script: """
+                            set -e
+                            cd ${localSumaformDirPath}
+                            terraform output -json configuration | jq -r '.controller.hostname'
+                        """,
+                            returnStdout: true
+                    ).trim()
+
+                    serverHostname = sh(
+                            script: """
+                            set -e
+                            cd ${localSumaformDirPath}
+                            terraform output -json configuration | jq -r '.server.hostname'
+                        """,
+                            returnStdout: true
+                    ).trim()
+
+                    // Print the values for confirmation
+                    echo "Extracted controller hostname: ${controllerHostname}"
+                    echo "Extracted server hostname: ${serverHostname}"
+
+                } catch (Exception e) {
+                    error("Failed to extract hostnames: ${e.message}")
+                }
+
+            }
+
+            GString programCall = "${TestEnvironmentCleanerProgram} --url ${serverHostname} --product_version ${product_version} ${defaultResourcesToDeleteArgs} --mode"
+
+            stage('Delete the systems') {
+                sh(script: "${programCall} delete_systems")
+            }
+            stage('Delete config projects') {
+                sh(script: "${programCall} delete_config_projects")
+            }
+            stage('Delete software channels') {
+                sh(script: "${programCall} delete_software_channels")
+            }
+            stage('Delete activation keys') {
+                sh(script: "${programCall} delete_activation_keys")
+            }
+            stage('Delete minion users') {
+                sh(script: "${programCall} delete_users")
+            }
+            stage('Delete channel repositories') {
+                sh(script: "${programCall} delete_repositories")
+            }
+            stage('Delete salt keys') {
+                sh(script: "${programCall} delete_salt_keys")
+            }
+
+            stage('Delete ssh know hosts') {
+                sh(script: "${TestEnvironmentCleanerProgram} --url ${serverHostname} --product_version ${product_version} --mode delete_known_hosts")
+            }
+
+            stage('Delete distributions folders') {
+                sh(script: "${TestEnvironmentCleanerProgram} --url ${serverHostname} --product_version ${product_version} --mode delete_distributions")
+            }
+
+            stage('Delete client VMs') {
+
+                // Construct the --tf-resources-to-delete argument dynamically
+                ArrayList tfResourcesToDelete = []
+                if (params.clean_proxy) {
+                    tfResourcesToDelete.add('proxy')
+                }
+                if (params.clean_monitoring_server) {
+                    tfResourcesToDelete.add('monitoring-server')
+                }
+                if (params.clean_retail) {
+                    tfResourcesToDelete.add('retail')
+                }
+
+                // Join the resources into a comma-separated string if there are any to delete
+                String tfResourcesToDeleteArg = defaultResourcesToDelete.isEmpty() ? '' : "--tf-resources-to-delete ${defaultResourcesToDelete.join(' ')}"
+
+                // Execute Terracumber CLI to deploy the environment without clients
+                sh """
+                    ${environmentVars}
+                    set +x
+                    ${WORKSPACE}/terracumber-cli ${commonParams} --logfile ${logFile} --init --sumaform-backend ${sumaform_backend} --use-tf-resource-cleaner --init --runstep provision ${tfResourcesToDeleteArg}
+                """
+            }
+
+            stage('Redeploy the environment with new client VMs') {
+
+                // Run Terracumber to deploy the environment
+                sh """
+                    ${environmentVars}
+                    set +x
+                    ${WORKSPACE}/terracumber-cli ${commonParams} --logfile ${resultdirbuild}/sumaform.log --init --sumaform-backend ${sumaform_backend} --runstep provision
+                """
+            }
+
+            stage('Copy the new custom repository json file to controller') {
+                if (params.push_new_custom_repositories) {
+                    // Generate custom_repositories.json file in the workspace from the value passed by parameter
+                    if (params.custom_repositories?.trim()) {
+                        writeFile file: 'custom_repositories.json', text: params.custom_repositories, encoding: "UTF-8"
+                    }
+
+                    // Generate custom_repositories.json file in the workspace using a Python script - MI Identifiers passed by parameter
+                    if (params.mi_ids?.trim()) {
+                        node('manager-jenkins-node') {
+                            checkout scm
+                            res_python_script_ = sh(script: "python3 jenkins_pipelines/scripts/json_generator/maintenance_json_generator.py --mi_ids ${params.mi_ids}", returnStatus: true)
+                            echo "Build Validation JSON script return code:\n ${json_content}"
+                            if (res_python_script != 0) {
+                                error("MI IDs (${params.mi_ids}) passed by parameter are wrong (or already released)")
+                            }
+                        }
+                    }
+                    sh(script: "${TestEnvironmentCleanerProgram} --url ${controllerHostname} --product_version ${product_version} --mode update_custom_repositories")
+                }
+            }
+
+            stage('Sanity check') {
+                sh "${WORKSPACE}/terracumber-cli ${commonParams} --logfile ${resultdirbuild}/testsuite.log --runstep cucumber --cucumber-cmd 'cd /root/spacewalk/testsuite; ${exports} rake cucumber:build_validation_sanity_check'"
+            }
+
+        }
+        finally {
+            stage('Copy back tfstate') {
+                catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') {
+                    sh "cp ${localTfStateFile} ${targetedTfStateFile}"
+                }
+            }
+
+            stage('Rename tfstate to avoid copying it between runs') {
+                catchError(buildResult: 'FAILURE', stageResult: 'FAILURE') {
+                    sh "mv ${localTfStateFile} ${localTfStateFile}.old"
+                    sh "cp ${localTfStateFile}.old ${resultdirbuild}"
+                }
+            }
+        }
+    }
+}
+
+return this

jenkins_pipelines/environments/qe-build-validation-cleaning-pipeline

@@ -0,0 +1,57 @@
+#!/usr/bin/env groovy
+
+node {
+    // Default node
+    String nodeName = 'sumaform-cucumber'
+
+    // Check the value of targeted_project and adjust the nodeName accordingly
+    if (params.targeted_project.contains('PRV')) {
+        nodeName = 'sumaform-cucumber-provo'  // Use this for PRV projects
+    } else if (params.targeted_project.contains('NUE')) {
+        nodeName = 'sumaform-cucumber'  // Use this for NUE projects
+    }
+
+    // Run on the selected node
+    node(nodeName) {
+        properties([
+                buildDiscarder(logRotator(numToKeepStr: '5', artifactNumToKeepStr: '3')),
+                disableConcurrentBuilds(),
+                parameters([
+                        choice(name: 'targeted_project', choices: [
+                                'default',
+                                'manager-4.3-qe-build-validation-NUE',
+                                'manager-5.0-qe-build-validation-NUE',
+                                'manager-5.0-qe-build-validation-weekly-NUE',
+                                'uyuni-master-qe-build-validation-NUE',
+                                'manager-4.3-qe-build-validation-PRV',
+                                'manager-5.0-qe-build-validation-PRV',
+                                'uyuni-master-qe-build-validation-PRV'
+                        ], description: 'Path to the tf file to be used'),
+                        string(name: 'sumaform_gitrepo', defaultValue: 'https://github.com/uyuni-project/sumaform.git', description: 'Sumaform Git Repository'),
+                        string(name: 'sumaform_ref', defaultValue: 'master', description: 'Sumaform Git reference (branch, tag...)'),
+                        string(name: 'terracumber_gitrepo', defaultValue: 'https://github.com/uyuni-project/terracumber.git', description: 'Terracumber Git Repository'),
+                        string(name: 'terracumber_ref', defaultValue: 'master', description: 'Terracumber Git ref (branch, tag...)'),
+                        string(name: 'container_repository', defaultValue: 'registry.suse.de/suse/sle-15-sp6/update/products/manager50/update/containerfile', description: 'Proxy and server container registry'),
+                        booleanParam(name: 'clean_proxy', defaultValue: false, description: 'Clean Proxy, will remove all the artefacts related to Proxy and redeploy Proxy'),
+                        booleanParam(name: 'clean_monitoring_server', defaultValue: false, description: 'Clean Monitoring Server, will remove all the artefacts related to Monitoring Server and redeploy Monitoring Server'),
+                        booleanParam(name: 'clean_retail', defaultValue: false, description: 'Clean Retails artefacts, and redeploy terminal and dhcp if applicable'),
+                        booleanParam(name: 'push_new_custom_repositories', defaultValue: false, description: 'Push the new custom_repositories.json to controller'),
+                        text(name: 'mi_ids', defaultValue: '', description: 'MI Identifiers separated by comma or whitespaces (Option A)'),
+                        text(name: 'custom_repositories', defaultValue: '{}', description: 'MU Repositories in json format (Option B)')
+                ])
+        ])
+
+        stage('Checkout pipeline') {
+            checkout scm
+        }
+
+        // Define environment variables
+        env.sumaform_backend = 'libvirt'
+        env.terraform_bin = '/usr/bin/terraform'
+        env.terraform_bin_plugins = '/usr/bin'
+
+        // Load and run the pipeline
+        def pipeline = load "jenkins_pipelines/environments/common/pipeline-build-validation-cleanup.groovy"
+        pipeline.run(params)
+    }
+}

jenkins_pipelines/scripts/test_environment_cleaner/README.md

@@ -0,0 +1,37 @@
+# SUSE Manager API and SSH Management Script
+
+This Python script provides a command-line interface to manage various resources and configurations on a SUSE Manager server through the XML-RPC API and SSH commands. 
+The script enables testers to perform resource cleanup, SSH-based file management, and update custom repositories remotely.
+
+## Prerequisites
+
+- **Python 3.11**: Ensure Python 3.11 is installed.
+- **Dependencies**: Install the required packages using:
+
+```bash
+pip3.11 install -r test_environment_cleaner/requirements.txt
+```
+
+## Usage
+
+```commandline
+python3.11 TestEnvironmentCleaner.py --url <SUSE_Manager_URL> --mode <operation_mode> [options]
+```
+
+### Command-Line Arguments
+
+ - `--url`: (Required) URL of the SUSE Manager XML-RPC API.
+ - `--mode`: (Required) Operation mode. Choose one of the following:
+    `delete_users`: Deletes user accounts.
+    `delete_activation_keys`: Deletes activation keys.
+    `delete_config_projects`: Deletes configuration projects.
+    `delete_software_channels`: Deletes software channels.
+    `delete_systems`: Deletes managed systems.
+    `delete_repositories`: Deletes repositories.
+    `delete_salt_keys`: Deletes Salt keys.
+    `full_cleanup`: Runs a complete cleanup of selected resources.
+    `delete_distributions`: Deletes distributions from the server.
+    `delete_known_hosts`: Deletes known SSH hosts in server.
+    `update_custom_repositories`: Updates custom repositories in controller.
+ - `--default-resources-to-delete`: Optional list of resources (proxy, monitoring-server, retail) to enforce deletion during API cleanup operations.
+ - `--product_version`: SUSE Manager version (head, 5.1, 5.0, 4.3 or uyuni). Used for handling different paths in specific operations.

jenkins_pipelines/scripts/test_environment_cleaner/requirements.txt

@@ -0,0 +1,3 @@
+argparse
+logging
+paramiko

jenkins_pipelines/scripts/test_environment_cleaner/run_tests.sh

@@ -0,0 +1,7 @@
+#!/bin/bash
+
+# Set the PYTHONPATH to include the necessary directories
+export PYTHONPATH=$(pwd)/test_environment_cleaner_program
+
+# Run the tests
+python3.11 -m unittest discover -s tests